From 1641bbfbab021bfcca21a6b10cda6bf4e3e2410d Mon Sep 17 00:00:00 2001 From: Denis Zheleztsov Date: Thu, 22 Jun 2017 17:06:39 +0300 Subject: [PATCH] Memcached support --- README.md | 8 +++++++ build.sh | 2 +- main.go | 36 ++++++++++++++++++++++-------- rest/cache.go | 58 ++++++++++++++++++++++++++++++++++++++++++++++++ rest/zoo.go | 61 ++++++++++++++++++++++++++++++++++++++++++++++----- 5 files changed, 150 insertions(+), 15 deletions(-) create mode 100644 rest/cache.go diff --git a/README.md b/README.md index b8b75d3..8764b7a 100644 --- a/README.md +++ b/README.md @@ -36,12 +36,20 @@ Zookeeper HTTP rest API Usage of ./zoorest: -listen string Address to listen (default "127.0.0.1:8889") + -mc + Enable memcached support + -mchosts string + Memcached servers. Comma separated (default "127.0.0.1:11211") + -mcprefix string + Memcached key prefix (default "zoorest") -path string Zk root path (default "/") -zk string Zk servers. Comma separated (default "127.0.0.1:2181") ``` +NOTE: If memcached support is enabled zkStat metrics will not be returned. + ## API ### List node childrens diff --git a/build.sh b/build.sh index fddaa68..2692acd 100755 --- a/build.sh +++ b/build.sh @@ -16,7 +16,7 @@ ENTRYPOINT go build -ldflags "-linkmode external -extldflags -static" && mv zoor EOF # Build builder -docker build -t zoorest_builder -f Dockerfile.builder . +docker build --no-cache -t zoorest_builder -f Dockerfile.builder . # Build bin docker run -v $(pwd)/out:/out zoorest_builder diff --git a/main.go b/main.go index e07c85b..71c9780 100644 --- a/main.go +++ b/main.go @@ -7,9 +7,12 @@ import ( ) var ( - zk string - listen string - path string + zk string + listen string + path string + mc bool + mcHosts string + mcPrefix string ) // init ... @@ -17,24 +20,39 @@ func init() { flag.StringVar(&zk, "zk", "127.0.0.1:2181", "Zk servers. Comma separated") flag.StringVar(&listen, "listen", "127.0.0.1:8889", "Address to listen") flag.StringVar(&path, "path", "/", "Zk root path") + flag.BoolVar(&mc, "mc", false, "Enable memcached support") + flag.StringVar(&mcHosts, "mchosts", "127.0.0.1:11211", "Memcached servers. Comma separated") + flag.StringVar(&mcPrefix, "mcprefix", "zoorest", "Memcached key prefix") flag.Parse() } // main ... func main() { var z rest.Zk - hosts := strings.Split(zk, ",") + hosts := getSlice(zk, ",") z.Hosts = hosts conn, err := z.InitConnection() if err != nil { panic(err) } - zoo := rest.ZooNode{ - path, - conn, - z, - } + var zoo rest.ZooNode + zoo.Path = path + zoo.Conn = conn + zoo.Zoo = z + + var MC rest.MC + MC.Hosts = getSlice(mcHosts, ",") + MC.Prefix = mcPrefix + MC.Enabled = mc + MC.Client = MC.InitConnection() + + zoo.MC = MC rest.Serve(listen, zoo) } + +// getSlice returm slice +func getSlice(s string, delimeter string) []string { + return strings.Split(s, ",") +} diff --git a/rest/cache.go b/rest/cache.go new file mode 100644 index 0000000..b5258ed --- /dev/null +++ b/rest/cache.go @@ -0,0 +1,58 @@ +package rest + +import ( + "crypto/md5" + "github.com/bradfitz/gomemcache/memcache" + "strings" +) + +// Memcached connection settings +type MC struct { + Enabled bool `json:"enabled"` + Hosts []string `json:"hosts"` + Prefix string `json:"prefix"` + Client *memcache.Client +} + +// InitConnection to memcached cluster +func (mc MC) InitConnection() *memcache.Client { + client := memcache.New(mc.Hosts...) + return client +} + +// StoreToCache save data to memecache +func (mc MC) StoreToCache(key string, data []byte) error { + prefixed_key := strings.Join([]string{mc.Prefix, key}, "_") + + var item memcache.Item + // item.Key = hashKey(prefixed_key) + item.Key = prefixed_key + item.Value = data + + // store + err := mc.Client.Set(&item) + + return err +} + +// (mc MC) GetFromCache ... +func (mc MC) GetFromCache(key string) ([]byte, error) { + var data *memcache.Item + prefixed_key := strings.Join([]string{mc.Prefix, key}, "_") + + // data, err := mc.Client.Get(hashKey(prefixed_key)) + data, err := mc.Client.Get(prefixed_key) + if err != nil { + return []byte(""), err + } + + return data.Value, err +} + +// hashKey return md5sum of prefixed_key +func hashKey(key string) string { + h := md5.New() + sum := h.Sum([]byte(key)) + + return string(sum) +} diff --git a/rest/zoo.go b/rest/zoo.go index 47b2ab7..12cb503 100644 --- a/rest/zoo.go +++ b/rest/zoo.go @@ -12,6 +12,7 @@ type ZooNode struct { Path string Conn *zk.Conn Zoo Zk + MC MC } //Zk Zookeeper connection settings @@ -43,10 +44,23 @@ func (z ZooNode) GetChildrens(path string) Ls { lsPath = strings.Replace(lsPath, "//", "/", 1) } - log.Print("ls: ", lsPath) + log.Print("V1 LS: ", lsPath) var l Ls l.State = "OK" + l.Path = path + + if z.MC.Enabled { + data, err := z.MC.GetFromCache(lsPath) + if err != nil { + log.Print("V1 LS ERROR: ", err.Error()) + } else { + log.Print("We are get it from memecache!") + childrens := strings.Split(string(data), ",") + l.Childrens = childrens + return l + } + } childrens, zkStat, err := z.Conn.Children(lsPath) if err != nil { @@ -55,15 +69,22 @@ func (z ZooNode) GetChildrens(path string) Ls { return l } + // Store to cache + if z.MC.Enabled { + err := z.MC.StoreToCache(lsPath, []byte(strings.Join(childrens, ","))) + if err != nil { + log.Print("V1 LS: ", err.Error()) + } + } + l.Error = "" l.Childrens = childrens - l.Path = path l.ZkStat = zkStat return l } -// GetNode ... +// GetNode data func (z ZooNode) GetNode(path string) Get { var getPath string getPath = strings.Join([]string{z.Path, path}, "") @@ -75,10 +96,21 @@ func (z ZooNode) GetNode(path string) Get { getPath = strings.Replace(getPath, "//", "/", 1) } - log.Print("get: ", getPath) + log.Print("V1 GET: ", getPath) var g Get g.State = "OK" + g.Path = path + + // Get data from memcached + if z.MC.Enabled { + if data, err := z.MC.GetFromCache(getPath); err != nil { + log.Print("V1 GET: ", err.Error()) + } else { + g.Data = data + return g + } + } data, zkStat, err := z.Conn.Get(getPath) if err != nil { @@ -87,9 +119,16 @@ func (z ZooNode) GetNode(path string) Get { return g } + // Store to cache + if z.MC.Enabled { + err := z.MC.StoreToCache(getPath, data) + if err != nil { + log.Print("V1 LS: ", err.Error()) + } + } + g.Error = "" g.Data = data - g.Path = path g.ZkStat = zkStat return g @@ -149,6 +188,12 @@ func (z ZooNode) UpdateNode(path string, content []byte) string { return err.Error() } + if z.MC.Enabled { + if err := z.MC.StoreToCache(upPath, content); err != nil { + log.Print("V1 update ERROR: ", err.Error()) + } + } + return path } @@ -167,6 +212,12 @@ func (z ZooNode) CreateChild(path string, content []byte) string { return err.Error() } + if z.MC.Enabled { + if err := z.MC.StoreToCache(crPath, content); err != nil { + log.Print("V1 create ERROR: ", err.Error()) + } + } + return path }