FIX: Check mount before map image

* Most logs moved to Sirupsen/logrus package

* Rewrite EnsureZooPath function

.travis.yml

@@ -0,0 +1,14 @@
+language: go
+go:
+  - "1.6"
+  - "1.7.x"
+  - "1.8"
+  - "1.9"
+
+before_install:
+  - go get github.com/gorilla/websocket
+  - go get github.com/samuel/go-zookeeper/zk
+
+# tests script
+script:
+  - go build -o _rbmd

README.md

@@ -1,7 +1,17 @@
 # RBMD
 
+[![Build Status](https://travis-ci.org/Difrex/rbmd.svg?branch=master)](https://travis-ci.org/Difrex/rbmd)
+
+![logo](/img/logo.png)
+
 RBD mount wrapper cluster
 
+**ALLOW FOR STAGING**
+
+Current status: *development*, *testing*
+
+**ALLOW FOR STAGING**
+
 <!-- markdown-toc start - Don't edit this section. Run M-x markdown-toc-generate-toc again -->
 **Table of Contents**
 
@@ -11,18 +21,23 @@ RBD mount wrapper cluster
         - [Example](#example)
     - [Build](#build)
     - [API](#api)
-        - [GET /status](#get-status)
+        - [GET /v1/status](#get-v1status)
             - [Example](#example)
-        - [GET /node](#get-node)
+        - [GET /v1/node](#get-v1node)
             - [Example](#example)
-        - [GET /health](#get-health)
+        - [GET /v1/health](#get-v1health)
             - [Example](#example)
-        - [POST /mount](#post-mount)
+        - [POST /v1/mount](#post-v1mount)
             - [Example](#example)
-        - [POST /umount](#post-umount)
+        - [POST /v1/umount](#post-v1umount)
             - [Example](#example)
-        - [POST /resolve](#post-resolve)
+        - [POST /v1/resolve](#post-v1resolve)
             - [Example](#example)
+        - [GET /v1/metrics](#get-v1metrics)
+            - [Example](#example)
+    - [Systemd](#systemd)
+- [Troubleshooting](#troubleshooting)
+    - [Umount request is stuck after deadly.](#umount-request-is-stuck-after-deadly)
 - [AUTHORS](#authors)
 - [LICENSE](#license)
 
@@ -39,11 +54,13 @@ RBD mount wrapper cluster
 ```
 Usage of ./rbmd:
   -listen string
-    	HTTP API listen address (default "0.0.0.0:9076")
+    	HTTP API listen address (default "127.0.0.1:9076")
   -tick int
     	Tick time loop (default 5)
+  -version
+    	Show version info and exit
   -ws string
-    	Websockets listen address (default "0.0.0.0:7690")
+    	Websockets listen address (default "127.0.0.1:7690")
   -zk string
     	Zookeeper servers comma separated (default "127.0.0.1:2181")
   -zkPath string
@@ -60,14 +77,13 @@ Required Go > 1.6
 
 ```
 git clone https://github.com/rbmd/rbmd.git && cd rbmd
-GOPATH=$(pwd) go get github.com/gorilla/websocket
-GOPATH=$(pwd) go get github.com/samuel/go-zookeeper/zk
+GOPATH=$(pwd) go get -t -v ./...
 GOPATH=$(pwd) go build
 ```
 
 ## API
 
-### GET /status
+### GET /v1/status
 
 Return JSON of quorum status
 
@@ -95,14 +111,14 @@ Return JSON of quorum status
 }
 ```
 
-### GET /node
+### GET /v1/node
 
 Return JSON of node stats 
 
 #### Example
 ```json
 {
-  "node": "difrex-mac.wargaming.net",
+  "node": "node.example.com",
   "ip": {
     "v4": [
       "169.254.156.1"
@@ -116,7 +132,7 @@ Return JSON of node stats
 }
 ```
 
-### GET /health
+### GET /v1/health
 
 Return string with quorum health check result
 
@@ -132,10 +148,18 @@ curl 127.0.0.1:9076/health
 alive.
 ```
 
-### POST /mount
+### POST /v1/mount
 
 Map rbd image and mount it
 
+Allowed mount options:
+ * ro
+ * noatime
+ * relatime
+ * nosuid
+ * noexec
+ * nodiratime
+
 #### Example
 
 Accept JSON
@@ -168,7 +192,7 @@ On failure
 }
 ```
 
-### POST /umount
+### POST /v1/umount
 
 Unmount filesystem and unmap RBD device
 
@@ -196,12 +220,12 @@ On success
 On failure
 ```json
 {
-    "state": "FAIL"
+    "state": "FAIL",
     "message": "Not found"
 }
 ```
 
-### POST /resolve
+### POST /v1/resolve
 
 Remove deadly node from quorum.
 
@@ -214,6 +238,44 @@ Accept JSON
 }
 ```
 
+### GET /v1/metrics
+
+Return some metrics
+
+#### Example
+
+```json
+{
+  "goroutines": 9,
+  "nodes": 2,
+  "mountstotal": 0,
+  "cgocall": 1
+}
+```
+
+## Systemd
+
+Example unit
+```ini
+[Unit]
+Description=RBMD
+After=network.target
+
+[Service]
+ExecStart=/usr/bin/rbmd -listen 0.0.0.0:9076 -ws 0.0.0.0:7690 -zk node1:2181,node2:2181,node3:2181
+KillMode=control-group
+Restart=on-failure
+
+[Install]
+WantedBy=multi-user.target
+```
+
+# Troubleshooting
+
+## Umount request is stuck after deadly.
+
+Remove Zk node */rbmd/cluster/node.fqdn/requests/umount*
+
 # AUTHORS
 
 Denis Zheleztsov <difrex.punk@gmail.com>

_config.yml

@@ -0,0 +1 @@
+theme: jekyll-theme-hacker

build.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+cat > Dockerfile.builder <<EOF
+FROM golang
+
+MAINTAINER Denis Zheleztsov <difrex.punk@gmail.com>
+
+RUN go get github.com/Difrex/rbmd/rbmd
+RUN cd /go/src/github.com/Difrex/rbmd && go get -t -v ./... || true
+
+WORKDIR /go/src/github.com/Difrex/rbmd
+
+ENTRYPOINT go build -ldflags "-linkmode external -extldflags -static" -o rbmd-linux-amd64 && mv rbmd-linux-amd64 /out
+EOF
+
+# Build builder
+docker build --no-cache -t rbmd_builder -f Dockerfile.builder .
+# Build bin
+docker run -v $(pwd)/out:/out rbmd_builder
+

main.go

@@ -2,18 +2,19 @@ package main
 
 import (
 	"flag"
-	"rbmd"
 	"strings"
+
+	"github.com/Difrex/rbmd/rbmd"
 	// "log"
 )
 
 var (
-	zk string
+	zk     string
 	zkPath string
-	tick int
+	tick   int
 	listen string
-	ws string
-	v bool
+	ws     string
+	v      bool
 )
 
 func init() {
@@ -31,7 +32,7 @@ func main() {
 	if v {
 		rbmd.VersionShow()
 	}
-	
+
 	config := rbmd.Zk{
 		strings.Split(zk, ","),
 		zkPath,

rbmd/config.go

@@ -6,4 +6,3 @@ type Zk struct {
 	Path  string
 	Tick  int
 }
-

rbmd/connection.go

@@ -16,4 +16,3 @@ func (conf Zk) InitConnection() (*zk.Conn, error) {
 
 	return conn, err
 }
-

rbmd/http.go

@@ -1,23 +1,24 @@
 package rbmd
 
 import (
-	"net/http"
-	"log"
 	"encoding/json"
+	"net/http"
 	"strings"
+
 	"time"
-	
+
+	log "github.com/Sirupsen/logrus"
 	"github.com/gorilla/websocket"
 )
 
 //ServerConf configuration of http api server
 type ServerConf struct {
 	Addr string
-	Ws string
+	Ws   string
 }
 
 //MountHandler /mount location
-func (wr wrr) MountHandler (w http.ResponseWriter, r *http.Request) {
+func (wr wrr) MountHandler(w http.ResponseWriter, r *http.Request) {
 	decoder := json.NewDecoder(r.Body)
 	var m RBDDevice
 	err := decoder.Decode(&m)
@@ -49,7 +50,7 @@ func (wr wrr) MountHandler (w http.ResponseWriter, r *http.Request) {
 }
 
 //UmountHandler /umount location
-func (wr wrr) UmountHandler (w http.ResponseWriter, r *http.Request) {
+func (wr wrr) UmountHandler(w http.ResponseWriter, r *http.Request) {
 	decoder := json.NewDecoder(r.Body)
 	var m RBDDevice
 	err := decoder.Decode(&m)
@@ -80,26 +81,24 @@ func (wr wrr) UmountHandler (w http.ResponseWriter, r *http.Request) {
 	w.Write(state)
 }
 
-
-//Resolve resolve request
-type Resolve struct {
-	Node string `json:"node"`
-}
-
 //ResolveHandler resolve `deadly.` state. /resolve location
-func (wr wrr) ResoleHandler(w http.ResponseWriter, r *http.Request) {
-	decoder := json.NewDecoder(r.Body)
+func (wr wrr) ResolveHandler(w http.ResponseWriter, r *http.Request) {
 	var res Resolve
+
+	decoder := json.NewDecoder(r.Body)
 	err := decoder.Decode(&res)
-	var msE []byte
 	if err != nil {
+		var msE []byte
 		msE, _ = json.Marshal(MountState{"FAIL", "JSON parse failure"})
+		w.WriteHeader(500)
 		w.Write(msE)
 		return
 	}
 
 	if err := wr.z.ResolveRequest(res); err != nil {
+		log.Error(err.Error())
 		w.WriteHeader(500)
+		return
 	}
 
 	w.WriteHeader(200)
@@ -108,45 +107,57 @@ func (wr wrr) ResoleHandler(w http.ResponseWriter, r *http.Request) {
 //wrr API
 type wrr struct {
 	Fqdn string
-	z ZooNode
+	z    ZooNode
 }
 
 //ServeHTTP start http server
 func (s ServerConf) ServeHTTP(z ZooNode, fqdn string) {
 
 	// Return JSON of full quorum status
-	http.HandleFunc("/status", func(w http.ResponseWriter, r *http.Request) {
+	http.HandleFunc("/v1/status", func(w http.ResponseWriter, r *http.Request) {
 		w.Write(z.GetState())
 	})
 
 	// Return string with quorum health check result
-	http.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
+	http.HandleFunc("/v1/health", func(w http.ResponseWriter, r *http.Request) {
 		w.Write([]byte(z.GetQuorumHealth()))
 	})
 
 	// Return JSON of node description
-	http.HandleFunc("/node", func(w http.ResponseWriter, r *http.Request) {
+	http.HandleFunc("/v1/node", func(w http.ResponseWriter, r *http.Request) {
 		n := GetNodeState(fqdn)
 		state, err := json.Marshal(n)
 		if err != nil {
 			log.Fatal(err)
 		}
 		w.Write(state)
+		return
 	})
 
-	wr := wrr{
-		fqdn,
-		z,
-	}
-	
+	// Return JSON mertrics
+	http.HandleFunc("/v1/metrics", func(w http.ResponseWriter, r *http.Request) {
+		m, err := GetMetrics(z)
+		if err != nil {
+			w.WriteHeader(500)
+			return
+		}
+		state, err := json.Marshal(m)
+		if err != nil {
+			log.Fatal(err)
+		}
+		w.Write(state)
+	})
+
+	wr := wrr{fqdn, z}
+
 	// Mount volume. Accept JSON. Return JSON.
-	http.HandleFunc("/mount", wr.MountHandler)
+	http.HandleFunc("/v1/mount", wr.MountHandler)
 
 	// Umount volume. Accept JSON. Return JSON.
-	http.HandleFunc("/umount", wr.UmountHandler)
+	http.HandleFunc("/v1/umount", wr.UmountHandler)
 
 	// Umount volume. Accept JSON. Return JSON.
-	http.HandleFunc("/resolve", wr.ResoleHandler)
+	http.HandleFunc("/v1/resolve", wr.ResolveHandler)
 
 	server := &http.Server{
 		Addr: s.Addr,
@@ -157,7 +168,7 @@ func (s ServerConf) ServeHTTP(z ZooNode, fqdn string) {
 //Writer ws
 type Writer struct {
 	Upgrader websocket.Upgrader
-	z ZooNode
+	z        ZooNode
 }
 
 //WriteStatusWs wrtite quorum status to websockets client
@@ -165,27 +176,49 @@ func (wr Writer) WriteStatusWs(w http.ResponseWriter, r *http.Request) {
 
 	c, err := wr.Upgrader.Upgrade(w, r, nil)
 	if err != nil {
-		log.Print("[Ws ERROR] Upgrade: ", err)
+		log.Error("[Ws] Upgrade: ", err.Error())
+		c.Close()
+		return
 	}
 
 	mt, _, err := c.ReadMessage()
 	if err != nil {
-		log.Print("[Ws ERROR] Read error: ", err)
+		log.Error("[Ws] Read error: ", err.Error())
 		// break
+		c.Close()
 		return
 	}
-	
-	go func() {
-		for {
-			err = c.WriteMessage(mt, wr.z.GetState())
-			if err != nil {
-				log.Print("[Ws ERROR] Write err: ", err)
-				defer c.Close()
-				break
-			}
-			time.Sleep(time.Duration(1) * time.Second)
+
+	// Write first state message after upgrade
+	err = c.WriteMessage(mt, wr.z.GetState())
+	if err != nil {
+		log.Error("[Ws] Write err: ", err.Error())
+		c.Close()
+		return
+	}
+
+	// Add watcher to cluster log
+	// logPath := strings.Join([]string{wr.z.Path, "log", "quorum"}, "/")
+	// log.Info(logPath)
+	// _, _, ch, err := wr.z.Conn.ChildrenW(logPath)
+	// if err != nil {
+	// 	log.Error("Cant add watcher", err.Error())
+	// 	c.Close()
+	// 	return
+	// }
+
+	for {
+		// log.Info("Run sockets loop")
+		// st := <-ch
+		// log.Info("got zk event ", st.Server)
+		time.Sleep(time.Second * 5)
+		err = c.WriteMessage(mt, wr.z.GetState())
+		if err != nil {
+			log.Error("[Ws] Write err: ", err.Error())
+			defer c.Close()
+			return
 		}
-	}()
+	}
 }
 
 //ServeWebSockets start websockets server

rbmd/leader.go

@@ -1,9 +1,9 @@
 package rbmd
 
 import (
-	"strings"
-	"log"
 	"encoding/json"
+	"log"
+	"strings"
 	"time"
 )
 
@@ -65,7 +65,7 @@ func (z ZooNode) CheckAndSetHealth(childrens []string) {
 				z.SetQuorumHealth("deadly.")
 				z.SetDeadlyReason(childNode)
 				return
-			} 
+			}
 		}
 	}
 
@@ -144,7 +144,7 @@ func (z ZooNode) FindLeader(fqdn string) {
 	if err != nil {
 		log.Print("[ERROR] ", err)
 	}
-	
+
 	var state bool
 
 	myState, _, _ := z.Conn.Get(strings.Join([]string{z.Path, "/cluster/", fqdn, "/state"}, ""))
@@ -193,7 +193,7 @@ func (z ZooNode) CompareChilds(node Node) (bool, []string) {
 				z.UpdateQuorum(childrens)
 				continue
 			}
-			
+
 			// Compare updated time
 			if node.Updated < childNode.Updated {
 				childrens, _, err = z.Conn.Children(strings.Join([]string{z.Path, "/cluster"}, ""))
@@ -225,7 +225,7 @@ func (z ZooNode) DestroyNode(fqdn string) ([]string, string) {
 		z.RMR(childPath)
 		z.SetQuorumHealth(strings.Join([]string{"resizing. node ", fqdn}, ""))
 	}
-	
+
 	childrens, _, err := z.Conn.Children(strings.Join([]string{z.Path, "/cluster"}, ""))
 	if err != nil {
 		log.Print("[zk ERROR] ", err)
@@ -235,9 +235,7 @@ func (z ZooNode) DestroyNode(fqdn string) ([]string, string) {
 	return childrens, strings.Join(message, "")
 }
 
-
-//CheckMounts ...
-// Check mounts on down node
+// CheckMounts on down node
 func CheckMounts(nodeStat []byte) (bool, []string) {
 	var node Node
 
@@ -252,7 +250,7 @@ func CheckMounts(nodeStat []byte) (bool, []string) {
 	var message []string
 	if len(node.Mounts) > 0 {
 		message = append(message, "deadly. ", "Reason: ", " NODE: ", node.Node)
-		for _, mount := range  node.Mounts {
+		for _, mount := range node.Mounts {
 			message = append(message, ", mountpoint: ", mount.Mountpoint, ", block: ", mount.Block, ", pool: ", mount.Pool)
 		}
 		return false, message

rbmd/metrics.go

@@ -0,0 +1,48 @@
+package rbmd
+
+import (
+	"encoding/json"
+	"runtime"
+	"strings"
+)
+
+//Metrics metrics statistic
+type Metrics struct {
+	Goroutines  int   `json:"goroutines"`
+	Nodes       int   `json:"nodes"`
+	MountsTotal int   `json:"mountstotal"`
+	Cgocall     int64 `json:"cgocall"`
+}
+
+// GetMetrics ...
+func GetMetrics(z ZooNode) (Metrics, error) {
+	var q Quorum
+	var m Metrics
+
+	curQuorum, _, err := z.Conn.Get(strings.Join([]string{z.Path, "log", "quorum"}, "/"))
+	if err != nil {
+		return m, err
+	}
+
+	err = json.Unmarshal(curQuorum, &q)
+	if err != nil {
+		return m, err
+	}
+
+	m.Nodes = len(q.Quorum)
+	m.Goroutines = runtime.NumGoroutine()
+	m.Cgocall = runtime.NumCgoCall()
+	m.MountsTotal = GetTotalMounts(q.Quorum)
+
+	return m, nil
+}
+
+// GetTotalMounts ...
+func GetTotalMounts(n []Node) int {
+	mounts := 0
+	for _, node := range n {
+		mounts = mounts + len(node.Mounts)
+	}
+
+	return mounts
+}

rbmd/requests.go

@@ -1,49 +1,69 @@
 package rbmd
 
 import (
-	"log"
-	"strings"
 	"encoding/json"
+	"strings"
 	// "bytes"
 
+	log "github.com/Sirupsen/logrus"
 	"github.com/samuel/go-zookeeper/zk"
 )
 
-//RequestWatch watch for mount/umount requests
+// RequestWatch watch for mount/umount requests
 func (z ZooNode) RequestWatch(fqdn string) {
 	requestsPath := strings.Join([]string{z.Path, "cluster", fqdn, "requests"}, "/")
 	_, _, ch, err := z.Conn.ChildrenW(requestsPath)
 	if err != nil {
-		log.Print("[zk ERROR] ", err)
+		log.Error("[zk ERROR] ", err.Error())
 	}
 
 	for {
 		req := <-ch
-		log.Print("[DEBUG] ch path ", req.Path)
+		log.Info("ch path ", req.Path)
 		childrens, _, err := z.Conn.Children(requestsPath)
 		if err != nil {
+			log.Error(err.Error())
 			break
 		}
 		for _, child := range childrens {
 			p := strings.Join([]string{req.Path, child}, "/")
 			request, _, err := z.Conn.Get(p)
 			if err != nil {
-				log.Print("[zk ERROR] ", err)
+				log.Error("[zk] ", err.Error())
 			}
 
 			var r RBDDevice
 			err = json.Unmarshal(request, &r)
 			if err != nil {
-				log.Print("[ERROR] ", err)
+				log.Error("", err.Error())
 			}
 
-			// 1) Map RBD 2) Mount FS
+			if z.GetQuorumHealth() != "alive." && child != "resolve" {
+				z.RMR(p)
+				z.Answer(fqdn, child, []byte(""), "FAIL: cluster not alive.")
+				break
+			}
+
+			// 0) Check already mounted devices 1) Map RBD 2) Mount FS
 			if child == "mount" {
+				m, err := z.CheckMounted(r)
+				if err != nil {
+					z.RMR(p)
+					z.Answer(fqdn, child, []byte(err.Error()), "FAIL")
+					log.Error("[ERROR] Mapping error: ", err.Error())
+					break
+				}
+				if !m {
+					z.RMR(p)
+					z.Answer(fqdn, child, []byte("Already mounted"), "FAIL")
+					log.Error("[ERROR] Mapping error: Already mounted")
+					return
+				}
 				std, err := r.MapDevice()
 				if err != nil {
 					z.RMR(p)
 					z.Answer(fqdn, child, std, "FAIL")
-					log.Print("[ERROR] Mapping error: ", string(std), err)
+					log.Error("[ERROR] Mapping error: ", string(std), err.Error())
 					break
 				}
 				err = r.MountFS(string(std))
@@ -51,11 +71,11 @@ func (z ZooNode) RequestWatch(fqdn string) {
 					r.UnmapDevice()
 					z.RMR(p)
 					z.Answer(fqdn, child, std, "FAIL")
-					log.Print("[ERROR] Mount filesystem error: ", err)
+					log.Print("[ERROR] Mount filesystem error: ", err.Error())
 					break
 				}
 				z.Answer(fqdn, child, std, "OK")
-			// 1) Unmount FS 2) Unmap RBD
+				// 1) Unmount FS 2) Unmap RBD
 			} else if child == "umount" {
 				err := r.UnmountFS()
 				if err != nil {
@@ -73,12 +93,13 @@ func (z ZooNode) RequestWatch(fqdn string) {
 				}
 				z.Answer(fqdn, child, std, "OK")
 			} else if child == "resolve" && z.GetLeader() == fqdn {
+				log.Warn("Got resolve request ", r.Node)
 				if err := z.Resolve(fqdn); err != nil {
-					log.Print("[ERROR] ", err)
+					log.Error(err.Error())
 					z.RMR(p)
 				}
 			} else {
-				log.Print("[ERROR] Unknown request: ", child)
+				log.Error("Unknown request: ", child)
 				z.RMR(p)
 			}
 			z.RMR(p)
@@ -87,21 +108,29 @@ func (z ZooNode) RequestWatch(fqdn string) {
 	}
 }
 
+//Resolve resolve request
+type Resolve struct {
+	Node string `json:"node"`
+}
+
 //Resolve delete node from quorum
 func (z ZooNode) Resolve(fqdn string) error {
 	resolvePath := strings.Join([]string{z.Path, "cluster", fqdn, "requests", "resolve"}, "/")
 
 	r, _, err := z.Conn.Get(resolvePath)
 	if err != nil {
+		log.Error(err.Error())
 		return err
 	}
-	
+
 	var res Resolve
 	if err := json.Unmarshal(r, &res); err != nil {
+		log.Error(err.Error())
 		return err
 	}
 
 	deadlyNodePath := strings.Join([]string{z.Path, "cluster", res.Node}, "/")
+	log.Warn("Trying resolve. Remove ", res.Node, " from quorum")
 	z.RMR(resolvePath)
 	z.RMR(deadlyNodePath)
 
@@ -115,18 +144,20 @@ func (z ZooNode) ResolveRequest(r Resolve) error {
 
 	jsReq, err := json.Marshal(r)
 	if err != nil {
+		log.Error(err.Error())
 		return err
 	}
 
-	_, err = z.Conn.Create(resolvePath, jsReq, 0, zk.WorldACL(zk.PermAll))
+	z.EnsureZooPath(resolvePath)
+	_, err = z.Conn.Set(resolvePath, jsReq, -1)
 	if err != nil {
-		_, err := z.Conn.Set(resolvePath, jsReq, -1)
+		_, err := z.Conn.Create(resolvePath, jsReq, 0, zk.WorldACL(zk.PermAll))
 		if err != nil {
-			log.Print("[zk ERROR] ", err)
+			log.Error("Cant create resolve request node ", err.Error())
 			return err
 		}
 	}
-	
+
 	return nil
 }
 
@@ -137,15 +168,15 @@ func (z ZooNode) Answer(fqdn string, req string, stderr []byte, t string) {
 	answer := MountState{t, string(stderr)}
 	answerJSON, err := json.Marshal(answer)
 	if err != nil {
-		log.Print("[ERROR] ", err)
+		log.Error(err.Error())
 	}
 
 	_, err = z.Conn.Create(answerPath, answerJSON, 0, zk.WorldACL(zk.PermAll))
 	if err != nil {
-		log.Print("[zk ERROR] ", err)
+		log.Error("[zk] ", err.Error())
 		_, err := z.Conn.Set(answerPath, answerJSON, -1)
 		if err != nil {
-			log.Print("[zk ERROR] ", err)
+			log.Error("[zk] ", err.Error())
 		}
 	}
 }
@@ -192,16 +223,16 @@ func (z ZooNode) UmountRequest(r RBDDevice) error {
 //WatchAnswer watch for answer
 func (z ZooNode) WatchAnswer(fqdn string, t string) MountState {
 	answersPath := strings.Join([]string{z.Path, "cluster", fqdn, "answers"}, "/")
-	log.Print("[DEBUG] ", answersPath)
+	log.Debug(answersPath)
 	_, _, ch, err := z.Conn.ChildrenW(answersPath)
 	if err != nil {
-		log.Print("[zk ERROR] 107 ", err)
-		return MountState{"FAIL", "Zk error"}
+		log.Error("[zk] ", err.Error())
+		return MountState{"FAIL", "Zk error " + err.Error()}
 	}
 
 	var ms MountState
 	var p string
-	
+
 	for {
 		ans := <-ch
 		log.Print("[DEBUG] ch answer path ", ans.Path)
@@ -240,3 +271,36 @@ func (z ZooNode) WatchAnswer(fqdn string, t string) MountState {
 	z.RMR(p)
 	return ms
 }
+
+// CheckMounted Check already mounted devices
+func (z ZooNode) CheckMounted(r RBDDevice) (bool, error) {
+	nodes, _, err := z.Conn.Children(strings.Join([]string{z.Path, "cluster"}, "/"))
+	if err != nil {
+		return false, err
+	}
+
+	for _, node := range nodes {
+		var nodeState Node
+		statePath := strings.Join([]string{z.Path, "cluster", node, "state"}, "/")
+
+		state, _, err := z.Conn.Get(statePath)
+		if err != nil {
+			return false, err
+		}
+
+		err = json.Unmarshal(state, &nodeState)
+		if err != nil {
+			return false, err
+		}
+
+		if len(nodeState.Mounts) > 0 {
+			for _, mount := range nodeState.Mounts {
+				if mount.Image == r.Image && mount.Pool == r.Pool {
+					return false, nil
+				}
+			}
+		}
+	}
+
+	return true, nil
+}

rbmd/state.go

@@ -21,7 +21,7 @@ func Run(zoo Zk, s ServerConf) {
 	z := ZooNode{zoo.Path, connection, zoo}
 
 	z.CreateZkTree(fqdn)
-	go func () {
+	go func() {
 		for {
 			z.RequestWatch(fqdn)
 		}
@@ -77,13 +77,12 @@ func (z ZooNode) UpdateState(zkPath string, fqdn string) {
 	}
 }
 
-
 //jsonState HTTP API quorum status
 type jsonState struct {
-	Quorum map[string]Node `json:"quorum"`
-    Health string          `json:"health"`
-	DeadlyReason Node      `json:"deadlyreason"`
-	Leader string          `json:"leader"`
+	Quorum       map[string]Node `json:"quorum"`
+	Health       string          `json:"health"`
+	DeadlyReason Node            `json:"deadlyreason"`
+	Leader       string          `json:"leader"`
 }
 
 //GetState return cluster status

rbmd/sys.go

@@ -1,42 +1,41 @@
 package rbmd
 
 import (
-	"syscall"
+	"bytes"
 	"io/ioutil"
 	"net"
-	"log"
-	"strings"
-	"regexp"
-	"time"
 	"os/exec"
-	"bytes"
+	"regexp"
+	"strings"
+	"syscall"
+	"time"
 
-	// "fmt"
+	log "github.com/Sirupsen/logrus"
 )
 
 //ClusterStatus Quorum status struct
 type ClusterStatus struct {
 	Quorum []Node `json:"quorum"`
 	Health string `json:"health"`
-	Zk string     `json:"zk"`
+	Zk     string `json:"zk"`
 }
 
 //Node Node status struct
 type Node struct {
-	Node string     `json:"node"`
-	IP IPs          `json:"ip"`
+	Node    string  `json:"node"`
+	IP      IPs     `json:"ip"`
 	Updated int64   `json:"updated"`
-	Mounts []Mount  `json:"mounts"`
+	Mounts  []Mount `json:"mounts"`
 }
 
 // Mount struct
 type Mount struct {
 	Mountpoint string `json:"mountpoint"`
-	Mountopts string  `json:"mountopts"`
-	Fstype string     `json:"fstype"`
-	Pool string       `json:"pool"`
-	Image string      `json:"image"`
-	Block string      `json:"block"`
+	Mountopts  string `json:"mountopts"`
+	Fstype     string `json:"fstype"`
+	Pool       string `json:"pool"`
+	Image      string `json:"image"`
+	Block      string `json:"block"`
 }
 
 //IPs IP addresses
@@ -57,18 +56,18 @@ func GetMounts() []Mount {
 		mount := strings.Split(line, " ")
 		match, err := regexp.MatchString("^(/dev/rbd).*$", mount[0])
 		if err != nil {
-			log.Print("[ERROR] ", err)
+			log.Error(err.Error())
 		}
 		if match {
 			p := strings.Split(mount[0], "/")
-			pool, image := GetRBDPool(p[len(p) - 1])
+			pool, image := GetRBDPool(p[len(p)-1])
 			mounts = append(mounts, Mount{
 				mount[1],
 				mount[3],
 				mount[2],
 				pool,
 				image,
-				p[len(p) - 1],
+				p[len(p)-1],
 			})
 		}
 	}
@@ -93,10 +92,13 @@ func GetRBDPool(device string) (string, string) {
 		log.Fatal("[ERROR] Read failure ", err)
 	}
 
-	return string(pool), string(image)
+	p := strings.Trim(string(pool), "\n")
+	i := strings.Trim(string(image), "\n")
+
+	return p, i
 }
 
-//GetMyIPs Exclude 127.0.0.1 
+//GetMyIPs Exclude 127.0.0.1
 func GetMyIPs() IPs {
 	ifaces, err := net.Interfaces()
 	if err != nil {
@@ -114,9 +116,9 @@ func GetMyIPs() IPs {
 			var ip net.IP
 			switch v := addr.(type) {
 			case *net.IPNet:
-                ip = v.IP
+				ip = v.IP
 			case *net.IPAddr:
-                ip = v.IP
+				ip = v.IP
 			}
 			if ip.String() != "127.0.0.1" && ip.String() != "::1" {
 				match, err := regexp.MatchString("^.*:.*$", ip.String())
@@ -138,7 +140,6 @@ func GetMyIPs() IPs {
 	}
 }
 
-
 //GetNodeState Return Node struct
 func GetNodeState(fqdn string) Node {
 	var n Node
@@ -151,93 +152,117 @@ func GetNodeState(fqdn string) Node {
 	return n
 }
 
-
 //MountState status of mount/umount
 type MountState struct {
-	State string   `json:"state"`
+	State   string `json:"state"`
 	Message string `json:"message"`
 }
 
 //RBDDevice rbd block device struct
 type RBDDevice struct {
-	Node string       `json:"node"`
-	Pool string       `json:"pool"`
-	Image string      `json:"image"`
-	Block string      `json:"block"`
+	Node       string `json:"node"`
+	Pool       string `json:"pool"`
+	Image      string `json:"image"`
+	Block      string `json:"block"`
 	Mountpoint string `json:"mountpoint"`
-	Mountopts string  `json:"mountopts"`
-	Fstype string     `json:"fstype"`
+	Mountopts  string `json:"mountopts"`
+	Fstype     string `json:"fstype"`
 }
 
 //MapDevice map rbd block device
 func (r RBDDevice) MapDevice() ([]byte, error) {
 	image := strings.Join([]string{r.Pool, r.Image}, "/")
-	log.Print("[DEBUG] Mapping ", image)
+	log.Warn("[DEBUG] Mapping ", image)
 
 	var stdout bytes.Buffer
 	var stderr bytes.Buffer
-	
-	cmd := exec.Command("rbd", "map", image)
+
+	cmd := exec.Command("/usr/bin/rbd", "map", image)
 
 	cmd.Stdout = &stdout
 	cmd.Stderr = &stderr
 
 	err := cmd.Run()
 	if err != nil {
-		return []byte(stderr.String()) , err
+		log.Error(err.Error())
+		return []byte(strings.Join([]string{stderr.String(), stdout.String()}, " ")), err
 	}
 
 	o := stdout.String()
 
 	if strings.HasSuffix(o, "\n") {
-        o = o[ :len(o) - 1]
-    }
-	
+		o = o[:len(o)-1]
+	}
+
 	return []byte(o), nil
 }
 
+//MountFS mount file system
+func (r RBDDevice) MountFS(device string) error {
+	err := syscall.Mount(device, r.Mountpoint, r.Fstype, ParseMountOpts(r.Mountopts), "")
+	if err != nil {
+		log.Error("Cant mount ", device, err.Error())
+		return err
+	}
+
+	return nil
+}
+
+//ParseMountOpts parse RBDDevice.Mountopts. Return uintptr
+func ParseMountOpts(mountopts string) uintptr {
+	// Mount options map
+	opts := make(map[string]uintptr)
+	opts["ro"] = syscall.MS_RDONLY
+	opts["posixacl"] = syscall.MS_POSIXACL
+	opts["relatime"] = syscall.MS_RELATIME
+	opts["noatime"] = syscall.MS_NOATIME
+	opts["nosuid"] = syscall.MS_NOSUID
+	opts["noexec"] = syscall.MS_NOEXEC
+	opts["nodiratime"] = syscall.MS_NODIRATIME
+
+	var msOpts uintptr
+	if mountopts != "" {
+		for _, o := range strings.Split(mountopts, ",") {
+			msOpts = uintptr(msOpts | opts[o])
+		}
+		return msOpts
+	}
+
+	return 0
+}
+
 //UnmapDevice unmap rbd block device
 func (r RBDDevice) UnmapDevice() ([]byte, error) {
 	log.Print("[DEBUG] Umapping ", r.Block)
 
 	var stdout bytes.Buffer
 	var stderr bytes.Buffer
-	
-	cmd := exec.Command("rbd", "unmap", strings.Join([]string{"/dev/", r.Block}, ""))
+
+	cmd := exec.Command("/usr/bin/rbd", "unmap", strings.Join([]string{"/dev/", r.Block}, ""))
 
 	cmd.Stdout = &stdout
 	cmd.Stderr = &stderr
 
 	err := cmd.Run()
 	if err != nil {
-		return []byte(stderr.String()) , err
+		return []byte(stderr.String()), err
 	}
 
 	o := stdout.String()
 
 	if strings.HasSuffix(o, "\n") {
-        o = o[ :len(o) - 1]
-    }
-	
-	return []byte(o), nil
-}
-
-//MountFS mount file system
-func (r RBDDevice) MountFS(device string) error {
-	err := syscall.Mount(device, r.Mountpoint, r.Fstype, 0, r.Mountopts)
-	if err != nil {
-		log.Print("[DEBUG] sys 207 ", err)
-		return err
+		o = o[:len(o)-2]
 	}
 
-	return nil
+	return []byte(o), nil
 }
 
 //UnmountFS unmount file system
 func (r RBDDevice) UnmountFS() error {
 	err := syscall.Unmount(r.Mountpoint, 0)
+	log.Info("Try to umount ", r.Mountpoint)
 	if err != nil {
-		log.Print("[DEBUG] sys 207 ", err)
+		log.Error("Cant umount ", r.Mountpoint, err.Error())
 		return err
 	}
 

rbmd/version.go

@@ -1,13 +1,13 @@
 package rbmd
 
 import (
+	"fmt"
 	"os"
 	"runtime"
-	"fmt"
 )
 
 //VersionShow show version and exit
 func VersionShow() {
-	fmt.Println("RBMD 0.0.3", runtime.Version(), runtime.GOARCH)
+	fmt.Println("RBMD 0.2 test", runtime.Version(), runtime.GOARCH)
 	os.Exit(1)
 }

rbmd/zoo.go

@@ -1,9 +1,10 @@
 package rbmd
 
 import (
-	"github.com/samuel/go-zookeeper/zk"
-	"strings"
 	"log"
+	"strings"
+
+	"github.com/samuel/go-zookeeper/zk"
 	// "encoding/json"
 )
 
@@ -11,29 +12,32 @@ import (
 type ZooNode struct {
 	Path string
 	Conn *zk.Conn
-	Zoo Zk
+	Zoo  Zk
 }
 
-//EnsureZooPath create zookeeper path  
-func (z ZooNode) EnsureZooPath(node string) (string, error) {
+//EnsureZooPath create zookeeper path
+func (z ZooNode) EnsureZooPath(path string) (string, error) {
+	path = strings.Join([]string{z.Path, path}, "/")
+
 	flag := int32(0)
 	acl := zk.WorldACL(zk.PermAll)
 
-	zoopath := strings.Join([]string{z.Path, "/", node}, "")
-	s := strings.Split(zoopath, "/")
-
+	s := strings.Split(path, "/")
 	var p []string
 	var fullnodepath string
-	
+
 	for i := 1; i < len(s); i++ {
 		p = append(p, strings.Join([]string{"/", s[i]}, ""))
 	}
-	
+
 	for i := 0; i < len(p); i++ {
 		fullnodepath = strings.Join([]string{fullnodepath, p[i]}, "")
-		z.Conn.Create(fullnodepath, []byte(""), flag, acl)
+		exists, _, _ := z.Conn.Exists(fullnodepath)
+		if !exists {
+			z.Conn.Create(fullnodepath, []byte(""), flag, acl)
+		}
 	}
-	
+
 	return fullnodepath, nil
 }