FIX: Check mount before map image

* Most logs moved to Sirupsen/logrus package

* Rewrite EnsureZooPath function

.travis.yml

@@ -0,0 +1,14 @@
+language: go
+go:
+  - "1.6"
+  - "1.7.x"
+  - "1.8"
+  - "1.9"
+
+before_install:
+  - go get github.com/gorilla/websocket
+  - go get github.com/samuel/go-zookeeper/zk
+
+# tests script
+script:
+  - go build -o _rbmd

README.md

@@ -1,12 +1,16 @@
 # RBMD
 
+[![Build Status](https://travis-ci.org/Difrex/rbmd.svg?branch=master)](https://travis-ci.org/Difrex/rbmd)
+
+![logo](/img/logo.png)
+
 RBD mount wrapper cluster
 
-**NOT FOR PRODUCTION**
+**ALLOW FOR STAGING**
 
 Current status: *development*, *testing*
 
-**NOT FOR PRODUCTION**
+**ALLOW FOR STAGING**
 
 <!-- markdown-toc start - Don't edit this section. Run M-x markdown-toc-generate-toc again -->
 **Table of Contents**
@@ -17,20 +21,23 @@ Current status: *development*, *testing*
         - [Example](#example)
     - [Build](#build)
     - [API](#api)
-        - [GET /status](#get-status)
+        - [GET /v1/status](#get-v1status)
             - [Example](#example)
-        - [GET /node](#get-node)
+        - [GET /v1/node](#get-v1node)
             - [Example](#example)
-        - [GET /health](#get-health)
+        - [GET /v1/health](#get-v1health)
             - [Example](#example)
-        - [POST /mount](#post-mount)
+        - [POST /v1/mount](#post-v1mount)
             - [Example](#example)
-        - [POST /umount](#post-umount)
+        - [POST /v1/umount](#post-v1umount)
             - [Example](#example)
-        - [POST /resolve](#post-resolve)
+        - [POST /v1/resolve](#post-v1resolve)
             - [Example](#example)
-        - [GET /metrics](#get-metrics)
+        - [GET /v1/metrics](#get-v1metrics)
             - [Example](#example)
+    - [Systemd](#systemd)
+- [Troubleshooting](#troubleshooting)
+    - [Umount request is stuck after deadly.](#umount-request-is-stuck-after-deadly)
 - [AUTHORS](#authors)
 - [LICENSE](#license)
 
@@ -70,14 +77,13 @@ Required Go > 1.6
 
 ```
 git clone https://github.com/rbmd/rbmd.git && cd rbmd
-GOPATH=$(pwd) go get github.com/gorilla/websocket
-GOPATH=$(pwd) go get github.com/samuel/go-zookeeper/zk
+GOPATH=$(pwd) go get -t -v ./...
 GOPATH=$(pwd) go build
 ```
 
 ## API
 
-### GET /status
+### GET /v1/status
 
 Return JSON of quorum status
 
@@ -105,14 +111,14 @@ Return JSON of quorum status
 }
 ```
 
-### GET /node
+### GET /v1/node
 
 Return JSON of node stats 
 
 #### Example
 ```json
 {
-  "node": "difrex-mac.wargaming.net",
+  "node": "node.example.com",
   "ip": {
     "v4": [
       "169.254.156.1"
@@ -126,7 +132,7 @@ Return JSON of node stats
 }
 ```
 
-### GET /health
+### GET /v1/health
 
 Return string with quorum health check result
 
@@ -142,7 +148,7 @@ curl 127.0.0.1:9076/health
 alive.
 ```
 
-### POST /mount
+### POST /v1/mount
 
 Map rbd image and mount it
 
@@ -186,7 +192,7 @@ On failure
 }
 ```
 
-### POST /umount
+### POST /v1/umount
 
 Unmount filesystem and unmap RBD device
 
@@ -214,12 +220,12 @@ On success
 On failure
 ```json
 {
-    "state": "FAIL"
+    "state": "FAIL",
     "message": "Not found"
 }
 ```
 
-### POST /resolve
+### POST /v1/resolve
 
 Remove deadly node from quorum.
 
@@ -232,7 +238,7 @@ Accept JSON
 }
 ```
 
-### GET /metrics
+### GET /v1/metrics
 
 Return some metrics
 
@@ -247,6 +253,29 @@ Return some metrics
 }
 ```
 
+## Systemd
+
+Example unit
+```ini
+[Unit]
+Description=RBMD
+After=network.target
+
+[Service]
+ExecStart=/usr/bin/rbmd -listen 0.0.0.0:9076 -ws 0.0.0.0:7690 -zk node1:2181,node2:2181,node3:2181
+KillMode=control-group
+Restart=on-failure
+
+[Install]
+WantedBy=multi-user.target
+```
+
+# Troubleshooting
+
+## Umount request is stuck after deadly.
+
+Remove Zk node */rbmd/cluster/node.fqdn/requests/umount*
+
 # AUTHORS
 
 Denis Zheleztsov <difrex.punk@gmail.com>

_config.yml

@@ -0,0 +1 @@
+theme: jekyll-theme-hacker

build.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+cat > Dockerfile.builder <<EOF
+FROM golang
+
+MAINTAINER Denis Zheleztsov <difrex.punk@gmail.com>
+
+RUN go get github.com/Difrex/rbmd/rbmd
+RUN cd /go/src/github.com/Difrex/rbmd && go get -t -v ./... || true
+
+WORKDIR /go/src/github.com/Difrex/rbmd
+
+ENTRYPOINT go build -ldflags "-linkmode external -extldflags -static" -o rbmd-linux-amd64 && mv rbmd-linux-amd64 /out
+EOF
+
+# Build builder
+docker build --no-cache -t rbmd_builder -f Dockerfile.builder .
+# Build bin
+docker run -v $(pwd)/out:/out rbmd_builder
+

main.go

@@ -2,8 +2,9 @@ package main
 
 import (
 	"flag"
-	"rbmd"
 	"strings"
+
+	"github.com/Difrex/rbmd/rbmd"
 	// "log"
 )
 

rbmd/http.go

@@ -2,11 +2,12 @@ package rbmd
 
 import (
 	"encoding/json"
-	"log"
 	"net/http"
 	"strings"
+
 	"time"
 
+	log "github.com/Sirupsen/logrus"
 	"github.com/gorilla/websocket"
 )
 
@@ -80,18 +81,14 @@ func (wr wrr) UmountHandler(w http.ResponseWriter, r *http.Request) {
 	w.Write(state)
 }
 
-//Resolve resolve request
-type Resolve struct {
-	Node string `json:"node"`
-}
-
 //ResolveHandler resolve `deadly.` state. /resolve location
-func (wr wrr) ResoleHandler(w http.ResponseWriter, r *http.Request) {
-	decoder := json.NewDecoder(r.Body)
+func (wr wrr) ResolveHandler(w http.ResponseWriter, r *http.Request) {
 	var res Resolve
+
+	decoder := json.NewDecoder(r.Body)
 	err := decoder.Decode(&res)
-	var msE []byte
 	if err != nil {
+		var msE []byte
 		msE, _ = json.Marshal(MountState{"FAIL", "JSON parse failure"})
 		w.WriteHeader(500)
 		w.Write(msE)
@@ -99,7 +96,9 @@ func (wr wrr) ResoleHandler(w http.ResponseWriter, r *http.Request) {
 	}
 
 	if err := wr.z.ResolveRequest(res); err != nil {
+		log.Error(err.Error())
 		w.WriteHeader(500)
+		return
 	}
 
 	w.WriteHeader(200)
@@ -158,7 +157,7 @@ func (s ServerConf) ServeHTTP(z ZooNode, fqdn string) {
 	http.HandleFunc("/v1/umount", wr.UmountHandler)
 
 	// Umount volume. Accept JSON. Return JSON.
-	http.HandleFunc("/v1/resolve", wr.ResoleHandler)
+	http.HandleFunc("/v1/resolve", wr.ResolveHandler)
 
 	server := &http.Server{
 		Addr: s.Addr,
@@ -177,27 +176,49 @@ func (wr Writer) WriteStatusWs(w http.ResponseWriter, r *http.Request) {
 
 	c, err := wr.Upgrader.Upgrade(w, r, nil)
 	if err != nil {
-		log.Print("[Ws ERROR] Upgrade: ", err)
+		log.Error("[Ws] Upgrade: ", err.Error())
+		c.Close()
+		return
 	}
 
 	mt, _, err := c.ReadMessage()
 	if err != nil {
-		log.Print("[Ws ERROR] Read error: ", err)
+		log.Error("[Ws] Read error: ", err.Error())
 		// break
+		c.Close()
 		return
 	}
 
-	go func() {
-		for {
-			err = c.WriteMessage(mt, wr.z.GetState())
-			if err != nil {
-				log.Print("[Ws ERROR] Write err: ", err)
-				defer c.Close()
-				break
-			}
-			time.Sleep(time.Duration(1) * time.Second)
+	// Write first state message after upgrade
+	err = c.WriteMessage(mt, wr.z.GetState())
+	if err != nil {
+		log.Error("[Ws] Write err: ", err.Error())
+		c.Close()
+		return
+	}
+
+	// Add watcher to cluster log
+	// logPath := strings.Join([]string{wr.z.Path, "log", "quorum"}, "/")
+	// log.Info(logPath)
+	// _, _, ch, err := wr.z.Conn.ChildrenW(logPath)
+	// if err != nil {
+	// 	log.Error("Cant add watcher", err.Error())
+	// 	c.Close()
+	// 	return
+	// }
+
+	for {
+		// log.Info("Run sockets loop")
+		// st := <-ch
+		// log.Info("got zk event ", st.Server)
+		time.Sleep(time.Second * 5)
+		err = c.WriteMessage(mt, wr.z.GetState())
+		if err != nil {
+			log.Error("[Ws] Write err: ", err.Error())
+			defer c.Close()
+			return
 		}
-	}()
+	}
 }
 
 //ServeWebSockets start websockets server

rbmd/leader.go

@@ -235,8 +235,7 @@ func (z ZooNode) DestroyNode(fqdn string) ([]string, string) {
 	return childrens, strings.Join(message, "")
 }
 
-//CheckMounts ...
-// Check mounts on down node
+// CheckMounts on down node
 func CheckMounts(nodeStat []byte) (bool, []string) {
 	var node Node
 

rbmd/requests.go

@@ -2,45 +2,45 @@ package rbmd
 
 import (
 	"encoding/json"
-	"log"
 	"strings"
 	// "bytes"
 
+	log "github.com/Sirupsen/logrus"
 	"github.com/samuel/go-zookeeper/zk"
 )
 
-//RequestWatch watch for mount/umount requests
+// RequestWatch watch for mount/umount requests
 func (z ZooNode) RequestWatch(fqdn string) {
 	requestsPath := strings.Join([]string{z.Path, "cluster", fqdn, "requests"}, "/")
 	_, _, ch, err := z.Conn.ChildrenW(requestsPath)
 	if err != nil {
-		log.Print("[zk ERROR] ", err)
+		log.Error("[zk ERROR] ", err.Error())
 	}
 
 	for {
 		req := <-ch
-		log.Print("[DEBUG] ch path ", req.Path)
+		log.Info("ch path ", req.Path)
 		childrens, _, err := z.Conn.Children(requestsPath)
 		if err != nil {
+			log.Error(err.Error())
 			break
 		}
 		for _, child := range childrens {
 			p := strings.Join([]string{req.Path, child}, "/")
 			request, _, err := z.Conn.Get(p)
 			if err != nil {
-				log.Print("[zk ERROR] ", err)
+				log.Error("[zk] ", err.Error())
 			}
 
 			var r RBDDevice
 			err = json.Unmarshal(request, &r)
 			if err != nil {
-				log.Print("[ERROR] ", err)
+				log.Error("", err.Error())
 			}
 
-			if z.GetLeader() != "alive." {
+			if z.GetQuorumHealth() != "alive." && child != "resolve" {
 				z.RMR(p)
-				z.Answer(fqdn, child, []byte(""), "FAIL")
-				log.Print("[ERROR] Mapping error: ", err)
+				z.Answer(fqdn, child, []byte(""), "FAIL: cluster not alive.")
 				break
 			}
 
@@ -49,21 +49,21 @@ func (z ZooNode) RequestWatch(fqdn string) {
 				m, err := z.CheckMounted(r)
 				if err != nil {
 					z.RMR(p)
-					z.Answer(fqdn, child, []byte(""), "FAIL")
-					log.Print("[ERROR] Mapping error: ", err)
+					z.Answer(fqdn, child, []byte(err.Error()), "FAIL")
+					log.Error("[ERROR] Mapping error: ", err.Error())
 					break
 				}
 				if !m {
 					z.RMR(p)
 					z.Answer(fqdn, child, []byte("Already mounted"), "FAIL")
-					log.Print("[ERROR] Mapping error: ", err)
-					break
+					log.Error("[ERROR] Mapping error: Already mounted")
+					return
 				}
 				std, err := r.MapDevice()
 				if err != nil {
 					z.RMR(p)
 					z.Answer(fqdn, child, std, "FAIL")
-					log.Print("[ERROR] Mapping error: ", string(std), err)
+					log.Error("[ERROR] Mapping error: ", string(std), err.Error())
 					break
 				}
 				err = r.MountFS(string(std))
@@ -71,7 +71,7 @@ func (z ZooNode) RequestWatch(fqdn string) {
 					r.UnmapDevice()
 					z.RMR(p)
 					z.Answer(fqdn, child, std, "FAIL")
-					log.Print("[ERROR] Mount filesystem error: ", err)
+					log.Print("[ERROR] Mount filesystem error: ", err.Error())
 					break
 				}
 				z.Answer(fqdn, child, std, "OK")
@@ -93,12 +93,13 @@ func (z ZooNode) RequestWatch(fqdn string) {
 				}
 				z.Answer(fqdn, child, std, "OK")
 			} else if child == "resolve" && z.GetLeader() == fqdn {
+				log.Warn("Got resolve request ", r.Node)
 				if err := z.Resolve(fqdn); err != nil {
-					log.Print("[ERROR] ", err)
+					log.Error(err.Error())
 					z.RMR(p)
 				}
 			} else {
-				log.Print("[ERROR] Unknown request: ", child)
+				log.Error("Unknown request: ", child)
 				z.RMR(p)
 			}
 			z.RMR(p)
@@ -107,21 +108,29 @@ func (z ZooNode) RequestWatch(fqdn string) {
 	}
 }
 
+//Resolve resolve request
+type Resolve struct {
+	Node string `json:"node"`
+}
+
 //Resolve delete node from quorum
 func (z ZooNode) Resolve(fqdn string) error {
 	resolvePath := strings.Join([]string{z.Path, "cluster", fqdn, "requests", "resolve"}, "/")
 
 	r, _, err := z.Conn.Get(resolvePath)
 	if err != nil {
+		log.Error(err.Error())
 		return err
 	}
 
 	var res Resolve
 	if err := json.Unmarshal(r, &res); err != nil {
+		log.Error(err.Error())
 		return err
 	}
 
 	deadlyNodePath := strings.Join([]string{z.Path, "cluster", res.Node}, "/")
+	log.Warn("Trying resolve. Remove ", res.Node, " from quorum")
 	z.RMR(resolvePath)
 	z.RMR(deadlyNodePath)
 
@@ -135,14 +144,16 @@ func (z ZooNode) ResolveRequest(r Resolve) error {
 
 	jsReq, err := json.Marshal(r)
 	if err != nil {
+		log.Error(err.Error())
 		return err
 	}
 
-	_, err = z.Conn.Create(resolvePath, jsReq, 0, zk.WorldACL(zk.PermAll))
+	z.EnsureZooPath(resolvePath)
+	_, err = z.Conn.Set(resolvePath, jsReq, -1)
 	if err != nil {
-		_, err := z.Conn.Set(resolvePath, jsReq, -1)
+		_, err := z.Conn.Create(resolvePath, jsReq, 0, zk.WorldACL(zk.PermAll))
 		if err != nil {
-			log.Print("[zk ERROR] ", err)
+			log.Error("Cant create resolve request node ", err.Error())
 			return err
 		}
 	}
@@ -157,15 +168,15 @@ func (z ZooNode) Answer(fqdn string, req string, stderr []byte, t string) {
 	answer := MountState{t, string(stderr)}
 	answerJSON, err := json.Marshal(answer)
 	if err != nil {
-		log.Print("[ERROR] ", err)
+		log.Error(err.Error())
 	}
 
 	_, err = z.Conn.Create(answerPath, answerJSON, 0, zk.WorldACL(zk.PermAll))
 	if err != nil {
-		log.Print("[zk ERROR] ", err)
+		log.Error("[zk] ", err.Error())
 		_, err := z.Conn.Set(answerPath, answerJSON, -1)
 		if err != nil {
-			log.Print("[zk ERROR] ", err)
+			log.Error("[zk] ", err.Error())
 		}
 	}
 }
@@ -212,11 +223,11 @@ func (z ZooNode) UmountRequest(r RBDDevice) error {
 //WatchAnswer watch for answer
 func (z ZooNode) WatchAnswer(fqdn string, t string) MountState {
 	answersPath := strings.Join([]string{z.Path, "cluster", fqdn, "answers"}, "/")
-	log.Print("[DEBUG] ", answersPath)
+	log.Debug(answersPath)
 	_, _, ch, err := z.Conn.ChildrenW(answersPath)
 	if err != nil {
-		log.Print("[zk ERROR] 107 ", err)
-		return MountState{"FAIL", "Zk error"}
+		log.Error("[zk] ", err.Error())
+		return MountState{"FAIL", "Zk error " + err.Error()}
 	}
 
 	var ms MountState
@@ -269,8 +280,8 @@ func (z ZooNode) CheckMounted(r RBDDevice) (bool, error) {
 	}
 
 	for _, node := range nodes {
-		statePath := strings.Join([]string{z.Path, "cluster", node, "state"}, "/")
 		var nodeState Node
+		statePath := strings.Join([]string{z.Path, "cluster", node, "state"}, "/")
 
 		state, _, err := z.Conn.Get(statePath)
 		if err != nil {

rbmd/sys.go

@@ -3,14 +3,14 @@ package rbmd
 import (
 	"bytes"
 	"io/ioutil"
-	"log"
 	"net"
 	"os/exec"
 	"regexp"
 	"strings"
 	"syscall"
 	"time"
-	// "fmt"
+
+	log "github.com/Sirupsen/logrus"
 )
 
 //ClusterStatus Quorum status struct
@@ -56,7 +56,7 @@ func GetMounts() []Mount {
 		mount := strings.Split(line, " ")
 		match, err := regexp.MatchString("^(/dev/rbd).*$", mount[0])
 		if err != nil {
-			log.Print("[ERROR] ", err)
+			log.Error(err.Error())
 		}
 		if match {
 			p := strings.Split(mount[0], "/")
@@ -92,7 +92,10 @@ func GetRBDPool(device string) (string, string) {
 		log.Fatal("[ERROR] Read failure ", err)
 	}
 
-	return string(pool), string(image)
+	p := strings.Trim(string(pool), "\n")
+	i := strings.Trim(string(image), "\n")
+
+	return p, i
 }
 
 //GetMyIPs Exclude 127.0.0.1
@@ -169,45 +172,20 @@ type RBDDevice struct {
 //MapDevice map rbd block device
 func (r RBDDevice) MapDevice() ([]byte, error) {
 	image := strings.Join([]string{r.Pool, r.Image}, "/")
-	log.Print("[DEBUG] Mapping ", image)
+	log.Warn("[DEBUG] Mapping ", image)
 
 	var stdout bytes.Buffer
 	var stderr bytes.Buffer
 
-	cmd := exec.Command("rbd", "map", image)
+	cmd := exec.Command("/usr/bin/rbd", "map", image)
 
 	cmd.Stdout = &stdout
 	cmd.Stderr = &stderr
 
 	err := cmd.Run()
 	if err != nil {
-		return []byte(stderr.String()), err
-	}
-
-	o := stdout.String()
-
-	if strings.HasSuffix(o, "\n") {
-		o = o[:len(o)-1]
-	}
-
-	return []byte(o), nil
-}
-
-//UnmapDevice unmap rbd block device
-func (r RBDDevice) UnmapDevice() ([]byte, error) {
-	log.Print("[DEBUG] Umapping ", r.Block)
-
-	var stdout bytes.Buffer
-	var stderr bytes.Buffer
-
-	cmd := exec.Command("rbd", "unmap", strings.Join([]string{"/dev/", r.Block}, ""))
-
-	cmd.Stdout = &stdout
-	cmd.Stderr = &stderr
-
-	err := cmd.Run()
-	if err != nil {
-		return []byte(stderr.String()), err
+		log.Error(err.Error())
+		return []byte(strings.Join([]string{stderr.String(), stdout.String()}, " ")), err
 	}
 
 	o := stdout.String()
@@ -223,7 +201,7 @@ func (r RBDDevice) UnmapDevice() ([]byte, error) {
 func (r RBDDevice) MountFS(device string) error {
 	err := syscall.Mount(device, r.Mountpoint, r.Fstype, ParseMountOpts(r.Mountopts), "")
 	if err != nil {
-		log.Print("[DEBUG] sys 207 ", err)
+		log.Error("Cant mount ", device, err.Error())
 		return err
 	}
 
@@ -235,6 +213,7 @@ func ParseMountOpts(mountopts string) uintptr {
 	// Mount options map
 	opts := make(map[string]uintptr)
 	opts["ro"] = syscall.MS_RDONLY
+	opts["posixacl"] = syscall.MS_POSIXACL
 	opts["relatime"] = syscall.MS_RELATIME
 	opts["noatime"] = syscall.MS_NOATIME
 	opts["nosuid"] = syscall.MS_NOSUID
@@ -252,11 +231,38 @@ func ParseMountOpts(mountopts string) uintptr {
 	return 0
 }
 
+//UnmapDevice unmap rbd block device
+func (r RBDDevice) UnmapDevice() ([]byte, error) {
+	log.Print("[DEBUG] Umapping ", r.Block)
+
+	var stdout bytes.Buffer
+	var stderr bytes.Buffer
+
+	cmd := exec.Command("/usr/bin/rbd", "unmap", strings.Join([]string{"/dev/", r.Block}, ""))
+
+	cmd.Stdout = &stdout
+	cmd.Stderr = &stderr
+
+	err := cmd.Run()
+	if err != nil {
+		return []byte(stderr.String()), err
+	}
+
+	o := stdout.String()
+
+	if strings.HasSuffix(o, "\n") {
+		o = o[:len(o)-2]
+	}
+
+	return []byte(o), nil
+}
+
 //UnmountFS unmount file system
 func (r RBDDevice) UnmountFS() error {
 	err := syscall.Unmount(r.Mountpoint, 0)
+	log.Info("Try to umount ", r.Mountpoint)
 	if err != nil {
-		log.Print("[DEBUG] sys 207 ", err)
+		log.Error("Cant umount ", r.Mountpoint, err.Error())
 		return err
 	}
 

rbmd/version.go

@@ -8,6 +8,6 @@ import (
 
 //VersionShow show version and exit
 func VersionShow() {
-	fmt.Println("RBMD 0.1", runtime.Version(), runtime.GOARCH)
+	fmt.Println("RBMD 0.2 test", runtime.Version(), runtime.GOARCH)
 	os.Exit(1)
 }

rbmd/zoo.go

@@ -1,9 +1,10 @@
 package rbmd
 
 import (
-	"github.com/samuel/go-zookeeper/zk"
 	"log"
 	"strings"
+
+	"github.com/samuel/go-zookeeper/zk"
 	// "encoding/json"
 )
 
@@ -15,13 +16,13 @@ type ZooNode struct {
 }
 
 //EnsureZooPath create zookeeper path
-func (z ZooNode) EnsureZooPath(node string) (string, error) {
+func (z ZooNode) EnsureZooPath(path string) (string, error) {
+	path = strings.Join([]string{z.Path, path}, "/")
+
 	flag := int32(0)
 	acl := zk.WorldACL(zk.PermAll)
 
-	zoopath := strings.Join([]string{z.Path, "/", node}, "")
-	s := strings.Split(zoopath, "/")
-
+	s := strings.Split(path, "/")
 	var p []string
 	var fullnodepath string
 
@@ -31,7 +32,10 @@ func (z ZooNode) EnsureZooPath(node string) (string, error) {
 
 	for i := 0; i < len(p); i++ {
 		fullnodepath = strings.Join([]string{fullnodepath, p[i]}, "")
-		z.Conn.Create(fullnodepath, []byte(""), flag, acl)
+		exists, _, _ := z.Conn.Exists(fullnodepath)
+		if !exists {
+			z.Conn.Create(fullnodepath, []byte(""), flag, acl)
+		}
 	}
 
 	return fullnodepath, nil