Merge branch 'master' of vcs.bingo-boom.ru:difrex/surok into test

.gitignore

@@ -2,3 +2,4 @@ surok/__pycache__
 __pycache__
 *.pyc
 *.swp
+selfcheck

LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2106, Denis Zheleztsov <difrex.punk@gmail.com>
+Copyright (c) 2016, Denis Zheleztsov <difrex.punk@gmail.com>
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:

README.en.md

@@ -0,0 +1,8 @@
+# Surok
+
+Service discovery for Apache Mesos.
+
+* Jinja2 Templates
+* Discovery over mesos-dns
+* Applications config reload
+

conf/surok.json

@@ -1,7 +1,12 @@
 {
-    "marathon": "TODO",
+    "marathon": {
-    "confd": "conf.d",
+				"force": true,
+				"host": "http://marathon.mesos:8080",
+				"enabled": true
+		},
+    "confd": "/etc/surok/conf.d",
     "domain": "marathon.mesos",
     "wait_time": 20,
-    "lock_dir": "/var/tmp"
+    "lock_dir": "/var/tmp",
+    "loglevel": "info"
 }

debian/changelog

@@ -1,3 +1,43 @@
+surok (0.7-3) jessie; urgency=medium
+
+  * Small fix in marathon restart
+
+ -- Denis Zheleztsov <difrex@bingo-boom.ru>  Fri, 14 Oct 2016 12:41:31 +0300
+
+surok (0.7-2) testing; urgency=medium
+
+  * Fixed sys module
+
+ -- Denis Zheleztsov <difrex@bingo-boom.ru>  Fri, 14 Oct 2016 11:53:45 +0300
+
+surok (0.7-1) testing; urgency=medium
+
+  * New dependencies
+
+ -- Denis Zheleztsov <difrex@bingo-boom.ru>  Fri, 14 Oct 2016 11:39:36 +0300
+
+surok (0.7) testing; urgency=medium
+
+  * Marathon restart implementation
+  * WARNING: BROKEN BACKWARD COMPATIBILY WITH OLD MAIN CONFIG
+    !!! Please update your config first !!!
+
+ -- Denis Zheleztsov <difrex@bingo-boom.ru>  Fri, 14 Oct 2016 11:28:07 +0300
+
+surok (0.5.5) testing; urgency=medium
+
+  * #closes SD-10
+  * Group switch
+  * Version bump
+
+ -- Denis Zheleztsov <difrex@bingo-boom.ru>  Tue, 11 Oct 2016 15:14:44 +0300
+
+surok (0.3.2) testing; urgency=medium
+
+  * Remove ending '.' in hostname. (for those fucking libs that knows nothing about RFC) 
+
+ -- Denis Ryabyy <vv1r0x@gmail.com>  Fri, 12 Aug 2016 10:48:12 +0300
+
 surok (0.1-1) testing; urgency=low
 
   * Initial release (Closes: BBONL-1696)

debian/control

@@ -8,5 +8,5 @@ Vcs-Git: http://vcs.bingo-boom.ru/difrex/surok.git
 
 Package: surok
 Architecture: all
-Depends: python3-jinja2, python3-dnsq
+Depends: python3-jinja2, python3-dnsq, python3-requests
 Description: Service discovery for Apache Mesos clusters

debian/install

@@ -1,6 +1,6 @@
 conf/surok.json etc/surok/conf
-surok/templates.py usr/lib/python3/dist-packages/surok
+surok/templates.py opt/surok/surok
-surok/system.py usr/lib/python3/dist-packages/surok
+surok/system.py opt/surok/surok
-surok/__init__.py usr/lib/python3/dist-packages/surok
+surok/__init__.py opt/surok/surok
-surok/discovery.py usr/lib/python3/dist-packages/surok
+surok/discovery.py opt/surok/surok
-surok.py usr/bin
+surok.py opt/surok

debian/patches/path-change

@@ -1,36 +0,0 @@
-Description: <short summary of the patch>
- TODO: Put a short summary on the line above and replace this paragraph
- with a longer explanation of this change. Complete the meta-information
- with other relevant fields (see below for details). To make it easier, the
- information below has been extracted from the changelog. Adjust it or drop
- it.
- .
- surok (0.1-1) unstable; urgency=low
- .
-   * Initial release (Closes: #nnnn)  <nnnn is the bug number of your ITP>
-Author: Denis Zheleztsov <difrex@bingo-boom.ru>
-
----
-The information above should follow the Patch Tagging Guidelines, please
-checkout http://dep.debian.net/deps/dep3/ to learn about the format. Here
-are templates for supplementary fields that you might want to add:
-
-Origin: <vendor|upstream|other>, <url of original patch>
-Bug: <url in upstream bugtracker>
-Bug-Debian: https://bugs.debian.org/<bugnumber>
-Bug-Ubuntu: https://launchpad.net/bugs/<bugnumber>
-Forwarded: <no|not-needed|url proving that it has been forwarded>
-Reviewed-By: <name and email of someone who approved the patch>
-Last-Update: <YYYY-MM-DD>
-
---- surok-0.1.orig/surok.py
-+++ surok-0.1/surok.py
-@@ -9,7 +9,7 @@ from surok.discovery import resolve
- from surok.system import reload_conf
- 
- # Load base configurations
--f = open('conf/surok.json', 'r')
-+f = open('/etc/surok/conf/surok.json', 'r')
- conf = json.loads(f.read())
- print(conf)
- f.close()

debian/patches/series

@@ -1 +0,0 @@
-path-change

debian/source/format

@@ -1 +0,0 @@
-3.0 (quilt)

doc/ru/app.configuration.md

@@ -0,0 +1,33 @@
+# Конфигурация приложения
+
+/etc/surok/conf.d/app.json
+```
+{
+    "services": [
+      {
+        "name": "kioskservice",
+        "group": "production.romania",
+        "ports": ["web", "socket"]
+      }
+    ],
+    "conf_name": "kiosk",
+    "template": "/etc/surok/templates/kiosk.jj2",
+    "dest": "/etc/nginx/sites-available/kioskservice.conf",
+    "reload_cmd": "/bin/systemctl reload nginx",
+    "run_cmd": ["/usr/bin/node", "-c", "config.json"]
+}
+```
+
+Давайте разберем конфигурационный файл по опциям
+* services - array. Список хэшей с описанием сервисов
+    name - string. Имя сервиса. Это имя приложения в marathon
+    group - string. Группа в которой находится сервис. Группу можно узнать в marathon. Записывается в обратном порядке. Т.е. если у нас есть группа /webapps/php, то записывать её следует, как php.webapps
+    Если группа не указана, то сурок ожидает группу в переменной окружения SUROK_DISCOVERY_GROUP, если и SUROK_DISCOVERY_GROUP нет, то берется группа marathon(0.5.5).
+    ports - array.  Список имен портов сервиса. Не обязательная опция.
+* conf_name - string. Название конфига. Должен быть уникальным значением. Слежит для создания и чтения lock конфигурации.
+* template - string. Абсолютный путь к файлу шаблона.
+* dest - string. Абсолютный путь к файлу в который запишется результат генерации шаблона.
+* reload_cmd - string. Команда, которая будет выполнена в случае обноления конфига.
+    В reload_cmd можно использовать переменные окружения:
+    ```"reload_cmd": "/usr/bin/killall -9 calc || true && /usr/local/bin/calc -c /app/calc.conf ${CALC_NUM}"```
+* run_cmd(v0.6) - array. Список с командой на выполнение. Используется внутри контейнера вместо reload_cmd.

doc/ru/surok.configuration.md

@@ -0,0 +1,23 @@
+# Конфигурация Surok
+
+**/etc/surok/conf/surok.json**
+Разберем конфигурационный файл по опциям
+```
+{
+    "marathon": "10.0.1.199:8080",
+    "confd": "/etc/surok/conf.d",
+    "domain": "marathon.mesos",
+    "wait_time": 20,
+    "lock_dir": "/var/tmp",
+    "loglevel": "info|debug"
+    "container": true|false
+}
+```
+
+* marathon(v0.7) - string. Адрес Marathon Sheduler.
+* confd - strig. Абсолютный путь до директории с конфигурационными файлами приложений.
+* domain - string. Домен, который обслуживает mesos-dns.
+* wait_time - int. Время в секундах сколько Surok ждет до того, как начать заново делать запросы на обнаружение сервисов.
+* lock_dir - string. Абсолютный путь до директории с lock-конфигурациями.
+* loglevel - string. Уровень логирования.
+* container(v0.6) - boolean. Определяем внутри или нет контейнера запущен сурок. Меняется логика работы.

doc/ru/templates.md

@@ -0,0 +1,104 @@
+# Шиблоны
+
+Шаблоны для Surok пишутся на Jinja2. Возможно, стоит прочитать документацию.
+
+## Словарь my в шаблоне
+
+Surok заполняет словарь my и передает его в шаблон.
+```
+{
+   "services": {
+     "nginx": [
+         {
+          "name": "nginx.testing-kl92-s0.marathon.mesos.",
+          "port": "31200"
+         },
+         {
+          "name": "nginx.testing-kl123-s1.marathon.mesos.",
+          "port": "32230"
+         }
+      ],
+      "emailsender": [
+         {
+          "name": "emailsender.testing-kl92-s0.marathon.mesos.",
+          "port": "31201"
+         },
+         {
+          "name": "emailsender.testing-kl123-s1.marathon.mesos.",
+          "port": "32232"
+         }
+      ],
+     "service-with-defined-ports": {
+         "web": [
+           {
+             "name": "f.q.d.n",
+             "port": 12341
+           }
+         ],
+         "rpc": [
+           {
+             "name": "f.q.d.n",
+             "port": 12342
+           }
+         ]
+     }
+   },
+   "env": {
+      "HOME": "/var/lib/nginx"
+   }
+}
+```
+
+## Пример реального шаблона
+
+```
+upstream matrix-http {
+    hash $remote_addr;
+{% for server in my['services']['matrix']['http'] %}
+    server {{server['name']}}:{{server['port']}} max_fails=3;
+{% endfor %}
+}
+ 
+upstream riot-http {
+    hash $remote_addr;
+{% for server in my['services']['riot'] %}
+    server {{server['name']}}:{{server['port']}} max_fails=3;
+{% endfor %}
+}
+ 
+server {
+    listen 10.15.56.157:80;
+    server_name matrix.example.com;
+ 
+    client_max_body_size 10m;
+ 
+    location / {
+        proxy_pass http://riot-http;
+        proxy_set_header  X-Real-IP        $remote_addr;
+        proxy_set_header  Host             $http_host;
+        proxy_set_header  X-Forwarded-For  $proxy_add_x_forwarded_for;
+    }
+ 
+    location /_matrix/ {
+        proxy_pass http://matrix-http;
+        proxy_set_header  X-Real-IP        $remote_addr;
+        proxy_set_header  Host             $http_host;
+        proxy_set_header  X-Forwarded-For  $proxy_add_x_forwarded_for;
+    }
+ 
+}
+```
+Так для upstream matrix-http используются именованные порты, а для riot-http – нет.
+
+## Проверки в шаблоне
+
+Переменная _my['env']_ является классом python _os.environ_, что позваоляет нам строить различные проверки, например:
+
+```
+{% if my['env'].get('DB_HOST') %}
+host = '{{my['env']['DB_HOST']}}'
+{% else %}
+host = 'localhost'
+{% endif %}
+```
+

surok.py

@@ -1,6 +1,7 @@
 #!/usr/bin/python3
 
 from time import sleep
+import os
 from os import listdir
 from os.path import isfile, join
 import json
@@ -23,24 +24,27 @@ if args.config:
 # Read config file
 f = open(surok_conf, 'r')
 conf = json.loads(f.read())
-print(conf)
 f.close()
 
 
 # Get app configurations
 # Return list of patches to app discovery configuration
 def get_configs():
-    confs = [f for f in listdir(conf['confd']) if isfile( join(conf['confd'], f) )]
+    confs = [f for f in listdir(conf['confd']) if isfile(
+        join(conf['confd'], f))]
     return confs
 
 
 # Get Surok App configuration
 # Read app conf from file and return dict
 def load_app_conf(app):
-    f = open( conf['confd'] + '/' + app )
+    # Load OS environment to app_conf
-    c = json.loads( f.read() )
+    f = open(conf['confd'] + '/' + app)
+    c = json.loads(f.read())
     f.close()
 
+    c['env'] = os.environ
+
     return c
 
 
@@ -56,20 +60,23 @@ while 1:
     for app in confs:
         app_conf = load_app_conf(app)
 
+        # Will be removed later
+        # For old configs
+        try:
+            loglevel = conf['loglevel']
+        except:
+            conf['loglevel'] = 'info'
+
         # Resolve services
         app_hosts = resolve(app_conf, conf)
 
         # Populate my dictionary
-        my = { "services": app_hosts,
+        my = {"services": app_hosts,
-               "conf_name": app_conf['conf_name']
+              "conf_name": app_conf['conf_name']}
-        }
 
         # Generate config from template
         service_conf = gen(my, app_conf['template'])
 
-        stdout, first =  reload_conf(service_conf, app_conf, first)
+        first = reload_conf(service_conf, app_conf, first, conf)
-        print(stdout)
-
-
-    sleep( conf['wait_time'] )
 
+    sleep(conf['wait_time'])

surok/discovery.py

@@ -1,18 +1,106 @@
 import dns.resolver
+import dns.query
+from dns.exception import DNSException
+import logging
+import sys
 
+
+# Logger configuration
+# This need to be moved
+def get_logger():
+    # Configure logging
+    FORMAT = '%(asctime) %(message)s'
+    logging.basicConfig(format=FORMAT)
+    logger = logging.getLogger(__name__)
+    return logger
+
+
+# Resolve service from mesos-dns SRV record
+# return dict {"servicename": [{"name": "service.f.q.d.n.", "port": 9999}]}
 def resolve(app, conf):
     hosts = {}
     services = app['services']
     domain = conf['domain']
-    for service in services:
+    logger = get_logger()
-        hosts[service['name']] = []
-        try:
-            for rdata in dns.resolver.query('_' + service['name'] + '.' + service['group'] + '._tcp.' + domain, 'SRV'):
-                info = str(rdata).split()
-                server = { 'name': info[3], 'port': info[2] }
 
-                hosts[ service['name'] ].append(server)
+    for service in services:
-        except Exception as e:
+        hosts[service['name']] = {}
-            print("Could not resolve " + service['name'] + '.' + service['group'] + '._tcp.' + domain)
+
+        group = get_group(service, app)
+        if group is False:
+            logger.error('Group is not defined in config, SUROK_DISCOVERY_GROUP and MARATHON_APP_ID')
+            logger.error('Not in Mesos launch?')
+            sys.exit(2)
+        
+        # Port name from app config
+        ports = None
+        try:
+            ports = service['ports']
+        except:
+            pass
+
+        # This is fast fix for port naming
+        # Will be rewrite later
+        fqdn = ''
+        if ports is not None:
+            for port_name in ports:
+                fqdn = '_' + port_name + '.' + '_' + service['name'] + '.' + group + '._tcp.' + domain
+                hosts[service['name']][port_name] = do_query(fqdn, conf['loglevel'])
+        else:          
+            fqdn = '_' + service['name'] + '.' + group + '._tcp.' + domain
+            hosts[service['name']] = do_query(fqdn, conf['loglevel'])
 
     return hosts
+
+
+# Do SRV queries
+# Return array: [{"name": "f.q.d.n", "port": 8876}]
+def do_query(fqdn, loglevel):
+    logger = get_logger()
+    servers = []
+    try:
+        query = dns.resolver.query(fqdn, 'SRV')
+        query.lifetime = 1.0
+
+        for rdata in query:
+            info = str(rdata).split()
+            server = {'name': info[3][:-1], 'port': info[2]}
+            servers.append(server)
+    except DNSException as e:
+        if loglevel != 'info':
+            logger.error("Could not resolve " + fqdn + ': ' + str(e))
+
+    return servers
+
+
+# Groups switch
+# Priority: config, environment, marathon environment
+def get_group(service, app):
+    # Check group in app conf
+    if 'group' in service:
+        return service['group']
+    # Check environment variable
+    elif app['env'].get('SUROK_DISCOVERY_GROUP'):
+        return app['env']['SUROK_DISCOVERY_GROUP']
+    # Check marathon environment variable
+    elif app['env'].get('MARATHON_APP_ID'):
+        group = parse_marathon_app_id(app['env']['MARATHON_APP_ID'])
+        return group
+    else:
+        return False
+
+
+# Parse MARATHON_APP_ID
+# Return marathon.group
+def parse_marathon_app_id(marathon_app_id):
+    marathon_app_id = marathon_app_id.split('/')
+    group = ''
+    counter = len(marathon_app_id) - 2
+    i = 0
+    while counter > i:
+        group = group + marathon_app_id[counter]
+        if counter != i + 1:
+            group += '.'
+        counter -= 1
+
+    return group

surok/system.py

@@ -1,6 +1,10 @@
 import os
+import sys
+import logging
+import requests
 
 
+# Get old configuration
 def get_old(name, service_conf):
 
     try:
@@ -26,7 +30,7 @@ def write_lock(name, service_conf):
 
 
 def do_reload(service_conf, app_conf):
-    print( 'Write new configuration of ' + app_conf['conf_name'] )
+    logging.warning('Write new configuration of ' + app_conf['conf_name'])
 
     f = open(app_conf['dest'], 'w')
     f.write(service_conf)
@@ -39,16 +43,46 @@ def do_reload(service_conf, app_conf):
     return stdout
 
 
-def reload_conf(service_conf, app_conf, first):
+def reload_conf(service_conf, app_conf, first, conf):
 
     # Check first loop
-    if first == True:
+    if first is True:
         stdout = do_reload(service_conf, app_conf)
         first = False
-        return stdout, first
+        logging.info(stdout)
+        return first
+
+    # Check marathon enabled in configuration
+    if conf['marathon']['enabled'] is True:
+        restart_self_in_marathon(conf['marathon'])
         
     if get_old(app_conf['conf_name'], service_conf) != 1:
         stdout = do_reload(service_conf, app_conf)
-        return stdout, first
+        logging.info(stdout)
+        return first
     else:
-        return 'Same config ' + app_conf['conf_name'] + ' Skip reload', first
+        if conf['loglevel'] == 'debug':
+            logging.debug('Same config ' +
+                          app_conf['conf_name'] +
+                          ' Skip reload')
+        return first
+
+
+# Do POST request to marathon API
+# /v2/apps//app/name/restart
+def restart_self_in_marathon(marathon):
+    host = marathon['host']
+
+    # Check MARATHON_APP_ID environment varible
+    if os.environ.get('MARATHON_APP_ID') is not True:
+        logging.error('Cannot find MARATHON_APP_ID. Not in Mesos?')
+        sys.exit(2)
+    app_id = os.environ['MARATHON_APP_ID']
+    uri = 'http://' + host + '/v2/apps/' + app_id + '/restart'
+
+    # Ok. In this step we made restart request to Marathon
+    if marathon['force'] is True:
+        r = requests.post(uri, data = {'force': 'true'})
+    else:
+        r = requests.post(uri, data = {'force': 'false'})
+    

surok/templates.py

@@ -1,4 +1,4 @@
-from jinja2 import Environment, PackageLoader, Template
+from jinja2 import Template
 import os