Difrex/surok
Difrex/surok
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #17 from Surkoveds/doc

Browse Source 
Release 0.8
This commit is contained in:
Denis 2017-02-07 16:04:34 +02:00 committed by GitHub
commit 44a10b7fbb
26 changed files with 1668 additions and 380 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
AUTHORS
View File

@ -1,2 +1,3 @@
Denis Zheleztsov <difrex.punk@gmail.com>
Denis Ryabyy <vv1r0x@gmail.com>
Evgeniy Vasilev <oren-ibc@yandex.ru>

19
README.ru.md
View File

@ -1,5 +1,5 @@
# Surok
[![Build Status](https://travis-ci.org/Difrex/surok.svg?branch=master)](https://travis-ci.org/Difrex/surok)
[![Build Status](https://travis-ci.org/Surkoveds/surok.svg?branch=master)](https://travis-ci.org/Surkoveds/surok)
Обнаружение сервисов для Apache Mesos.
@ -17,10 +17,27 @@ cd build
deb-пакет будет лежать в build/out
Сборка базового docker-образа surok
Ubuntu Xenial
```
cd build
./build.sh surok_image
```
Alpine image
```
cd build
./build.sh alpine
```
CentOS image
```
cd build
./build.sh centos
```
ENTRYPOINT : ```cd /opt/surok && pytho3 surok.py -c /etc/surok/conf/surok.json```
## Документация
[Wiki](https://github.com/Surkoveds/surok/wiki)
## Известные проблемы

2
build/runtests.sh
View File

@ -5,7 +5,7 @@ set -e
. functions.sh
function run_tests() {
docker run -ti -v $(pwd)/tests.py:/opt/surok/tests.py \
docker run --rm -ti -v $(pwd)/tests.py:/opt/surok/tests.py \
-v $(pwd)/tests_entrypoint.sh:/tests_entrypoint.sh \
--entrypoint /tests_entrypoint.sh \
surok_base:latest

423
build/tests.py Normal file → Executable file
View File

@ -1,71 +1,372 @@
#!/usr/bin/python3
import unittest
import json
import os
import re
import sys
import surok.config
import surok.logger
import surok.discovery
import hashlib
from surok.config import Config
class Logger(surok.logger.Logger):
_out=''
_err=''
def _log2err(self,out):
self._err+=out
def _log2out(self,out):
self._out+=out
def geterr(self):
return self._err
def getout(self):
return self._out
def reset(self):
self._err=''
self._out=''
class DiscoveryTestingTemplate:
_testing={}
_testing_fqdn_a={
"test.zzz0.test":['10.0.0.1','10.1.0.1'],
"test.zzz1.test":['10.0.1.1','10.1.1.1'],
"test.zzz2.test":['10.0.2.1','10.1.2.1'],
"test.zzz3.test":['10.0.3.1','10.1.3.1']
}
_testing_fqdn_srv={}
def __init__(self):
super().__init__()
self._orig_logger=surok.logger.Logger()
def do_query_a(self,fqdn):
res=self._testing_fqdn_a.get(fqdn,[])
if res:
return res
else:
self._orig_logger.error('Testing FQDN '+fqdn+' not found in test A records')
sys.exit(2)
def do_query_srv(self,fqdn):
res=self._testing_fqdn_srv.get(fqdn,[])
if res or fqdn.startswith('_tname_e.') or fqdn.find('._udp.'):
return res
else:
self._orig_logger.error('Testing FQDN '+fqdn+' not found in test SRV records')
sys.exit(2)
def update_data(self):
class_name=self.__class__.__name__
tgen={
"name": ["zzz0","zzy0","zzy1","zzz1"],
"host": ["test.zzz0.test","test.zzz1.test","test.zzz2.test","test.zzz3.test"],
"serv": ["tname_aa","tname_ab","tname_ba","tname_bb"],
"ports": [12341,12342,12343,12344],
"servicePorts": [21221,21222,21223,21224]
}
if self._testing.get(class_name,True):
if class_name == 'DiscoveryMarathon':
_tasks=[]
_ports={}
for id in (0,1,2,3):
ports=[]+tgen['ports']
servicePorts=[]+tgen['servicePorts']
appId='/'.join(str(tgen['name'][id]+'.xxx.yyy.').split('.')[::-1])
_ports[appId]=[]
for pid in (0,1,2,3):
ports[pid]+=pid*10
servicePorts[pid]+=pid*100
for prot in ['tcp','udp']:
if pid<2 or prot == 'tcp':
_ports[appId].append({'containerPort': 0,
'hostPort': 0,
'labels': {},
'name': tgen['serv'][pid],
'protocol': prot,
'servicePort': servicePorts[pid]})
_tasks.append({'appId':appId,
'host':tgen['host'][id],
'ports':ports,
'servicePorts':servicePorts})
#_tname_a._zzy0.yyy.xxx._tcp.marathon.mesos
self._tasks=_tasks
self._ports=_ports
elif class_name == 'DiscoveryMesos':
for id in (0,1,2,3):
ports=[]+tgen['ports']
for pid in (0,1,2,3):
ports[pid]+=pid*10
for prot in ['tcp','udp']:
if pid<2 or prot == 'tcp':
for fqdn in ['_'+tgen['serv'][pid]+'._'+tgen['name'][id]+'.xxx.yyy._'+prot+'.'+self._config['mesos'].get('domain'),
'_'+tgen['name'][id]+'.xxx.yyy._'+prot+'.'+self._config['mesos'].get('domain')]:
if not self._testing_fqdn_srv.get(fqdn):
self._testing_fqdn_srv[fqdn]=[]
self._testing_fqdn_srv[fqdn].append({'name':tgen['host'][id],'port':ports[pid]})
self._testing[class_name]=False
class DiscoveryMesos(DiscoveryTestingTemplate,surok.discovery.DiscoveryMesos):
pass
class DiscoveryMarathon(DiscoveryTestingTemplate,surok.discovery.DiscoveryMarathon):
pass
class Discovery(surok.discovery.Discovery):
_discoveries={}
def __init__(self):
self._config=Config()
self._logger=Logger()
if not self._discoveries.get('mesos_dns'):
self._discoveries['mesos_dns']=DiscoveryMesos()
if not self._discoveries.get('marathon_api'):
self._discoveries['marathon_api']=DiscoveryMarathon()
class Test01_Logger(unittest.TestCase):
def test_01_logger_default_level(self):
logger = Logger()
self.assertEqual(logger.get_level(), 'info')
def test_02_logger_output_levels(self):
message='log message'
tests={
'debug':{
'assertIn':['ERROR: {}','WARNING: {}','INFO: {}','DEBUG: {}'],
'assertNotIn':[]
},
'info':{
'assertIn':['ERROR: {}','WARNING: {}','INFO: {}'],
'assertNotIn':['DEBUG: {}']
},
'warning':{
'assertIn':['ERROR: {}','WARNING: {}'],
'assertNotIn':['INFO: {}','DEBUG: {}']
},
'error':{
'assertIn':['ERROR: {}'],
'assertNotIn':['WARNING: {}','INFO: {}','DEBUG: {}']
}
}
logger = Logger()
for value01 in tests.keys():
logger.reset()
logger.set_level(value01)
logger.error(message)
logger.warning(message)
logger.info(message)
logger.debug(message)
resmessage=logger.geterr()+logger.getout()
for test_name in tests[value01].keys():
for test_value in tests[value01][test_name]:
with self.subTest(msg='Testing Logger for ...', loglevel=value01):
test_message=test_value.format(message)
eval('self.{}(test_message,resmessage)'.format(test_name))
class Test02_LoadConfig(unittest.TestCase):
def test_01_default_values(self):
config=Config()
with self.subTest(msg='Testing default values for Config.', dump=config.dump()):
self.assertEqual(config.get('confd'), '/etc/surok/conf.d')
self.assertEqual(config.get('default_discovery'), 'mesos_dns')
self.assertEqual(config.get('lock_dir'), '/var/tmp')
self.assertEqual(config.get('loglevel'), 'info')
self.assertEqual(dict(config.get('marathon',{})).get('enabled'), False)
self.assertEqual(dict(config.get('mesos',{})).get('enabled'), False)
self.assertEqual(dict(config.get('memcached',{})).get('enabled'), False)
self.assertEqual(config.get('version'), '0.7')
self.assertEqual(config.get('wait_time'), 20)
def test_02_main_conf(self):
config=Config('/etc/surok/conf/surok.json')
with self.subTest(msg='Testing load config for Config.', dump=config.dump()):
self.assertEqual(config.hash(), '545c20b322a6ba5fef9c7d2416d80178f26a924b')
def test_03_apps_conf(self):
tests=[
{
'env':{},
'self_check.json':'a4e109b9fec696776fd3df091b607e9c1489748c',
'marathon_check.json':'6be7f26d421d4a0a2e7b089184be0c0e3a50f986'
},
{
'env':{'SUROK_DISCOVERY_GROUP':'xxx.yyy'},
'self_check.json':'38ab770ff2ba69bf70673288425337ff3c18a807',
'marathon_check.json':'7b0cb4eab2d8e0f901cc567df28b17279af21baa'
},
{
'env':{'MARATHON_APP_ID':'/xxx/yyy/zzz'},
'self_check.json':'cbd2a15179649d0e06f98bd64e024481a944d65c',
'marathon_check.json':'08a382d14285feb1f22b92ba597ecf73d654a2e0'
}
]
config=Config()
for test in tests:
config.set('env',test['env'])
config.update_apps()
for app in config.apps:
with self.subTest(msg='Testing AppConfig for ...', env=test['env'], conf_name=app.get('conf_name'), dump=app.dump()):
self.assertEqual(test[app.get('conf_name')],app.hash())
def test_04_apps_conf(self):
tests={
'confd':{
'assertEqual': ['/var', '/var/tmp', '/etc/surok/conf.d'],
'assertNotEqual': [20, '/var/tmp1', '/etc/surok/conf/surok.json', 1, None, True]
},
'default_discovery':{
'assertEqual':['marathon_api', 'mesos_dns'],
'assertNotEqual':[20, 'test', None]
},
'lock_dir':{
'assertEqual':['/var', '/etc/surok/conf.d', '/var/tmp'],
'assertNotEqual':[20, '/var/tmp1', '/etc/surok/conf/surok.json', 1, None, True]
},
'loglevel':{
'assertEqual':['error', 'debug', 'info', 'warning'],
'assertNotEqual':['errrr', 'DEBUG','warn', 'test', 1, None, True]
},
'version':{
'assertEqual': ['0.7', '0.8'],
'assertNotEqual': ['0,7', '07', '0.9', 0.7, 0.8, None]
},
'wait_time':{
'assertEqual': [10, 15, 20],
'assertNotEqual': ['10', '15', None, True]
}
}
config=Config()
for name01 in tests.keys():
oldvalue=config.get(name01)
for test_name in tests[name01].keys():
for value01 in tests[name01][test_name]:
config.set_config({name01:value01})
test_value=config.get(name01)
with self.subTest(msg='Testing Config Change for values...', name=name01, value=value01, test_value=test_value):
eval('self.{}(value01, test_value)'.format(test_name))
config.set(name01,oldvalue)
class TestLoadConfig(unittest.TestCase):
def test_main_conf(self):
# Load base configurations
surok_conf = '/etc/surok/conf/surok.json'
# Read config file
f = open(surok_conf, 'r')
conf = json.loads(f.read())
f.close()
self.assertIn('confd', conf)
self.assertTrue(os.path.isdir(conf['confd']))
self.assertIn('domain', conf)
self.assertIn('wait_time', conf)
self.assertIn('lock_dir', conf)
self.assertTrue(os.path.isdir(conf['lock_dir']))
class TestLogger(unittest.TestCase):
def test_info(self):
from surok.logger import make_message
m = make_message
self.assertIn('INFO', m({'level': 'INFO', 'raw': 'log message'}))
def test_warning(self):
from surok.logger import make_message
m = make_message
self.assertIn('WARNING', m({'level': 'WARNING', 'raw': 'log message'}))
def test_error(self):
from surok.logger import make_message
m = make_message
self.assertIn('ERROR', m({'level': 'ERROR', 'raw': 'log message'}))
def test_info(self):
from surok.logger import make_message
m = make_message
self.assertIn('DEBUG', m({'level': 'DEBUG', 'raw': 'log message'}))
class TestMemcachedDiscovery(unittest.TestCase):
def test_discovery_memcache(self):
from surok.system import discovery_memcached
# Load base configurations
surok_conf = '/etc/surok/conf/surok.json'
# Read config file
f = open(surok_conf, 'r')
conf = json.loads(f.read())
f.close()
self.assertEqual(discovery_memcached(conf), [])
class TestGetGroup(unittest.TestCase):
def test_get_group(self):
from surok.discovery import get_group
self.assertFalse(get_group({}, {'env': os.environ}))
class Test03_Discovery(unittest.TestCase):
def test_01_discovery(self):
tests={
'T':{ #mesos_enabled
'T':{ #marathon_enabled
'0.7':{ #version
'mesos_dns':{ #default_discovery
'marathon_check.json':'ef55fb10c20df700cb715f4836eadb2d0cfa9cc1', #app['conf_name']
'self_check.json':'53b8ddc27e357620f01ea75a7ab827cd90c77446'
},
'marathon_api':{
'marathon_check.json':'ef55fb10c20df700cb715f4836eadb2d0cfa9cc1',
'self_check.json':'53b8ddc27e357620f01ea75a7ab827cd90c77446'
}
},
'0.8':{
'mesos_dns':{
'marathon_check.json':'2016238426eb8ee7df9c4c016b6aecbfdf251a9b',
'self_check.json':'b6279a3e8e2fbbc78c6b302ef109bd6e2b456d9f'
},
'marathon_api':{
'marathon_check.json':'2016238426eb8ee7df9c4c016b6aecbfdf251a9b',
'self_check.json':'b6279a3e8e2fbbc78c6b302ef109bd6e2b456d9f'
}
}
},
'F':{
'0.7':{
'mesos_dns':{
'marathon_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f',
'self_check.json':'53b8ddc27e357620f01ea75a7ab827cd90c77446'
},
'marathon_api':{
'marathon_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f',
'self_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f'
}
},
'0.8':{
'mesos_dns':{
'marathon_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f',
'self_check.json':'b6279a3e8e2fbbc78c6b302ef109bd6e2b456d9f'
},
'marathon_api':{
'marathon_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f',
'self_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f'
}
}
}
},
'F':{
'T':{
'0.7':{
'mesos_dns':{
'marathon_check.json':'ef55fb10c20df700cb715f4836eadb2d0cfa9cc1',
'self_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f'
},
'marathon_api':{
'marathon_check.json':'ef55fb10c20df700cb715f4836eadb2d0cfa9cc1',
'self_check.json':'53b8ddc27e357620f01ea75a7ab827cd90c77446'
}
},
'0.8':{
'mesos_dns':{
'marathon_check.json':'2016238426eb8ee7df9c4c016b6aecbfdf251a9b',
'self_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f'
},
'marathon_api':{
'marathon_check.json':'2016238426eb8ee7df9c4c016b6aecbfdf251a9b',
'self_check.json':'b6279a3e8e2fbbc78c6b302ef109bd6e2b456d9f'
}
}
},
'F':{
'0.7':{
'mesos_dns':{
'marathon_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f',
'self_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f'
},
'marathon_api':{
'marathon_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f',
'self_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f'
}
},
'0.8':{
'mesos_dns':{
'marathon_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f',
'self_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f'
},
'marathon_api':{
'marathon_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f',
'self_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f'
}
}
}
}
}
config=Config('/etc/surok/conf/surok.json')
config.set('env',{'SUROK_DISCOVERY_GROUP':'xxx.yyy'})
discovery=Discovery()
for mesos_enabled in tests.keys():
for marathon_enabled in tests[mesos_enabled].keys():
for version in tests[mesos_enabled][marathon_enabled].keys():
for default_discovery in tests[mesos_enabled][marathon_enabled][version].keys():
config.set_config({'default_discovery':default_discovery,
'mesos':{'enabled':(mesos_enabled=='T')},
'marathon':{'enabled':(marathon_enabled=='T')},
'version':version})
discovery.update_data()
for app in config.apps:
conf_name=app.get('conf_name')
with self.subTest(msg='Testing Discovery for values...', config=config.dump(), conf_name=conf_name):
self.assertEqual(hashlib.sha1(json.dumps(discovery.resolve(app), sort_keys=True).encode()).hexdigest(),
tests[mesos_enabled][marathon_enabled][version][default_discovery][conf_name])
if __name__ == '__main__':
unittest.main()

12
conf.d/marathon_check.json Normal file
View File

@ -0,0 +1,12 @@
{
"services": [
{
"name": "zzy*",
"ports": ["tname_a*"]
}
],
"template": "templates/selfcheck.jj2",
"dest": "selfcheck",
"reload_cmd": "/bin/echo selfcheck ok",
"discovery": "marathon_api"
}

22
conf.d/self_check.json Normal file
View File

@ -0,0 +1,22 @@
{
"services": [
{
"name": "zzy0",
"ports": ["tname_aa", "tname_ab", "tname_ba", "tname_bb", "tname_d"]
},
{
"name": "zzy1",
"ports": ["tname_aa", "tname_ab", "tname_ba", "tname_bb"]
},
{
"name": "zzz0",
"ports": ["tname_aa", "tname_bb"]
},
{
"name": "zzz1"
}
],
"template": "templates/selfcheck.jj2",
"dest": "selfcheck",
"reload_cmd": "/bin/echo selfcheck ok"
}

12
conf.d/selfcheck.json
View File

@ -1,12 +0,0 @@
{
"services": [
{
"name": "",
"group": "mesos"
}
],
"conf_name": "selfcheck",
"template": "templates/selfcheck.jj2",
"dest": "selfcheck",
"reload_cmd": "/bin/echo selfcheck ok"
}

13
conf/surok.json
View File

@ -1,15 +1,20 @@
{
"version": "0.8",
"marathon": {
"enabled": false,
"restart": false,
"force": true,
"host": "http://marathon.mesos:8080",
"enabled": false
"host": "http://marathon.mesos:8080"
},
"mesos": {
"enabled": false,
"domain": "marathon.mesos"
},
"default_discovery": "mesos_dns",
"confd": "/etc/surok/conf.d",
"domain": "marathon.mesos",
"wait_time": 20,
"lock_dir": "/var/tmp",
"loglevel": "info",
"container": false,
"memcached": {
"enabled": false,
"discovery": {

9
debian/changelog vendored
View File

@ -1,3 +1,12 @@
surok (0.8) testing; urgency=medium
* Version bump
* new config structure
* Port type
* Discovery over Marathon API
-- Denis Zheleztsov <difrex.punk@gmail.com> Tue, 07 Feb 2017 12:41:52 +0300
surok (0.7.4.3) testing; urgency=medium
* Fixed #9

4
debian/install vendored
View File

@ -1,7 +1,11 @@
conf/surok.json etc/surok/conf
conf.d/self_check.json etc/surok/conf.d
conf.d/marathon_check.json etc/surok/conf.d
templates/selfcheck.jj2 etc/surok/templates
surok/templates.py opt/surok/surok
surok/system.py opt/surok/surok
surok/logger.py opt/surok/surok
surok/__init__.py opt/surok/surok
surok/discovery.py opt/surok/surok
surok/config.py opt/surok/surok
surok.py opt/surok

4
docs/App-config-file.html
View File

@ -34,9 +34,9 @@
}
</code>
* <strong>services</strong>. List of hashes with required services for app.
1. <em>name</em> - string. App name in Marathon.
1. <em>name</em> - string. App name in Marathon. If you use a Marathon discovery, you can use the "*" at the end of the string to indicate any character.
2. <em>group</em> - string. App group in Marathon. Optional. Discovery policy: 1) config 2) SUROK<em>DISCOVERY</em>GROUP environment variable 3) Marathon API
3. <em>ports</em> - list. Name of opened port. In marathon of course. Optional.
3. <em>ports</em> - list. Name of opened port. In marathon of course. If you use a Marathon discovery, you can use the "*" at the end of the string to indicate any character. Optional.
* <strong>conf<em>name</strong>. Unique app config name.
* <strong>template</strong>. Jinja2 template location.
* <strong>dest</strong>. Destination config path.

27
docs/App-config-file.md
View File

@ -5,26 +5,27 @@ conf.d/myapp.json
{
"services": [
{
"name": "myapp",
"group": "backend.production",
"ports": ["proxy", "api"]
},
{
"name": "nginx",
"group": "frontend.production"
"name": "kioskservice",
"group": "production.romania",
"ports": ["web", "socket"]
}
],
"conf_name": "myapp_backend_production",
"template": "/etc/surok/templates/myapp.jj2",
"dest": "/etc/myapp/myapp.cfg",
"reload_cmd": "killall -9 myapp; /usr/local/bin/myapp -config /etc/myapp/myapp.cfg"
"discovery": "mesos_dns",
"group": "dev.web",
"conf_name": "kiosk",
"template": "/etc/surok/templates/kiosk.jj2",
"dest": "/etc/nginx/sites-available/kioskservice.conf",
"reload_cmd": "/sbin/nginx -t && /bin/systemctl reload nginx",
"run_cmd": ["/usr/bin/node", "-c", "config.json"]
}
```
* **services**. List of hashes with required services for app.
1. _name_ - string. App name in Marathon.
1. _name_ - string. App name in Marathon. If you use a Marathon discovery, you can use the "*" at the end of the string to indicate any character.
2. _group_ - string. App group in Marathon. Optional. Discovery policy: 1) config 2) SUROK_DISCOVERY_GROUP environment variable 3) Marathon API
3. _ports_ - list. Name of opened port. In marathon of course. Optional.
3. _ports_ - list. Name of opened port. In marathon of course. If you use a Marathon discovery, you can use the "*" at the end of the string to indicate any character. Optional.
* **conf_name**. Unique app config name.
* **template**. Jinja2 template location.
* **dest**. Destination config path.
* **reload_cmd**. Command to execute if generated config is changed.
* **discovery**. Use custom discovery for app.
* **group**. Default group for all required services.

15
docs/Main-config-file.html
View File

@ -19,12 +19,21 @@
<code>
{
"marathon": {
"enabled": false,
"restart": false,
"force": true,
"host": "http://marathon.mesos:8080",
"enabled": false
"host": "http://marathon.mesos:8080"
},
"consul": {
"enabled": false,
"domain": "service.dc1.consul"
},
"mesos":{
"enabled": true,
"domain": "marathon.mesos"
},
"default_discovery": "mesos_dns",
"confd": "/etc/surok/conf.d",
"domain": "marathon.mesos",
"wait_time": 20,
"lock_dir": "/var/tmp",
"loglevel": "info",

74
docs/Main-config-file.md
View File

@ -1,21 +1,25 @@
# Surok main config file
# Surok main config file (0.8.x)
Default location is /etc/surok/conf/surok.json
Default location is **/etc/surok/conf/surok.json**
conf/surok.json
```
{
"version": "0.8"
"marathon": {
"enabled": false,
"restart": false,
"force": true,
"host": "http://marathon.mesos:8080",
"enabled": false
"host": "http://marathon.mesos:8080"
},
"mesos":{
"enabled": true,
"domain": "marathon.mesos"
},
"default_discovery": "mesos_dns",
"confd": "/etc/surok/conf.d",
"domain": "marathon.mesos",
"wait_time": 20,
"lock_dir": "/var/tmp",
"loglevel": "info",
"container": false,
"memcached": {
"enabled": false,
"discovery": {
@ -27,10 +31,52 @@ conf/surok.json
}
}
```
* **marathon section**. Restarting app over marathon api if config changed. Disabled by default.
* **confd**. Directory where located configs apps.
* **domain**. Domain served by mesos-dns.
* **lock_dir**. Directory where surok writes temporary configs after resolving.
* **wait_time**. Sleep time in main loop.
* **container**. Not implemented.
* **memcached section**. Memcached support. Disabled by default.
## Config file options
* **version** - *string. Optional. "0.7" by default.*
Config files and templates version. Accept "0.7" or "0.8".
* "0.7" - config files <= 0.7.х version
* "0.8" - >= 0.8.x config files version
##### 0.8 version
* **marathon**, **mesos**, **consul**, **memcached** - *dict/hash. Optional. '{"enable":false}'. by default*
Surok working with folowing systems. If system is disabled parameters will be ignored.
* **enable** - *boolean. Optional. false by default*
Enable/disable system for usage.
Specific variables:
* For Marathon API "marathon"
* **force** - *boolean. Optional. true by default*
Force restart container over API.
* **restart** - *boolean. Optional. false by default*
Enable/disable restarting container
* **host** - *string. Optional. "http://marathon.mesos:8080" by default*
Marathon address.
* For mesos-dns "mesos"
* **domain** - *string. Optional. "marathon.mesos" by default*
mesos-dns private domain
* For Memcached "memcached"
* **hosts** - memcached hosts
* **discovery**
* **enabled** - boolean. Enable/disable disovery memcached service
* **service** - string. memcached app name
* **group** - string. memcached app group
* **default_discovery** - *string. Optional. "mesos_dns" by default*
Accept values:
* "mesos_dns" - mesos-dns
* "marathon_api"- Marathon API
* **confd** - *strig. Required.*
Path to directory with app config files.
* **wait_time** - *int. Required.*
Time in seconds how much Surok waits before starting to re-do the requests for service discovery
* **lock_dir** - *string. Required.*
Path to directory where Surok write lock-files.
* **loglevel** - *string. Optional. "info" by default*
Logleve. Accept values: "debug", "info", "warning", "error"
##### < 0.8 versions
* **marathon**
* **enabled** - boolean. Enable/disable container restart. Renamed to "restart" in 0.8 version.
* **domain** - string. mesos-dns private domain. Moved to "mesos" hashtable in 0.8 version.
Discovery over mesos-dns enabled all times.

1
docs/README.md
View File

@ -31,6 +31,7 @@ Service discovery for Apache Mesos.
* Denis Zheleztsov <difrex.punk@gmail.com>
* Denis Ryabyy <vv1r0x@gmail.com>
* Evgeniy Vasilev <oren-ibc@yandex.ru>
## LICENSE

161
docs/Templates.md
View File

@ -5,8 +5,11 @@
- [Templates](#templates)
- [my dictionary in templates](#my-dictionary-in-templates)
- [0.8 version](#08-version)
- [Real template example](#real-template-example)
- [Checks in template](#checks-in-template)
- [0.7 version](#07-version)
- [Real template example](#real-template-example)
<!-- markdown-toc end -->
@ -15,55 +18,74 @@ Surok using Jinja2 for templates. [Jinja2 documentation](http://jinja.pocoo.org/
## my dictionary in templates
### 0.8 version
```
{
"services": {
"nginx": [
"asterisk": [
{
"name": "nginx.testing-kl92-s0.marathon.mesos.",
"port": "31200",
"ip": ["10.10.10.1"]
"ip": [
"10.0.0.1",
"11.0.0.1"
],
"tcp": {
"rpc":31200,
"web":31201,
"sip":32000
},
"udp": {
"sip":31201
}
},
{
"name": "nginx.testing-kl123-s1.marathon.mesos.",
"port": "32230",
"ip": ["10.10.10.2"]
}
"ip": [
"10.0.0.2",
"11.0.0.2"
],
"emailsender": [
{
"name": "emailsender.testing-kl92-s0.marathon.mesos.",
"port": "31201",
"ip": ["10.10.10.1"]
"tcp": {
"rpc":31210,
"web":31211,
"sip":32010
},
{
"name": "emailsender.testing-kl123-s1.marathon.mesos.",
"port": "32232",
"ip": ["10.10.10.1"]
"udp": {
"sip":31211
}
}
],
"service-with-defined-ports": {
"web": [
"email": [
{
"name": "f.q.d.n",
"port": 12341
"name": "nginx.testing-kl92-s0.marathon.mesos.",
"ip": [
"10.0.0.1"
],
"tcp": {
"smtp":31200,
"pop":31201
}
}
],
"rpc": [
"anyport": [
{
"name": "f.q.d.n",
"port": 12342
}
"name": "nginx.testing-kl92-s0.marathon.mesos.",
"ip": [
"10.0.0.1"
],
"tcp": [
31200,
31201
]
}
},
]
"env": {
"HOME": "/var/lib/nginx"
}
}
```
## Real template example
#### Real template example
nginx config
```
@ -104,7 +126,7 @@ server {
}
```
## Checks in template
#### Checks in template
_my['env']_ is a python os.environ class. Look bellow:
```
@ -114,3 +136,90 @@ host = '{{my['env']['DB_HOST']}}'
host = 'localhost'
{% endif %}
```
### 0.7 version
my dictionary in template
```
{
"services": {
"nginx": [
{
"name": "nginx.testing-kl92-s0.marathon.mesos.",
"port": "31200"
},
{
"name": "nginx.testing-kl123-s1.marathon.mesos.",
"port": "32230"
}
],
"emailsender": [
{
"name": "emailsender.testing-kl92-s0.marathon.mesos.",
"port": "31201"
},
{
"name": "emailsender.testing-kl123-s1.marathon.mesos.",
"port": "32232"
}
],
"service-with-defined-ports": {
"name-of-port0": [
{
"name": "f.q.d.n",
"port": 12341
}
],
"name-of-port2": [
{
"name": "f.q.d.n",
"port": 12342
}
]
}
},
"env": {
"HOME": "/var/lib/nginx"
}
}
```
#### Real template example
```
upstream matrix-http {
hash $remote_addr;
{% for server in my['services']['matrix']['http'] %}
server {{server['name']}}:{{server['port']}} max_fails=3;
{% endfor %}
}
upstream riot-http {
hash $remote_addr;
{% for server in my['services']['riot'] %}
server {{server['name']}}:{{server['port']}} max_fails=3;
{% endfor %}
}
server {
listen 10.15.56.157:80;
server_name matrix.example.com;
client_max_body_size 10m;
location / {
proxy_pass http://riot-http;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header Host $http_host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
location /_matrix/ {
proxy_pass http://matrix-http;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header Host $http_host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
}
```

4
docs/ru/app.configuration.md
View File

@ -10,6 +10,8 @@
"ports": ["web", "socket"]
}
],
"discovery": "mesos_dns",
"group": "dev.web",
"conf_name": "kiosk",
"template": "/etc/surok/templates/kiosk.jj2",
"dest": "/etc/nginx/sites-available/kioskservice.conf",
@ -31,3 +33,5 @@
В reload_cmd можно использовать переменные окружения:
```"reload_cmd": "/usr/bin/killall -9 calc || true && /usr/local/bin/calc -c /app/calc.conf ${CALC_NUM}"```
* run_cmd(v0.6) - array. Список с командой на выполнение. Используется внутри контейнера вместо reload_cmd.
* discovery - Опциональный. переопределяет метод обнаружения сервисов для приложения
* group - Опциональный. Группа по-умолчанию.

89
docs/ru/surok.configuration.md
View File

@ -1,30 +1,81 @@
# Конфигурация Surok (0.7.x)
**/etc/surok/conf/surok.json**
Разберем конфигурационный файл по опциям
# Конфигурация Surok (0.8.x)
**/etc/surok/conf/surok.json** Разберем конфигурационный файл по опциям
```
{
"version": "0.8"
"marathon": {
"enabled": false,
"restart": false,
"force": true,
"host": "marathon.mesos:8080",
"enabled": true
"host": "http://marathon.mesos:8080"
},
"mesos":{
"enabled": true,
"domain": "marathon.mesos"
},
"default_discovery": "mesos_dns",
"confd": "/etc/surok/conf.d",
"domain": "marathon.mesos",
"wait_time": 20,
"lock_dir": "/var/tmp",
"loglevel": "info|debug"
"container": true|false
"loglevel": "info",
"memcached": {
"enabled": false,
"discovery": {
"enabled": false,
"service": "memcached",
"group": "system"
},
"hosts": ["localhost:11211"]
}
}
```
## Опции файла конфигурации
* **version** - *string. Не обязательный. По умолчанию "0.7".*
Версия файлов конфигурации, шаблонов. На текущий момент может принимать значения "0.7" или "0.8".
* значение "0.7" - файлы конфигурации версии 0.7.х и более ранних
* значение "0.8" - файлы конфигурации версии 0.8
##### версия 0.8
* **marathon**, **mesos**, **consul**, **memcached** - *dict/hash. Не обязательный. По умолчанию '{"enable":false}'.*
Системы с которыми работает сурок. Если система выключена, то параметры системы и их наличие уже не важны.
* **enable** - *boolean. Не обязательный. По умолчанию false.*
Доступность системы для использования.
специфичные параметры:
* для Marathon API "marathon"
* **force** - *boolean. Не обязательный. По умолчанию true.*
Рестарт контейнера с force или нет.
* **restart** - *boolean. Не обязательный. По умолчанию false.*
Вкл/выкл. рестарта контейнера
* **host** - *string. Не обязательный. По умолчанию "http://marathon.mesos:8080".*
Адрес Marathon.
* для mesos DNS "mesos"
* **domain** - *string. Не обязательный. По умолчанию "marathon.mesos".*
Приватный домен Mesos DNS
* для Memcached "memcached"
* **hosts** -
* **discovery** -
* **enabled** -
* **service** -
* **group** -
* **default_discovery** - *string. Не обязательный. По умолчанию "mesos_dns".*
Может принимать значения:
* "mesos_dns" - Mesos DNS
* "marathon_api"- Marathon API
* **confd** - *strig. Обязательный.*
Абсолютный путь до директории с конфигурационными файлами приложений.
* **wait_time** - *int. Обязательный.*
Время в секундах сколько Surok ждет до того, как начать заново делать запросы на обнаружение сервисов.
* **lock_dir** - *string. Обязательный.*
Абсолютный путь до директории с lock-конфигурациями.
* **loglevel** - *string. Не обязательный. По умолчанию "info".*
Уровень логирования. Может принимать значения: "debug", "info", "warning", "error"
##### версия 0.7 и более ранние
Особенности для файла конфигурации
* **marathon**
* **enabled** - boolean. Вкл/выкл. рестарта контейнера. В версии 0.8 переименована в "restart".
* **domain** - string. Приватный домен Mesos DNS. В версии 0.8 перемещен в dict "mesos".
Обнаружение Mesos DNS включено всегда.
* marathon(v0.7) - hash. В текущей версии отвечает за перезапуск контейнера. Обнаружение сервисов через Marathon пока недоступно.
1. force - boolean. Рестарт контейнера с force или нет.
2. host - string. Адрес Marathon.
3. enabled - boolean. Вкл/выкл.
* confd - strig. Абсолютный путь до директории с конфигурационными файлами приложений.
* domain - string. Домен, который обслуживает mesos-dns.
* wait_time - int. Время в секундах сколько Surok ждет до того, как начать заново делать запросы на обнаружение сервисов.
* lock_dir - string. Абсолютный путь до директории с lock-конфигурациями.
* loglevel - string. Уровень логирования.
* container(v0.6) - boolean. Определяем внутри или нет контейнера запущен сурок. Меняется логика работы.

125
docs/ru/templates.md
View File

@ -4,6 +4,127 @@
## Словарь my в шаблоне
### Версия 0.8
Surok заполняет словарь my и передает его в шаблон.
```
{
"services": {
"asterisk": [
{
"name": "nginx.testing-kl92-s0.marathon.mesos.",
"ip": [
"10.0.0.1",
"11.0.0.1"
],
"tcp": {
"rpc":31200,
"web":31201,
"sip":32000
},
"udp": {
"sip":31201
}
},
{
"name": "nginx.testing-kl123-s1.marathon.mesos.",
"ip": [
"10.0.0.2",
"11.0.0.2"
],
"tcp": {
"rpc":31210,
"web":31211,
"sip":32010
},
"udp": {
"sip":31211
}
}
],
"email": [
{
"name": "nginx.testing-kl92-s0.marathon.mesos.",
"ip": [
"10.0.0.1"
],
"tcp": {
"smtp":31200,
"pop":31201
}
}
],
"anyport": [
{
"name": "nginx.testing-kl92-s0.marathon.mesos.",
"ip": [
"10.0.0.1"
],
"tcp": [
31200,
31201
]
}
]
"env": {
"HOME": "/var/lib/nginx"
}
}
```
## Пример реального шаблона
```
upstream matrix-http {
hash $remote_addr;
{% for server in my['services']['matrix'] %}
server {{server['name']}}:{{server['tcp']['http']}} max_fails=3;
{% endfor %}
}
upstream riot-http {
hash $remote_addr;
{% for server in my['services']['riot'] %}
server {{server['name']}}:{{server['tcp'][0]}} max_fails=3;
{% endfor %}
}
server {
listen 10.15.56.157:80;
server_name matrix.example.com;
client_max_body_size 10m;
location / {
proxy_pass http://riot-http;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header Host $http_host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
location /_matrix/ {
proxy_pass http://matrix-http;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header Host $http_host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
}
```
Так для upstream matrix-http используются именованные порты, а для riot-http – нет.
## Проверки в шаблоне
Переменная _my['env']_ является классом python _os.environ_, что позваоляет нам строить различные проверки, например:
```
{% if my['env'].get('DB_HOST') -%}
host = '{{my['env']['DB_HOST']}}'
{% else -%}
host = 'localhost'
{% endif -%}
```
### Версия 0.7
Surok заполняет словарь my и передает его в шаблон.
```
{
@ -29,13 +150,13 @@ Surok заполняет словарь my и передает его в шаб
}
],
"service-with-defined-ports": {
"web": [
"name-of-port0": [
{
"name": "f.q.d.n",
"port": 12341
}
],
"rpc": [
"name-of-port2": [
{
"name": "f.q.d.n",
"port": 12342

67
surok.py
View File

@ -7,71 +7,42 @@ from os.path import isfile, join
import json
import argparse
from surok.templates import gen
from surok.discovery import resolve
from surok.discovery import Discovery
from surok.system import reload_conf
from surok.logger import Logger
from surok.config import Config
# Load base configurations
surok_conf = '/etc/surok/conf/surok.json'
logger = Logger()
# Command line arguments
parser = argparse.ArgumentParser()
parser.add_argument('-c', '--config', help='surok.json path')
args = parser.parse_args()
if args.config:
surok_conf = args.config
# Read config file
f = open(surok_conf, 'r')
conf = json.loads(f.read())
f.close()
# Get app configurations
# Return list of patches to app discovery configuration
def get_configs():
confs = [f for f in listdir(conf['confd']) if isfile(
join(conf['confd'], f))]
return sorted(confs)
# Get Surok App configuration
# Read app conf from file and return dict
def load_app_conf(app):
# Load OS environment to app_conf
f = open(conf['confd'] + '/' + app)
c = json.loads(f.read())
f.close()
c['env'] = os.environ
return c
# Load base configurations
config = Config(args.config if args.config else '/etc/surok/conf/surok.json')
# Main loop
###########
#
discovery = Discovery()
while 1:
confs = get_configs()
for app in confs:
app_conf = load_app_conf(app)
# Update config from discovery object
discovery.update_data()
for app in config.apps:
# Will be removed later
# For old configs
loglevel = 'info'
if 'loglevel' in conf:
loglevel = conf['loglevel']
# Resolve services
app_hosts = resolve(app_conf, conf)
app_hosts = discovery.resolve(app)
# Populate my dictionary
my = {"services": app_hosts,
"conf_name": app_conf['conf_name']}
"conf_name": app['conf_name']}
logger.debug('my=', my)
# Generate config from template
service_conf = gen(my, app_conf['template'])
service_conf = gen(my, app['template'])
reload_conf(service_conf, app_conf, conf, app_hosts)
reload_conf(service_conf, app, config, app_hosts)
sleep(conf['wait_time'])
sleep(config['wait_time'])

15
surok.spec
View File

@ -1,6 +1,6 @@
Summary: Simple service discovery for Apache Mesos clusters
Name: surok
Version: 0.7.4.3
Version: 0.8
Release: 1
License: BSD
Group: admin
@ -25,10 +25,12 @@ install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok/logger.py %{buildr
install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok/system.py %{buildroot}/opt/surok/surok
install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok/discovery.py %{buildroot}/opt/surok/surok
install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok/templates.py %{buildroot}/opt/surok/surok
install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok/config.py %{buildroot}/opt/surok/surok
install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok.py %{buildroot}/opt/surok
mkdir -p %{buildroot}/etc/surok/{conf,conf.d,templates}
install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/conf/surok.json %{buildroot}/etc/surok/conf
install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/conf.d/selfcheck.json %{buildroot}/etc/surok/conf.d
install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/conf.d/self_check.json %{buildroot}/etc/surok/conf.d
install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/conf.d/marathon_check.json %{buildroot}/etc/surok/conf.d
install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/templates/selfcheck.jj2 %{buildroot}/etc/surok/templates
install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/requriments.txt %{buildroot}/opt/surok
@ -59,7 +61,11 @@ cd /opt/surok && pip3 install -r requriments.txt
/opt/surok/surok/templates.py
/opt/surok/surok/templates.pyc
/opt/surok/surok/templates.pyo
/etc/surok/conf.d/selfcheck.json
/opt/surok/surok/config.py
/opt/surok/surok/config.pyc
/opt/surok/surok/config.pyo
/etc/surok/conf.d/self_check.json
/etc/surok/conf.d/marathon_check.json
/etc/surok/templates/selfcheck.jj2
/opt/surok/requriments.txt
%defattr(-,root,root,-)
@ -67,6 +73,9 @@ cd /opt/surok && pip3 install -r requriments.txt
%changelog
* Tue Feb 7 2017 Denis Zheleztsov <difrex.punk@gmail.com>
- New major release
- Discovery over marathon api
* Mon Nov 14 2016 Denis Zheleztsov <difrex.punk@gmail.com> -
- Initial build.

420
surok/config.py Normal file
View File

@ -0,0 +1,420 @@
# Public names
__all__ = ['Config', 'AppConfig']
import hashlib
import importlib
import json
import os
from .logger import *
# Logger link
# Config singleton link
_config_singleton = None
'''
Test values
==================================================
key - key
value - value of key
type_value - type of value
type_par - additional parameters for test
'''
'''
Public Config object
==================================================
.set_config(conf_data) - set config data
Use: conf_data(str type) - path of json config file
conf_data(dict type) - dict with config
.set(key,value) - set config key
.get(key) - get config key
.update_apps() - update apps config data
.apps - Apps object. List of AppConfig oblects
'''
class _ConfigTemplate(dict):
_conf = {}
def _init_conf(self, params):
conf = {}
for k in params.keys():
if params[k].get('params'):
conf[k] = self._init_conf(params[k].get('params'))
else:
if params[k].get('value') is not None:
conf[k] = params[k].get('value')
return conf
def __init__(self, *conf_data):
self._logger = Logger()
if not self._conf:
self._conf = self._init_conf(self._params)
for c in conf_data:
self.set_config(c)
def _set_conf_params(self, oldconf, testconf, params):
conf = oldconf if oldconf else {}
for key in testconf.keys():
resvalue = None
param = params.get(key)
oldvalue = conf.get(key)
testvalue = testconf.get(key)
if param is None:
self._logger.error('Parameter "', key, '" value "', testvalue,
'" type is "', type(testvalue).__name__, '" not found')
else:
type_param = param.get('type')
resvalue = []
if type(testvalue).__name__ != 'list':
testvalue = [testvalue]
for testitem in testvalue:
if self._test_value(key, testitem, param):
if 'dict' in type_param:
if param.get('params'):
res = self._set_conf_params(
oldvalue, testitem, param.get('params'))
if res is not None:
resvalue.append(res)
else:
resvalue.append(testitem)
if 'list' not in type_param:
resvalue = list([None] + resvalue).pop()
if resvalue is not None and 'do' in type_param:
if not self._do_type_set(key, resvalue, param):
self._logger.warning(
'Parameter "', key, '" current "', resvalue, '" type is "', type(resvalue).__name__, '" testing failed')
resvalue = None
if resvalue is not None:
conf[key] = resvalue
return conf
def _test_value(self, key, value, param):
type_param = param.get('type')
type_value = [
x for x in type_param if x in ['str', 'int', 'bool', 'dict']]
if type_value:
if type(value).__name__ not in type_value:
self._logger.error(
'Parameter "', key, '" must be ', type_value,
' types, current "', value, '" (', type(value).__name__, ')')
return False
if 'value' in type_param:
if value not in param.get('values', []):
self._logger.error(
'Value "', value, '" of key "', key, '" unknown')
return False
if 'dir' in type_param:
if not os.path.isdir(value):
self._logger.error('Path "{}" not present'.format(value))
return False
elif 'file' in type_param:
if not os.path.isfile(value):
self._logger.error('File "{}" not present'.format(value))
return False
return True
else:
self._logger.error(
'Type for testing "{}" unknown'.format(type_value))
return False
def set_config(self, conf_data):
conf = {}
if type(conf_data).__name__ == 'str':
try:
self._logger.debug('Open file ', conf_data)
f = open(conf_data, 'r')
json_data = f.read()
f.close()
conf = json.loads(json_data)
except OSError as err:
self._logger.error("OS error: {0}".format(err))
pass
except ValueError as err:
self._logger.error('JSON format error: {0}'.format(err))
pass
except:
self._logger.error('Load main config file failed')
pass
elif type(conf).__name__ == 'dict':
conf = conf_data
else:
return False
self._conf = self._set_conf_params(self._conf, conf, self._params)
self._logger.debug('Conf=', self._conf)
def keys(self):
return self._conf.keys()
def dump(self):
return json.dumps(self._conf, sort_keys=True, indent=2)
def _do_type_set(self, key, value, params):
self._logger.error('_do_type_set handler is not defined')
return False
def hash(self):
return hashlib.sha1(json.dumps(self._conf, sort_keys=True).encode()).hexdigest()
def set(self, key, value):
self._conf[key] = value
def __setitem__(self, key, value):
self.set(key, value)
def get(self, key, default=None):
return self._conf.get(key, default)
def __getitem__(self, key):
return self.get(key)
def __contains__(self, item):
return bool(item in self._conf)
def __len__(self):
return self._conf.__len__()
def __str__(self):
return self._conf.__str__()
def __repr__(self):
return self._conf.__repr__()
class Config(_ConfigTemplate):
_params = {
'marathon': {
'params': {
'force': {
'value': True,
'type': ['bool']
},
'host': {
'value': 'http://marathon.mesos:8080',
'type': ['str']
},
'enabled': {
'value': False,
'type': ['bool']
},
'restart': {
'value': False,
'type': ['bool']
}
},
'type': ['dict']
},
'mesos': {
'params': {
"domain": {
'value': "marathon.mesos",
'type': ['str']
},
"enabled": {
'value': False,
'type': ['bool']
}
},
'type': ['dict']
},
'memcached': {
'params': {
'enabled': {
'value': False,
'type': ['bool']
},
'discovery': {
'params': {
'enabled': {
'value': False,
'type': ['bool']
},
'service': {
'value': 'memcached',
'type': ['str']
},
'group': {
'value': 'system',
'type': ['str']
}
},
'type': ['dict']
},
'hosts': {
'value': ['localhost:11211'],
'type': ['list', 'str']
}
},
'type': ['dict']
},
'version': {
'value': '0.7',
'type': ['str', 'value'],
'values': ['0.7', '0.8']
},
'confd': {
'value': '/etc/surok/conf.d',
'type': ['str', 'dir']
},
'wait_time': {
'value': 20,
'type': ['int']
},
'lock_dir': {
'value': '/var/tmp',
'type': ['str', 'dir']
},
'default_discovery': {
'value': 'mesos_dns',
'type': ['str', 'value'],
'values': ['mesos_dns', 'marathon_api']
},
'loglevel': {
'value': 'info',
'type': ['str', 'do'],
'do': 'set_loglevel'
}
}
def __new__(cls, *args):
global _config_singleton
if _config_singleton is None:
_config_singleton = super(Config, cls).__new__(cls)
return _config_singleton
def __init__(self, *conf_data):
super().__init__(*conf_data)
self.apps = _Apps()
def set_config(self, conf_data):
super().set_config(conf_data)
if self.get('version') == '0.7':
domain = self.get('domain')
if domain is not None and self.get('mesos') is None:
self.set('mesos', {'domain': domain, 'enabled': True})
def _do_type_set(self, key, value, param):
if param.get('do') == 'set_loglevel':
if self._logger.set_level(value):
return True
return False
def update_apps(self):
self.apps.reset()
for app in sorted([os.path.join(self.get('confd'), f) for f in os.listdir(self.get('confd')) if os.path.isfile(os.path.join(self.get('confd'), f))]):
self.apps.set(AppConfig(app))
'''
Private Apps object
==================================================
.get(app) - get _AppConfig object
.set(app) - set _AppConfig object
'''
class _Apps:
_apps = {}
_items = []
def get(self, key):
return self._apps.get(key)
def set(self, app):
self._apps[app.get('conf_name')] = app
def reset(self):
keys = [] + list(self.keys())
for k in keys:
del self._apps[k]
def __getitem__(self, key):
return self.get(key)
def __contains__(self, item):
return bool(item in self._apps)
def __iter__(self):
self._items = sorted(self.keys())
return self
def __next__(self):
if self._items:
return self.get(self._items.pop(0))
raise StopIteration
def keys(self):
return self._apps.keys()
'''
Public AppConfig object
==================================================
.set_config(conf_data) - set config data
Use: conf_data(str type) - path of json config file
conf_data(dict type) - dict with config
.set(key,value) - set config key
.get(key) - get config key
'''
class AppConfig(_ConfigTemplate):
_params = {
'conf_name': {
'type': ['str']
},
'services': {
'params': {
'name': {
'type': ['str']
},
'ports': {
'type': ['list', 'str']
},
'discovery': {
'type': ['str']
},
'group': {
'type': ['str']
}
},
'type': ['list', 'dict']
},
'template': {
'type': ['str', 'file']
},
'dest': {
'type': ['str']
},
'reload_cmd': {
'type': ['str']
},
'discovery': {
'type': ['str']
},
'group': {
'type': ['str']
}
}
def __init__(self, *conf_data):
self._config = Config()
super().__init__(*conf_data)
def set_config(self, conf_data):
super().set_config(conf_data)
self._conf.setdefault(
'discovery', self._config.get('default_discovery'))
self._conf.setdefault('group', self._get_default_group())
if type(conf_data).__name__ == 'str':
self._conf.setdefault('conf_name', os.path.basename(conf_data))
def _get_default_group(self):
env = self._config.get('env', dict(os.environ))
# Check environment variable
if env.get('SUROK_DISCOVERY_GROUP'):
return env['SUROK_DISCOVERY_GROUP']
# Check marathon environment variable
elif env.get('MARATHON_APP_ID'):
return ".".join(env['MARATHON_APP_ID'].split('/')[-2:0:-1])

269
surok/discovery.py
View File

@ -1,51 +1,47 @@
# Public names
__all__ = ['Discovery', 'DiscoveryMesos', 'DiscoveryMarathon']
import dns.resolver
import dns.query
from dns.exception import DNSException
from .logger import info, warning, error, debug
import os
import sys
import requests
from dns.exception import DNSException
from .config import *
from .logger import *
# Discovery object
_discovery_singleton = None
# Resolve service from mesos-dns SRV record
# return dict {"servicename": [{"name": "service.f.q.d.n.", "port": 9999}]}
def resolve(app, conf):
hosts = {}
services = app['services']
domain = conf['domain']
class DiscoveryTemplate:
for service in services:
hosts[service['name']] = {}
def __init__(self):
self._config = Config()
self._logger = Logger()
group = get_group(service, app)
if group is False:
error('Group is not defined in config, SUROK_DISCOVERY_GROUP and MARATHON_APP_ID')
error('Not in Mesos launch?')
sys.exit(2)
def enabled(self):
return self._config[self._config_section].get('enabled', False)
# Port name from app config
ports = None
try:
ports = service['ports']
except:
def update_data(self):
pass
# This is fast fix for port naming
# Will be rewrite later
fqdn = ''
if ports is not None:
for port_name in ports:
fqdn = '_' + port_name + '.' + '_' + service['name'] + '.' + group + '._tcp.' + domain
hosts[service['name']][port_name] = do_query(fqdn, conf['loglevel'])
else:
fqdn = '_' + service['name'] + '.' + group + '._tcp.' + domain
hosts[service['name']] = do_query(fqdn, conf['loglevel'])
return hosts
# Do DNS queries
# Return array:
# ["10.10.10.1", "10.10.10.2"]
def do_query_a(self, fqdn):
servers = []
try:
resolver = dns.resolver.Resolver()
for a_rdata in resolver.query(fqdn, 'A'):
servers.append(a_rdata.address)
except DNSException as e:
self._logger.error("Could not resolve ", fqdn)
return servers
# Do DNS queries
# Return array:
# [{"name": "f.q.d.n", "port": 8876, "ip": ["10.10.10.1", "10.10.10.2"]}]
def do_query(fqdn, loglevel):
def do_query_srv(self, fqdn):
servers = []
try:
resolver = dns.resolver.Resolver()
@ -54,42 +50,183 @@ def do_query(fqdn, loglevel):
query = resolver.query(fqdn, 'SRV')
for rdata in query:
info = str(rdata).split()
name = info[3][:-1]
port = info[2]
server = {'name': name, 'port': port, 'ip': []}
a_query = resolver.query(name, 'A')
for a_rdata in a_query:
server['ip'].append(a_rdata.address)
servers.append(server)
servers.append({'name': info[3][:-1], 'port': info[2]})
except DNSException as e:
if loglevel != 'info':
error("Could not resolve " + fqdn)
self._logger.warning("Could not resolve ", fqdn)
return servers
# Groups switch
# Priority: config, environment, marathon environment
def get_group(service, app):
# Check group in app conf
if 'group' in service:
return service['group']
# Check environment variable
elif app['env'].get('SUROK_DISCOVERY_GROUP'):
return app['env']['SUROK_DISCOVERY_GROUP']
# Check marathon environment variable
elif app['env'].get('MARATHON_APP_ID'):
group = parse_marathon_app_id(app['env']['MARATHON_APP_ID'])
return group
class Discovery:
_discoveries = {}
def __new__(cls):
global _discovery_singleton
if _discovery_singleton is None:
_discovery_singleton = super(Discovery, cls).__new__(cls)
return _discovery_singleton
def __init__(self):
self._config = Config()
self._logger = Logger()
if not self._discoveries.get('mesos_dns'):
self._discoveries['mesos_dns'] = DiscoveryMesos()
if not self._discoveries.get('marathon_api'):
self._discoveries['marathon_api'] = DiscoveryMarathon()
def keys(self):
return self._discoveries.keys()
def resolve(self, app):
discovery = app.get('discovery', self._config.get('default_discovery'))
if discovery not in self.keys():
self._logger.warning('Discovery "', discovery, '" is not present')
return {}
if self._discoveries[discovery].enabled():
return self.compatible(self._discoveries[discovery].resolve(app))
else:
return False
self._logger.error('Discovery "', discovery, '" is disabled')
return {}
def update_data(self):
self._config.update_apps()
for d in self.keys():
if self._discoveries[d].enabled():
self._discoveries[d].update_data()
def compatible(self, hosts):
compatible_hosts = {}
if self._config.get('version') == '0.7':
for service in hosts.keys():
for host in hosts[service]:
ports = host.get('tcp', [])
if type(ports).__name__ == 'list':
compatible_hosts[service] = []
for port in ports:
compatible_hosts[service].append(
{'name': host['name'],
'ip': host['ip'],
'port': str(port)})
else:
compatible_hosts[service] = {}
for port in ports.keys():
compatible_host = compatible_hosts[
service].setdefault(port, [])
compatible_host.append({'name': host['name'],
'ip': host['ip'],
'port': ports[port]})
return compatible_hosts
return hosts
# Parse MARATHON_APP_ID
# Return marathon.group
def parse_marathon_app_id(marathon_app_id):
marathon_app_id = marathon_app_id.split('/')
del(marathon_app_id[-1])
marathon_app_id.reverse()
group = ".".join(marathon_app_id)[:-1]
return(group)
class DiscoveryMesos(DiscoveryTemplate):
_config_section = 'mesos'
def resolve(self, app):
hosts = {}
services = app.get('services')
domain = self._config[self._config_section].get('domain')
for service in services:
group = service.get('group', app.get('group'))
if group is None:
self._logger.error('Group for service "{}" of config "{}" not found'.format(
service['name'], app.get('conf_name')))
continue
ports = service.get('ports')
name = service['name']
hosts[name] = {}
serv = hosts[name]
self._logger.debug(
'group=', group, ' ports=', ports, ' name=', name, ' serv=', serv)
for prot in ['tcp', 'udp']:
if ports is not None:
for port_name in ports:
for d in self.do_query_srv('_' + port_name + '._' + name + '.' + group + '._' + prot + '.' + domain):
hostname = d['name']
serv.setdefault(hostname, {'name': hostname,
'ip': self.do_query_a(hostname)})
serv[hostname].setdefault(prot, {})
serv[hostname][prot][port_name] = d['port']
else:
for d in self.do_query_srv('_' + name + '.' + group + '._' + prot + '.' + domain):
hostname = d['name']
if serv.get(hostname) is None:
serv[hostname] = {'name': hostname,
'ip': self.do_query_a(hostname)}
if serv[hostname].get(prot) is None:
serv[hostname][prot] = []
serv[hostname][prot].extend([d['port']])
hosts[name] = list(serv.values())
return hosts
class DiscoveryMarathon(DiscoveryTemplate):
_config_section = 'marathon'
_tasks = []
_ports = {}
def update_data(self):
hostname = self._config[self._config_section].get('host')
try:
ports = {}
for app in requests.get(hostname + '/v2/apps').json()['apps']:
ports[app['id']] = {}
if app.get('container') is not None and app['container'].get('type') == 'DOCKER':
ports[app['id']] = app['container'][
'docker'].get('portMappings', [])
self._ports = ports
except:
self._logger.warning(
'Apps (', hostname, '/v2/apps) request from Marathon API is failed')
pass
try:
self._tasks = requests.get(hostname + '/v2/tasks').json()['tasks']
except:
self._logger.warning(
'Tasks (', hostname, '/v2/tasks) request from Marathon API is failed')
pass
def _test_mask(self, mask, value):
return (mask.endswith('*') and value.startswith(mask[:-1])) or mask == value
def resolve(self, app):
hosts = {}
services = app.get('services')
if not services:
services = [{'name': '*', 'ports': ['*']}]
for service in services:
# Convert xxx.yyy.zzz to /zzz/yyy/xxx/ format
group = service.get('group', app.get('group'))
if group is None:
self._logger.error('Group for service "{}" of config "{}" not found'.format(
service['name'], app.get('conf_name')))
continue
group = '/' + '/'.join(group.split('.')[::-1]) + '/'
service_mask = group + service['name']
for task in self._tasks:
if self._test_mask(service_mask, task['appId']):
name = '.'.join(
task['appId'][len(group):].split('/')[::-1])
hosts[name] = {}
serv = hosts[name]
hostname = task['host']
for task_port in self._ports[task['appId']]:
prot = task_port['protocol']
port_name = task_port['name']
port = task['ports'][
task['servicePorts'].index(task_port['servicePort'])]
if 'ports' in service:
for port_mask in service['ports']:
if self._test_mask(port_mask, port_name):
serv.setdefault(
hostname, {'name': hostname,
'ip': self.do_query_a(hostname)})
serv[hostname].setdefault(prot, {})
serv[hostname][prot][port_name] = port
else:
serv.setdefault(hostname, {'name': hostname,
'ip': self.do_query_a(hostname)})
serv[hostname].setdefault(prot, [])
serv[hostname][prot].extend([port])
hosts[name] = list(serv.values())
return hosts

94
surok/logger.py
View File

@ -1,36 +1,86 @@
# Public names
__all__ = ['Logger']
import sys
from time import time
import json
import time
# Logger singleton link
_logger_singleton = None
'''
Public Logger oblect
==================================================
.set_level(level) - set level messages
level - values 'debug', 'info', 'warning', 'error'
* error - write error message
* warning - write warning and error message
* info - write info, warning and error message
* debug - write all message
.get_level() - get level messages
.error(str) - write error message
.warning(str) - write warning message
.info(str) - write info message
.debug(str)- write error message
'''
def make_message(message):
cur_time = str(time())
m = '[' + cur_time + '] ' + message['level'] + ': ' + message['raw'] + "\n"
return m
class Logger:
_loglevel = 'info'
_msg_level = {
'debug': 'DEBUG',
'info': 'INFO',
'warning': 'WARNING',
'error': 'ERROR'
}
def __new__(cls, *args):
global _logger_singleton
if _logger_singleton is None:
_logger_singleton = super(Logger, cls).__new__(cls)
return _logger_singleton
def info(message):
req = {'level': 'INFO', 'raw': message}
m = make_message(req)
def __init__(self, *args):
if args:
self.set_level(args[0])
sys.stdout.write(m)
def set_level(self, level):
if level in ['debug', 'info', 'warning', 'error']:
self._loglevel = level
return True
else:
self.warning('Log level "', level, '" not valid')
return False
def get_level(self):
return self._loglevel
def warning(message):
req = {'level': 'WARNING', 'raw': message}
m = make_message(req)
def _make_message(self, level, message):
r = []
for m in message:
if type(m).__name__ == 'str':
r.append(m)
else:
r.append(json.dumps(m, sort_keys=True, indent=2))
return '[' + str(time.time()) + '] ' + self._msg_level[level] + ': ' + ''.join(r) + "\n"
sys.stderr.write(m)
def debug(self, *message):
if self.get_level() in ['debug']:
self._log2err(self._make_message('debug', message))
def info(self, *message):
if self.get_level() in ['debug', 'info']:
self._log2out(self._make_message('info', message))
def error(message):
req = {'level': 'ERROR', 'raw': message}
m = make_message(req)
def warning(self, *message):
if self.get_level() in ['debug', 'info', 'warning']:
self._log2out(self._make_message('warning', message))
sys.stderr.write(m)
def error(self, *message):
self._log2err(self._make_message('error', message))
def _log2err(self, out):
sys.stderr.write(out)
def debug(message):
req = {'level': 'DEBUG', 'raw': message}
m = make_message(req)
sys.stderr.write(m)
def _log2out(self, out):
sys.stdout.write(out)

45
surok/system.py
View File

@ -1,20 +1,20 @@
import os
import sys
import requests
from .discovery import resolve
from .logger import info, warning, error, debug
from .discovery import Discovery
from .logger import Logger
logger=Logger()
# Get old configuration
def get_old(name, service_conf):
try:
path = '/var/tmp/surok.' + name
f = open(path, 'r')
old = f.read()
f.close()
except Exception as e:
print(str(e))
logger.error(str(e))
return 0
if old == service_conf:
@ -53,7 +53,7 @@ def write_lock(name, service_conf):
def do_reload(service_conf, app_conf):
warning('Write new configuration of ' + app_conf['conf_name'])
logger.warning('Write new configuration of ' + app_conf['conf_name'])
f = open(app_conf['dest'], 'w')
f.write(service_conf)
@ -68,6 +68,7 @@ def do_reload(service_conf, app_conf):
# Discovery memcached servers
def discovery_memcached(conf):
discovery=Discovery()
memcache = conf['memcached']
app_conf = {
"services": [
@ -78,7 +79,7 @@ def discovery_memcached(conf):
]
}
hosts = resolve(app_conf, conf)
hosts = discovery.resolve(app_conf)
mc_servers = []
for server in hosts[memcache['discovery']['service']]:
@ -91,7 +92,8 @@ def discovery_memcached(conf):
# !!! NEED REFACTORING !!!
def reload_conf(service_conf, app_conf, conf, app_hosts):
# Check marathon enabled in configuration
if conf['marathon']['enabled'] is True:
if (conf.get('version','0.7')=='0.8' and conf['marathon'].get('restart',False)) or (
conf.get('version','0.7')=='0.7' and conf['marathon'].get('enable',False)):
if get_old(app_conf['conf_name'], service_conf) != 1:
restart_self_in_marathon(conf['marathon'])
@ -105,47 +107,38 @@ def reload_conf(service_conf, app_conf, conf, app_hosts):
mc_hosts = None
if conf['memcached']['discovery']['enabled'] is True:
mc_hosts = discovery_memcached(conf)
info('Discovered memcached hosts: ' + str(mc_hosts))
logger.info('Discovered memcached hosts: ' + str(mc_hosts))
else:
mc_hosts = conf['memcached']['hosts']
try:
mc = memcache.Client(mc_hosts)
if get_old_from_memcache(mc, app_conf['conf_name'], app_hosts) != 1:
stdout = do_reload(service_conf, app_conf)
info(stdout)
logger.info(stdout)
return True
except Exception as e:
error('Cannot connect to memcached: ' + str(e))
logger.error('Cannot connect to memcached: ' + str(e))
else:
warning('DEPRECATED main conf file. Please use new syntax!')
logger.warning('DEPRECATED main conf file. Please use new syntax!')
# End of memcache block
#######################
if get_old(app_conf['conf_name'], service_conf) != 1:
stdout = do_reload(service_conf, app_conf)
info(stdout)
logger.info(stdout)
return True
else:
if conf['loglevel'] == 'debug':
debug('Same config ' + app_conf['conf_name'] + ' Skip reload')
logger.debug('Same config ' + app_conf['conf_name'] + ' Skip reload')
return False
# Do POST request to marathon API
# /v2/apps//app/name/restart
def restart_self_in_marathon(marathon):
host = marathon['host']
# Check MARATHON_APP_ID environment varible
if os.environ.get('MARATHON_APP_ID') is not True:
error('Cannot find MARATHON_APP_ID. Not in Mesos?')
if not os.environ.get('MARATHON_APP_ID',False):
logger.error('Cannot find MARATHON_APP_ID. Not in Mesos?')
sys.exit(2)
app_id = os.environ['MARATHON_APP_ID']
uri = 'http://' + host + '/v2/apps/' + app_id + '/restart'
# Ok. In this step we made restart request to Marathon
if marathon['force'] is True:
r = requests.post(uri, data = {'force': 'true'})
else:
r = requests.post(uri, data = {'force': 'false'})
r = requests.post('http://'+marathon['host']+'/v2/apps/'+os.environ['MARATHON_APP_ID']+'/restart',
data={'force': marathon.get('force',False)})

13
surok_fedora.spec
View File

@ -1,6 +1,6 @@
Summary: Simple service discovery for Apache Mesos clusters
Name: surok
Version: 0.7.4.3
Version: 0.8
Release: 1.fc24
License: BSD
Group: admin
@ -25,10 +25,12 @@ install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok/logger.py %{buildr
install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok/system.py %{buildroot}/opt/surok/surok
install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok/discovery.py %{buildroot}/opt/surok/surok
install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok/templates.py %{buildroot}/opt/surok/surok
install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok/config.py %{buildroot}/opt/surok/surok
install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok.py %{buildroot}/opt/surok
mkdir -p %{buildroot}/etc/surok/{conf,conf.d,templates}
install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/conf/surok.json %{buildroot}/etc/surok/conf
install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/conf.d/selfcheck.json %{buildroot}/etc/surok/conf.d
install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/conf.d/self_check.json %{buildroot}/etc/surok/conf.d
install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/conf.d/marathon_check.json %{buildroot}/etc/surok/conf.d
install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/templates/selfcheck.jj2 %{buildroot}/etc/surok/templates
@ -43,13 +45,18 @@ rm -rf $RPM_BUILD_ROOT
/opt/surok/surok/discovery.py
/opt/surok/surok/system.py
/opt/surok/surok/templates.py
/etc/surok/conf.d/selfcheck.json
/opt/surok/surok/config.py
/etc/surok/conf.d/self_check.json
/etc/surok/conf.d/marathon_check.json
/etc/surok/templates/selfcheck.jj2
%defattr(-,root,root,-)
%doc
%changelog
* Tue Feb 7 2017 Denis Zheleztsov <difrex.punk@gmail.com>
- New major release
- Discovery over marathon api
* Mon Nov 14 2016 Denis Zheleztsov <difrex.punk@gmail.com> -
- Initial build.