Difrex/surok
Difrex/surok
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #17 from Surkoveds/doc

Browse Source 
Release 0.8
This commit is contained in:
Denis 2017-02-07 16:04:34 +02:00 committed by GitHub
commit 44a10b7fbb
26 changed files with 1668 additions and 380 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
AUTHORS
View File

@ -1,2 +1,3 @@
Denis Zheleztsov <difrex.punk@gmail.com> Denis Zheleztsov <difrex.punk@gmail.com>
Denis Ryabyy <vv1r0x@gmail.com> Denis Ryabyy <vv1r0x@gmail.com>
Evgeniy Vasilev <oren-ibc@yandex.ru>

19
README.ru.md
View File

@ -1,5 +1,5 @@
# Surok # Surok
[![Build Status](https://travis-ci.org/Difrex/surok.svg?branch=master)](https://travis-ci.org/Difrex/surok) [![Build Status](https://travis-ci.org/Surkoveds/surok.svg?branch=master)](https://travis-ci.org/Surkoveds/surok)
Обнаружение сервисов для Apache Mesos. Обнаружение сервисов для Apache Mesos.
@ -17,10 +17,27 @@ cd build
deb-пакет будет лежать в build/out deb-пакет будет лежать в build/out
Сборка базового docker-образа surok Сборка базового docker-образа surok
Ubuntu Xenial
``` ```
cd build cd build
./build.sh surok_image ./build.sh surok_image
``` ```
Alpine image
```
cd build
./build.sh alpine
```
CentOS image
```
cd build
./build.sh centos
```
ENTRYPOINT : ```cd /opt/surok && pytho3 surok.py -c /etc/surok/conf/surok.json```
## Документация
[Wiki](https://github.com/Surkoveds/surok/wiki)
## Известные проблемы ## Известные проблемы

2
build/runtests.sh
View File

@ -5,7 +5,7 @@ set -e
. functions.sh . functions.sh
function run_tests() { function run_tests() {
docker run -ti -v $(pwd)/tests.py:/opt/surok/tests.py \ docker run --rm -ti -v $(pwd)/tests.py:/opt/surok/tests.py \
-v $(pwd)/tests_entrypoint.sh:/tests_entrypoint.sh \ -v $(pwd)/tests_entrypoint.sh:/tests_entrypoint.sh \
--entrypoint /tests_entrypoint.sh \ --entrypoint /tests_entrypoint.sh \
surok_base:latest surok_base:latest

423
build/tests.py Normal file → Executable file
View File

@ -1,71 +1,372 @@
#!/usr/bin/python3
import unittest import unittest
import json import json
import os import os
import re import re
import sys
import surok.config
import surok.logger
import surok.discovery
import hashlib
from surok.config import Config
class Logger(surok.logger.Logger):
_out=''
_err=''
def _log2err(self,out):
self._err+=out
def _log2out(self,out):
self._out+=out
def geterr(self):
return self._err
def getout(self):
return self._out
def reset(self):
self._err=''
self._out=''
class DiscoveryTestingTemplate:
_testing={}
_testing_fqdn_a={
"test.zzz0.test":['10.0.0.1','10.1.0.1'],
"test.zzz1.test":['10.0.1.1','10.1.1.1'],
"test.zzz2.test":['10.0.2.1','10.1.2.1'],
"test.zzz3.test":['10.0.3.1','10.1.3.1']
}
_testing_fqdn_srv={}
def __init__(self):
super().__init__()
self._orig_logger=surok.logger.Logger()
def do_query_a(self,fqdn):
res=self._testing_fqdn_a.get(fqdn,[])
if res:
return res
else:
self._orig_logger.error('Testing FQDN '+fqdn+' not found in test A records')
sys.exit(2)
def do_query_srv(self,fqdn):
res=self._testing_fqdn_srv.get(fqdn,[])
if res or fqdn.startswith('_tname_e.') or fqdn.find('._udp.'):
return res
else:
self._orig_logger.error('Testing FQDN '+fqdn+' not found in test SRV records')
sys.exit(2)
def update_data(self):
class_name=self.__class__.__name__
tgen={
"name": ["zzz0","zzy0","zzy1","zzz1"],
"host": ["test.zzz0.test","test.zzz1.test","test.zzz2.test","test.zzz3.test"],
"serv": ["tname_aa","tname_ab","tname_ba","tname_bb"],
"ports": [12341,12342,12343,12344],
"servicePorts": [21221,21222,21223,21224]
}
if self._testing.get(class_name,True):
if class_name == 'DiscoveryMarathon':
_tasks=[]
_ports={}
for id in (0,1,2,3):
ports=[]+tgen['ports']
servicePorts=[]+tgen['servicePorts']
appId='/'.join(str(tgen['name'][id]+'.xxx.yyy.').split('.')[::-1])
_ports[appId]=[]
for pid in (0,1,2,3):
ports[pid]+=pid*10
servicePorts[pid]+=pid*100
for prot in ['tcp','udp']:
if pid<2 or prot == 'tcp':
_ports[appId].append({'containerPort': 0,
'hostPort': 0,
'labels': {},
'name': tgen['serv'][pid],
'protocol': prot,
'servicePort': servicePorts[pid]})
_tasks.append({'appId':appId,
'host':tgen['host'][id],
'ports':ports,
'servicePorts':servicePorts})
#_tname_a._zzy0.yyy.xxx._tcp.marathon.mesos
self._tasks=_tasks
self._ports=_ports
elif class_name == 'DiscoveryMesos':
for id in (0,1,2,3):
ports=[]+tgen['ports']
for pid in (0,1,2,3):
ports[pid]+=pid*10
for prot in ['tcp','udp']:
if pid<2 or prot == 'tcp':
for fqdn in ['_'+tgen['serv'][pid]+'._'+tgen['name'][id]+'.xxx.yyy._'+prot+'.'+self._config['mesos'].get('domain'),
'_'+tgen['name'][id]+'.xxx.yyy._'+prot+'.'+self._config['mesos'].get('domain')]:
if not self._testing_fqdn_srv.get(fqdn):
self._testing_fqdn_srv[fqdn]=[]
self._testing_fqdn_srv[fqdn].append({'name':tgen['host'][id],'port':ports[pid]})
self._testing[class_name]=False
class DiscoveryMesos(DiscoveryTestingTemplate,surok.discovery.DiscoveryMesos):
pass
class DiscoveryMarathon(DiscoveryTestingTemplate,surok.discovery.DiscoveryMarathon):
pass
class Discovery(surok.discovery.Discovery):
_discoveries={}
def __init__(self):
self._config=Config()
self._logger=Logger()
if not self._discoveries.get('mesos_dns'):
self._discoveries['mesos_dns']=DiscoveryMesos()
if not self._discoveries.get('marathon_api'):
self._discoveries['marathon_api']=DiscoveryMarathon()
class Test01_Logger(unittest.TestCase):
def test_01_logger_default_level(self):
logger = Logger()
self.assertEqual(logger.get_level(), 'info')
def test_02_logger_output_levels(self):
message='log message'
tests={
'debug':{
'assertIn':['ERROR: {}','WARNING: {}','INFO: {}','DEBUG: {}'],
'assertNotIn':[]
},
'info':{
'assertIn':['ERROR: {}','WARNING: {}','INFO: {}'],
'assertNotIn':['DEBUG: {}']
},
'warning':{
'assertIn':['ERROR: {}','WARNING: {}'],
'assertNotIn':['INFO: {}','DEBUG: {}']
},
'error':{
'assertIn':['ERROR: {}'],
'assertNotIn':['WARNING: {}','INFO: {}','DEBUG: {}']
}
}
logger = Logger()
for value01 in tests.keys():
logger.reset()
logger.set_level(value01)
logger.error(message)
logger.warning(message)
logger.info(message)
logger.debug(message)
resmessage=logger.geterr()+logger.getout()
for test_name in tests[value01].keys():
for test_value in tests[value01][test_name]:
with self.subTest(msg='Testing Logger for ...', loglevel=value01):
test_message=test_value.format(message)
eval('self.{}(test_message,resmessage)'.format(test_name))
class Test02_LoadConfig(unittest.TestCase):
def test_01_default_values(self):
config=Config()
with self.subTest(msg='Testing default values for Config.', dump=config.dump()):
self.assertEqual(config.get('confd'), '/etc/surok/conf.d')
self.assertEqual(config.get('default_discovery'), 'mesos_dns')
self.assertEqual(config.get('lock_dir'), '/var/tmp')
self.assertEqual(config.get('loglevel'), 'info')
self.assertEqual(dict(config.get('marathon',{})).get('enabled'), False)
self.assertEqual(dict(config.get('mesos',{})).get('enabled'), False)
self.assertEqual(dict(config.get('memcached',{})).get('enabled'), False)
self.assertEqual(config.get('version'), '0.7')
self.assertEqual(config.get('wait_time'), 20)
def test_02_main_conf(self):
config=Config('/etc/surok/conf/surok.json')
with self.subTest(msg='Testing load config for Config.', dump=config.dump()):
self.assertEqual(config.hash(), '545c20b322a6ba5fef9c7d2416d80178f26a924b')
def test_03_apps_conf(self):
tests=[
{
'env':{},
'self_check.json':'a4e109b9fec696776fd3df091b607e9c1489748c',
'marathon_check.json':'6be7f26d421d4a0a2e7b089184be0c0e3a50f986'
},
{
'env':{'SUROK_DISCOVERY_GROUP':'xxx.yyy'},
'self_check.json':'38ab770ff2ba69bf70673288425337ff3c18a807',
'marathon_check.json':'7b0cb4eab2d8e0f901cc567df28b17279af21baa'
},
{
'env':{'MARATHON_APP_ID':'/xxx/yyy/zzz'},
'self_check.json':'cbd2a15179649d0e06f98bd64e024481a944d65c',
'marathon_check.json':'08a382d14285feb1f22b92ba597ecf73d654a2e0'
}
]
config=Config()
for test in tests:
config.set('env',test['env'])
config.update_apps()
for app in config.apps:
with self.subTest(msg='Testing AppConfig for ...', env=test['env'], conf_name=app.get('conf_name'), dump=app.dump()):
self.assertEqual(test[app.get('conf_name')],app.hash())
def test_04_apps_conf(self):
tests={
'confd':{
'assertEqual': ['/var', '/var/tmp', '/etc/surok/conf.d'],
'assertNotEqual': [20, '/var/tmp1', '/etc/surok/conf/surok.json', 1, None, True]
},
'default_discovery':{
'assertEqual':['marathon_api', 'mesos_dns'],
'assertNotEqual':[20, 'test', None]
},
'lock_dir':{
'assertEqual':['/var', '/etc/surok/conf.d', '/var/tmp'],
'assertNotEqual':[20, '/var/tmp1', '/etc/surok/conf/surok.json', 1, None, True]
},
'loglevel':{
'assertEqual':['error', 'debug', 'info', 'warning'],
'assertNotEqual':['errrr', 'DEBUG','warn', 'test', 1, None, True]
},
'version':{
'assertEqual': ['0.7', '0.8'],
'assertNotEqual': ['0,7', '07', '0.9', 0.7, 0.8, None]
},
'wait_time':{
'assertEqual': [10, 15, 20],
'assertNotEqual': ['10', '15', None, True]
}
}
config=Config()
for name01 in tests.keys():
oldvalue=config.get(name01)
for test_name in tests[name01].keys():
for value01 in tests[name01][test_name]:
config.set_config({name01:value01})
test_value=config.get(name01)
with self.subTest(msg='Testing Config Change for values...', name=name01, value=value01, test_value=test_value):
eval('self.{}(value01, test_value)'.format(test_name))
config.set(name01,oldvalue)
class TestLoadConfig(unittest.TestCase): class Test03_Discovery(unittest.TestCase):
def test_main_conf(self): def test_01_discovery(self):
# Load base configurations tests={
surok_conf = '/etc/surok/conf/surok.json' 'T':{ #mesos_enabled
# Read config file 'T':{ #marathon_enabled
f = open(surok_conf, 'r') '0.7':{ #version
conf = json.loads(f.read()) 'mesos_dns':{ #default_discovery
f.close() 'marathon_check.json':'ef55fb10c20df700cb715f4836eadb2d0cfa9cc1', #app['conf_name']
'self_check.json':'53b8ddc27e357620f01ea75a7ab827cd90c77446'
self.assertIn('confd', conf) },
self.assertTrue(os.path.isdir(conf['confd'])) 'marathon_api':{
self.assertIn('domain', conf) 'marathon_check.json':'ef55fb10c20df700cb715f4836eadb2d0cfa9cc1',
self.assertIn('wait_time', conf) 'self_check.json':'53b8ddc27e357620f01ea75a7ab827cd90c77446'
self.assertIn('lock_dir', conf) }
self.assertTrue(os.path.isdir(conf['lock_dir'])) },
'0.8':{
'mesos_dns':{
class TestLogger(unittest.TestCase): 'marathon_check.json':'2016238426eb8ee7df9c4c016b6aecbfdf251a9b',
'self_check.json':'b6279a3e8e2fbbc78c6b302ef109bd6e2b456d9f'
def test_info(self): },
from surok.logger import make_message 'marathon_api':{
m = make_message 'marathon_check.json':'2016238426eb8ee7df9c4c016b6aecbfdf251a9b',
self.assertIn('INFO', m({'level': 'INFO', 'raw': 'log message'})) 'self_check.json':'b6279a3e8e2fbbc78c6b302ef109bd6e2b456d9f'
}
def test_warning(self): }
from surok.logger import make_message },
m = make_message 'F':{
self.assertIn('WARNING', m({'level': 'WARNING', 'raw': 'log message'})) '0.7':{
'mesos_dns':{
def test_error(self): 'marathon_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f',
from surok.logger import make_message 'self_check.json':'53b8ddc27e357620f01ea75a7ab827cd90c77446'
m = make_message },
self.assertIn('ERROR', m({'level': 'ERROR', 'raw': 'log message'})) 'marathon_api':{
'marathon_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f',
def test_info(self): 'self_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f'
from surok.logger import make_message }
m = make_message },
self.assertIn('DEBUG', m({'level': 'DEBUG', 'raw': 'log message'})) '0.8':{
'mesos_dns':{
'marathon_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f',
class TestMemcachedDiscovery(unittest.TestCase): 'self_check.json':'b6279a3e8e2fbbc78c6b302ef109bd6e2b456d9f'
},
def test_discovery_memcache(self): 'marathon_api':{
from surok.system import discovery_memcached 'marathon_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f',
'self_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f'
# Load base configurations }
surok_conf = '/etc/surok/conf/surok.json' }
# Read config file }
f = open(surok_conf, 'r') },
conf = json.loads(f.read()) 'F':{
f.close() 'T':{
'0.7':{
self.assertEqual(discovery_memcached(conf), []) 'mesos_dns':{
'marathon_check.json':'ef55fb10c20df700cb715f4836eadb2d0cfa9cc1',
'self_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f'
class TestGetGroup(unittest.TestCase): },
'marathon_api':{
def test_get_group(self): 'marathon_check.json':'ef55fb10c20df700cb715f4836eadb2d0cfa9cc1',
from surok.discovery import get_group 'self_check.json':'53b8ddc27e357620f01ea75a7ab827cd90c77446'
self.assertFalse(get_group({}, {'env': os.environ})) }
},
'0.8':{
'mesos_dns':{
'marathon_check.json':'2016238426eb8ee7df9c4c016b6aecbfdf251a9b',
'self_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f'
},
'marathon_api':{
'marathon_check.json':'2016238426eb8ee7df9c4c016b6aecbfdf251a9b',
'self_check.json':'b6279a3e8e2fbbc78c6b302ef109bd6e2b456d9f'
}
}
},
'F':{
'0.7':{
'mesos_dns':{
'marathon_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f',
'self_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f'
},
'marathon_api':{
'marathon_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f',
'self_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f'
}
},
'0.8':{
'mesos_dns':{
'marathon_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f',
'self_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f'
},
'marathon_api':{
'marathon_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f',
'self_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f'
}
}
}
}
}
config=Config('/etc/surok/conf/surok.json')
config.set('env',{'SUROK_DISCOVERY_GROUP':'xxx.yyy'})
discovery=Discovery()
for mesos_enabled in tests.keys():
for marathon_enabled in tests[mesos_enabled].keys():
for version in tests[mesos_enabled][marathon_enabled].keys():
for default_discovery in tests[mesos_enabled][marathon_enabled][version].keys():
config.set_config({'default_discovery':default_discovery,
'mesos':{'enabled':(mesos_enabled=='T')},
'marathon':{'enabled':(marathon_enabled=='T')},
'version':version})
discovery.update_data()
for app in config.apps:
conf_name=app.get('conf_name')
with self.subTest(msg='Testing Discovery for values...', config=config.dump(), conf_name=conf_name):
self.assertEqual(hashlib.sha1(json.dumps(discovery.resolve(app), sort_keys=True).encode()).hexdigest(),
tests[mesos_enabled][marathon_enabled][version][default_discovery][conf_name])
if __name__ == '__main__': if __name__ == '__main__':
unittest.main() unittest.main()

12
conf.d/marathon_check.json Normal file
View File

@ -0,0 +1,12 @@
{
"services": [
{
"name": "zzy*",
"ports": ["tname_a*"]
}
],
"template": "templates/selfcheck.jj2",
"dest": "selfcheck",
"reload_cmd": "/bin/echo selfcheck ok",
"discovery": "marathon_api"
}

22
conf.d/self_check.json Normal file
View File

@ -0,0 +1,22 @@
{
"services": [
{
"name": "zzy0",
"ports": ["tname_aa", "tname_ab", "tname_ba", "tname_bb", "tname_d"]
},
{
"name": "zzy1",
"ports": ["tname_aa", "tname_ab", "tname_ba", "tname_bb"]
},
{
"name": "zzz0",
"ports": ["tname_aa", "tname_bb"]
},
{
"name": "zzz1"
}
],
"template": "templates/selfcheck.jj2",
"dest": "selfcheck",
"reload_cmd": "/bin/echo selfcheck ok"
}

12
conf.d/selfcheck.json
View File

@ -1,12 +0,0 @@
{
"services": [
{
"name": "",
"group": "mesos"
}
],
"conf_name": "selfcheck",
"template": "templates/selfcheck.jj2",
"dest": "selfcheck",
"reload_cmd": "/bin/echo selfcheck ok"
}

35
conf/surok.json
View File

@ -1,22 +1,27 @@
{ {
"version": "0.8",
"marathon": { "marathon": {
"force": true, "enabled": false,
"host": "http://marathon.mesos:8080", "restart": false,
"enabled": false "force": true,
}, "host": "http://marathon.mesos:8080"
},
"mesos": {
"enabled": false,
"domain": "marathon.mesos"
},
"default_discovery": "mesos_dns",
"confd": "/etc/surok/conf.d", "confd": "/etc/surok/conf.d",
"domain": "marathon.mesos",
"wait_time": 20, "wait_time": 20,
"lock_dir": "/var/tmp", "lock_dir": "/var/tmp",
"loglevel": "info", "loglevel": "info",
"container": false, "memcached": {
"memcached": { "enabled": false,
"enabled": false, "discovery": {
"discovery": { "enabled": false,
"enabled": false, "service": "memcached",
"service": "memcached", "group": "system"
"group": "system" },
}, "hosts": ["localhost:11211"]
"hosts": ["localhost:11211"] }
}
} }

11
debian/changelog vendored
View File

@ -1,3 +1,12 @@
surok (0.8) testing; urgency=medium
* Version bump
* new config structure
* Port type
* Discovery over Marathon API
-- Denis Zheleztsov <difrex.punk@gmail.com> Tue, 07 Feb 2017 12:41:52 +0300
surok (0.7.4.3) testing; urgency=medium surok (0.7.4.3) testing; urgency=medium
* Fixed #9 * Fixed #9
@ -11,7 +20,7 @@ surok (0.7.4.3) testing; urgency=medium
surok (0.7.4.1) testing; urgency=medium surok (0.7.4.1) testing; urgency=medium
* closes #3 * closes #3
* Fixed bug with very long dns query lifetime * Fixed bug with very long dns query lifetime
-- Denis Zheleztsov <difrex.punk@gmail.com> Fri, 11 Nov 2016 21:15:02 +0300 -- Denis Zheleztsov <difrex.punk@gmail.com> Fri, 11 Nov 2016 21:15:02 +0300

4
debian/install vendored
View File

@ -1,7 +1,11 @@
conf/surok.json etc/surok/conf conf/surok.json etc/surok/conf
conf.d/self_check.json etc/surok/conf.d
conf.d/marathon_check.json etc/surok/conf.d
templates/selfcheck.jj2 etc/surok/templates
surok/templates.py opt/surok/surok surok/templates.py opt/surok/surok
surok/system.py opt/surok/surok surok/system.py opt/surok/surok
surok/logger.py opt/surok/surok surok/logger.py opt/surok/surok
surok/__init__.py opt/surok/surok surok/__init__.py opt/surok/surok
surok/discovery.py opt/surok/surok surok/discovery.py opt/surok/surok
surok/config.py opt/surok/surok
surok.py opt/surok surok.py opt/surok

4
docs/App-config-file.html
View File

@ -34,9 +34,9 @@
} }
</code> </code>
* <strong>services</strong>. List of hashes with required services for app. * <strong>services</strong>. List of hashes with required services for app.
1. <em>name</em> - string. App name in Marathon. 1. <em>name</em> - string. App name in Marathon. If you use a Marathon discovery, you can use the "*" at the end of the string to indicate any character.
2. <em>group</em> - string. App group in Marathon. Optional. Discovery policy: 1) config 2) SUROK<em>DISCOVERY</em>GROUP environment variable 3) Marathon API 2. <em>group</em> - string. App group in Marathon. Optional. Discovery policy: 1) config 2) SUROK<em>DISCOVERY</em>GROUP environment variable 3) Marathon API
3. <em>ports</em> - list. Name of opened port. In marathon of course. Optional. 3. <em>ports</em> - list. Name of opened port. In marathon of course. If you use a Marathon discovery, you can use the "*" at the end of the string to indicate any character. Optional.
* <strong>conf<em>name</strong>. Unique app config name. * <strong>conf<em>name</strong>. Unique app config name.
* <strong>template</strong>. Jinja2 template location. * <strong>template</strong>. Jinja2 template location.
* <strong>dest</strong>. Destination config path. * <strong>dest</strong>. Destination config path.

27
docs/App-config-file.md
View File

@ -5,26 +5,27 @@ conf.d/myapp.json
{ {
"services": [ "services": [
{ {
"name": "myapp", "name": "kioskservice",
"group": "backend.production", "group": "production.romania",
"ports": ["proxy", "api"] "ports": ["web", "socket"]
},
{
"name": "nginx",
"group": "frontend.production"
} }
], ],
"conf_name": "myapp_backend_production", "discovery": "mesos_dns",
"template": "/etc/surok/templates/myapp.jj2", "group": "dev.web",
"dest": "/etc/myapp/myapp.cfg", "conf_name": "kiosk",
"reload_cmd": "killall -9 myapp; /usr/local/bin/myapp -config /etc/myapp/myapp.cfg" "template": "/etc/surok/templates/kiosk.jj2",
"dest": "/etc/nginx/sites-available/kioskservice.conf",
"reload_cmd": "/sbin/nginx -t && /bin/systemctl reload nginx",
"run_cmd": ["/usr/bin/node", "-c", "config.json"]
} }
``` ```
* **services**. List of hashes with required services for app. * **services**. List of hashes with required services for app.
1. _name_ - string. App name in Marathon. 1. _name_ - string. App name in Marathon. If you use a Marathon discovery, you can use the "*" at the end of the string to indicate any character.
2. _group_ - string. App group in Marathon. Optional. Discovery policy: 1) config 2) SUROK_DISCOVERY_GROUP environment variable 3) Marathon API 2. _group_ - string. App group in Marathon. Optional. Discovery policy: 1) config 2) SUROK_DISCOVERY_GROUP environment variable 3) Marathon API
3. _ports_ - list. Name of opened port. In marathon of course. Optional. 3. _ports_ - list. Name of opened port. In marathon of course. If you use a Marathon discovery, you can use the "*" at the end of the string to indicate any character. Optional.
* **conf_name**. Unique app config name. * **conf_name**. Unique app config name.
* **template**. Jinja2 template location. * **template**. Jinja2 template location.
* **dest**. Destination config path. * **dest**. Destination config path.
* **reload_cmd**. Command to execute if generated config is changed. * **reload_cmd**. Command to execute if generated config is changed.
* **discovery**. Use custom discovery for app.
* **group**. Default group for all required services.

17
docs/Main-config-file.html
View File

@ -19,12 +19,21 @@
<code> <code>
{ {
"marathon": { "marathon": {
"force": true, "enabled": false,
"host": "http://marathon.mesos:8080", "restart": false,
"enabled": false "force": true,
"host": "http://marathon.mesos:8080"
}, },
"consul": {
"enabled": false,
"domain": "service.dc1.consul"
},
"mesos":{
"enabled": true,
"domain": "marathon.mesos"
},
"default_discovery": "mesos_dns",
"confd": "/etc/surok/conf.d", "confd": "/etc/surok/conf.d",
"domain": "marathon.mesos",
"wait_time": 20, "wait_time": 20,
"lock_dir": "/var/tmp", "lock_dir": "/var/tmp",
"loglevel": "info", "loglevel": "info",

92
docs/Main-config-file.md
View File

@ -1,36 +1,82 @@
# Surok main config file # Surok main config file (0.8.x)
Default location is /etc/surok/conf/surok.json Default location is **/etc/surok/conf/surok.json**
conf/surok.json
``` ```
{ {
"version": "0.8"
"marathon": { "marathon": {
"force": true, "enabled": false,
"host": "http://marathon.mesos:8080", "restart": false,
"enabled": false "force": true,
}, "host": "http://marathon.mesos:8080"
},
"mesos":{
"enabled": true,
"domain": "marathon.mesos"
},
"default_discovery": "mesos_dns",
"confd": "/etc/surok/conf.d", "confd": "/etc/surok/conf.d",
"domain": "marathon.mesos",
"wait_time": 20, "wait_time": 20,
"lock_dir": "/var/tmp", "lock_dir": "/var/tmp",
"loglevel": "info", "loglevel": "info",
"container": false,
"memcached": { "memcached": {
"enabled": false, "enabled": false,
"discovery": { "discovery": {
"enabled": false, "enabled": false,
"service": "memcached", "service": "memcached",
"group": "system" "group": "system"
}, },
"hosts": ["localhost:11211"] "hosts": ["localhost:11211"]
} }
} }
``` ```
* **marathon section**. Restarting app over marathon api if config changed. Disabled by default.
* **confd**. Directory where located configs apps. ## Config file options
* **domain**. Domain served by mesos-dns. * **version** - *string. Optional. "0.7" by default.*
* **lock_dir**. Directory where surok writes temporary configs after resolving. Config files and templates version. Accept "0.7" or "0.8".
* **wait_time**. Sleep time in main loop. * "0.7" - config files <= 0.7.х version
* **container**. Not implemented. * "0.8" - >= 0.8.x config files version
* **memcached section**. Memcached support. Disabled by default.
##### 0.8 version
* **marathon**, **mesos**, **consul**, **memcached** - *dict/hash. Optional. '{"enable":false}'. by default*
Surok working with folowing systems. If system is disabled parameters will be ignored.
* **enable** - *boolean. Optional. false by default*
Enable/disable system for usage.
Specific variables:
* For Marathon API "marathon"
* **force** - *boolean. Optional. true by default*
Force restart container over API.
* **restart** - *boolean. Optional. false by default*
Enable/disable restarting container
* **host** - *string. Optional. "http://marathon.mesos:8080" by default*
Marathon address.
* For mesos-dns "mesos"
* **domain** - *string. Optional. "marathon.mesos" by default*
mesos-dns private domain
* For Memcached "memcached"
* **hosts** - memcached hosts
* **discovery**
* **enabled** - boolean. Enable/disable disovery memcached service
* **service** - string. memcached app name
* **group** - string. memcached app group
* **default_discovery** - *string. Optional. "mesos_dns" by default*
Accept values:
* "mesos_dns" - mesos-dns
* "marathon_api"- Marathon API
* **confd** - *strig. Required.*
Path to directory with app config files.
* **wait_time** - *int. Required.*
Time in seconds how much Surok waits before starting to re-do the requests for service discovery
* **lock_dir** - *string. Required.*
Path to directory where Surok write lock-files.
* **loglevel** - *string. Optional. "info" by default*
Logleve. Accept values: "debug", "info", "warning", "error"
##### < 0.8 versions
* **marathon**
* **enabled** - boolean. Enable/disable container restart. Renamed to "restart" in 0.8 version.
* **domain** - string. mesos-dns private domain. Moved to "mesos" hashtable in 0.8 version.
Discovery over mesos-dns enabled all times.

1
docs/README.md
View File

@ -31,6 +31,7 @@ Service discovery for Apache Mesos.
* Denis Zheleztsov <difrex.punk@gmail.com> * Denis Zheleztsov <difrex.punk@gmail.com>
* Denis Ryabyy <vv1r0x@gmail.com> * Denis Ryabyy <vv1r0x@gmail.com>
* Evgeniy Vasilev <oren-ibc@yandex.ru>
## LICENSE ## LICENSE

179
docs/Templates.md
View File

@ -5,8 +5,11 @@
- [Templates](#templates) - [Templates](#templates)
- [my dictionary in templates](#my-dictionary-in-templates) - [my dictionary in templates](#my-dictionary-in-templates)
- [Real template example](#real-template-example) - [0.8 version](#08-version)
- [Checks in template](#checks-in-template) - [Real template example](#real-template-example)
- [Checks in template](#checks-in-template)
- [0.7 version](#07-version)
- [Real template example](#real-template-example)
<!-- markdown-toc end --> <!-- markdown-toc end -->
@ -15,55 +18,74 @@ Surok using Jinja2 for templates. [Jinja2 documentation](http://jinja.pocoo.org/
## my dictionary in templates ## my dictionary in templates
### 0.8 version
``` ```
{ {
"services": { "services": {
"nginx": [ "asterisk": [
{ {
"name": "nginx.testing-kl92-s0.marathon.mesos.", "name": "nginx.testing-kl92-s0.marathon.mesos.",
"port": "31200", "ip": [
"ip": ["10.10.10.1"] "10.0.0.1",
"11.0.0.1"
],
"tcp": {
"rpc":31200,
"web":31201,
"sip":32000
},
"udp": {
"sip":31201
}
}, },
{ {
"name": "nginx.testing-kl123-s1.marathon.mesos.", "name": "nginx.testing-kl123-s1.marathon.mesos.",
"port": "32230", "ip": [
"ip": ["10.10.10.2"] "10.0.0.2",
"11.0.0.2"
],
"tcp": {
"rpc":31210,
"web":31211,
"sip":32010
},
"udp": {
"sip":31211
}
} }
], ],
"emailsender": [ "email": [
{ {
"name": "emailsender.testing-kl92-s0.marathon.mesos.", "name": "nginx.testing-kl92-s0.marathon.mesos.",
"port": "31201", "ip": [
"ip": ["10.10.10.1"] "10.0.0.1"
}, ],
{ "tcp": {
"name": "emailsender.testing-kl123-s1.marathon.mesos.", "smtp":31200,
"port": "32232", "pop":31201
"ip": ["10.10.10.1"] }
} }
], ],
"service-with-defined-ports": { "anyport": [
"web": [ {
{ "name": "nginx.testing-kl92-s0.marathon.mesos.",
"name": "f.q.d.n", "ip": [
"port": 12341 "10.0.0.1"
} ],
], "tcp": [
"rpc": [ 31200,
{ 31201
"name": "f.q.d.n", ]
"port": 12342 }
} ]
]
}
},
"env": { "env": {
"HOME": "/var/lib/nginx" "HOME": "/var/lib/nginx"
} }
} }
``` ```
## Real template example #### Real template example
nginx config nginx config
``` ```
@ -104,7 +126,7 @@ server {
} }
``` ```
## Checks in template #### Checks in template
_my['env']_ is a python os.environ class. Look bellow: _my['env']_ is a python os.environ class. Look bellow:
``` ```
@ -114,3 +136,90 @@ host = '{{my['env']['DB_HOST']}}'
host = 'localhost' host = 'localhost'
{% endif %} {% endif %}
``` ```
### 0.7 version
my dictionary in template
```
{
"services": {
"nginx": [
{
"name": "nginx.testing-kl92-s0.marathon.mesos.",
"port": "31200"
},
{
"name": "nginx.testing-kl123-s1.marathon.mesos.",
"port": "32230"
}
],
"emailsender": [
{
"name": "emailsender.testing-kl92-s0.marathon.mesos.",
"port": "31201"
},
{
"name": "emailsender.testing-kl123-s1.marathon.mesos.",
"port": "32232"
}
],
"service-with-defined-ports": {
"name-of-port0": [
{
"name": "f.q.d.n",
"port": 12341
}
],
"name-of-port2": [
{
"name": "f.q.d.n",
"port": 12342
}
]
}
},
"env": {
"HOME": "/var/lib/nginx"
}
}
```
#### Real template example
```
upstream matrix-http {
hash $remote_addr;
{% for server in my['services']['matrix']['http'] %}
server {{server['name']}}:{{server['port']}} max_fails=3;
{% endfor %}
}
upstream riot-http {
hash $remote_addr;
{% for server in my['services']['riot'] %}
server {{server['name']}}:{{server['port']}} max_fails=3;
{% endfor %}
}
server {
listen 10.15.56.157:80;
server_name matrix.example.com;
client_max_body_size 10m;
location / {
proxy_pass http://riot-http;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header Host $http_host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
location /_matrix/ {
proxy_pass http://matrix-http;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header Host $http_host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
}
```

4
docs/ru/app.configuration.md
View File

@ -10,6 +10,8 @@
"ports": ["web", "socket"] "ports": ["web", "socket"]
} }
], ],
"discovery": "mesos_dns",
"group": "dev.web",
"conf_name": "kiosk", "conf_name": "kiosk",
"template": "/etc/surok/templates/kiosk.jj2", "template": "/etc/surok/templates/kiosk.jj2",
"dest": "/etc/nginx/sites-available/kioskservice.conf", "dest": "/etc/nginx/sites-available/kioskservice.conf",
@ -31,3 +33,5 @@
В reload_cmd можно использовать переменные окружения: В reload_cmd можно использовать переменные окружения:
```"reload_cmd": "/usr/bin/killall -9 calc || true && /usr/local/bin/calc -c /app/calc.conf ${CALC_NUM}"``` ```"reload_cmd": "/usr/bin/killall -9 calc || true && /usr/local/bin/calc -c /app/calc.conf ${CALC_NUM}"```
* run_cmd(v0.6) - array. Список с командой на выполнение. Используется внутри контейнера вместо reload_cmd. * run_cmd(v0.6) - array. Список с командой на выполнение. Используется внутри контейнера вместо reload_cmd.
* discovery - Опциональный. переопределяет метод обнаружения сервисов для приложения
* group - Опциональный. Группа по-умолчанию.

91
docs/ru/surok.configuration.md
View File

@ -1,30 +1,81 @@
# Конфигурация Surok (0.7.x) # Конфигурация Surok (0.8.x)
**/etc/surok/conf/surok.json** Разберем конфигурационный файл по опциям
**/etc/surok/conf/surok.json**
Разберем конфигурационный файл по опциям
``` ```
{ {
"version": "0.8"
"marathon": { "marathon": {
"force": true, "enabled": false,
"host": "marathon.mesos:8080", "restart": false,
"enabled": true "force": true,
"host": "http://marathon.mesos:8080"
}, },
"mesos":{
"enabled": true,
"domain": "marathon.mesos"
},
"default_discovery": "mesos_dns",
"confd": "/etc/surok/conf.d", "confd": "/etc/surok/conf.d",
"domain": "marathon.mesos",
"wait_time": 20, "wait_time": 20,
"lock_dir": "/var/tmp", "lock_dir": "/var/tmp",
"loglevel": "info|debug" "loglevel": "info",
"container": true|false "memcached": {
"enabled": false,
"discovery": {
"enabled": false,
"service": "memcached",
"group": "system"
},
"hosts": ["localhost:11211"]
}
} }
``` ```
## Опции файла конфигурации
* **version** - *string. Не обязательный. По умолчанию "0.7".*
Версия файлов конфигурации, шаблонов. На текущий момент может принимать значения "0.7" или "0.8".
* значение "0.7" - файлы конфигурации версии 0.7.х и более ранних
* значение "0.8" - файлы конфигурации версии 0.8
##### версия 0.8
* **marathon**, **mesos**, **consul**, **memcached** - *dict/hash. Не обязательный. По умолчанию '{"enable":false}'.*
Системы с которыми работает сурок. Если система выключена, то параметры системы и их наличие уже не важны.
* **enable** - *boolean. Не обязательный. По умолчанию false.*
Доступность системы для использования.
специфичные параметры:
* для Marathon API "marathon"
* **force** - *boolean. Не обязательный. По умолчанию true.*
Рестарт контейнера с force или нет.
* **restart** - *boolean. Не обязательный. По умолчанию false.*
Вкл/выкл. рестарта контейнера
* **host** - *string. Не обязательный. По умолчанию "http://marathon.mesos:8080".*
Адрес Marathon.
* для mesos DNS "mesos"
* **domain** - *string. Не обязательный. По умолчанию "marathon.mesos".*
Приватный домен Mesos DNS
* для Memcached "memcached"
* **hosts** -
* **discovery** -
* **enabled** -
* **service** -
* **group** -
* **default_discovery** - *string. Не обязательный. По умолчанию "mesos_dns".*
Может принимать значения:
* "mesos_dns" - Mesos DNS
* "marathon_api"- Marathon API
* **confd** - *strig. Обязательный.*
Абсолютный путь до директории с конфигурационными файлами приложений.
* **wait_time** - *int. Обязательный.*
Время в секундах сколько Surok ждет до того, как начать заново делать запросы на обнаружение сервисов.
* **lock_dir** - *string. Обязательный.*
Абсолютный путь до директории с lock-конфигурациями.
* **loglevel** - *string. Не обязательный. По умолчанию "info".*
Уровень логирования. Может принимать значения: "debug", "info", "warning", "error"
##### версия 0.7 и более ранние
Особенности для файла конфигурации
* **marathon**
* **enabled** - boolean. Вкл/выкл. рестарта контейнера. В версии 0.8 переименована в "restart".
* **domain** - string. Приватный домен Mesos DNS. В версии 0.8 перемещен в dict "mesos".
Обнаружение Mesos DNS включено всегда.
* marathon(v0.7) - hash. В текущей версии отвечает за перезапуск контейнера. Обнаружение сервисов через Marathon пока недоступно.
1. force - boolean. Рестарт контейнера с force или нет.
2. host - string. Адрес Marathon.
3. enabled - boolean. Вкл/выкл.
* confd - strig. Абсолютный путь до директории с конфигурационными файлами приложений.
* domain - string. Домен, который обслуживает mesos-dns.
* wait_time - int. Время в секундах сколько Surok ждет до того, как начать заново делать запросы на обнаружение сервисов.
* lock_dir - string. Абсолютный путь до директории с lock-конфигурациями.
* loglevel - string. Уровень логирования.
* container(v0.6) - boolean. Определяем внутри или нет контейнера запущен сурок. Меняется логика работы.

137
docs/ru/templates.md
View File

@ -4,6 +4,127 @@
## Словарь my в шаблоне ## Словарь my в шаблоне
### Версия 0.8
Surok заполняет словарь my и передает его в шаблон.
```
{
"services": {
"asterisk": [
{
"name": "nginx.testing-kl92-s0.marathon.mesos.",
"ip": [
"10.0.0.1",
"11.0.0.1"
],
"tcp": {
"rpc":31200,
"web":31201,
"sip":32000
},
"udp": {
"sip":31201
}
},
{
"name": "nginx.testing-kl123-s1.marathon.mesos.",
"ip": [
"10.0.0.2",
"11.0.0.2"
],
"tcp": {
"rpc":31210,
"web":31211,
"sip":32010
},
"udp": {
"sip":31211
}
}
],
"email": [
{
"name": "nginx.testing-kl92-s0.marathon.mesos.",
"ip": [
"10.0.0.1"
],
"tcp": {
"smtp":31200,
"pop":31201
}
}
],
"anyport": [
{
"name": "nginx.testing-kl92-s0.marathon.mesos.",
"ip": [
"10.0.0.1"
],
"tcp": [
31200,
31201
]
}
]
"env": {
"HOME": "/var/lib/nginx"
}
}
```
## Пример реального шаблона
```
upstream matrix-http {
hash $remote_addr;
{% for server in my['services']['matrix'] %}
server {{server['name']}}:{{server['tcp']['http']}} max_fails=3;
{% endfor %}
}
upstream riot-http {
hash $remote_addr;
{% for server in my['services']['riot'] %}
server {{server['name']}}:{{server['tcp'][0]}} max_fails=3;
{% endfor %}
}
server {
listen 10.15.56.157:80;
server_name matrix.example.com;
client_max_body_size 10m;
location / {
proxy_pass http://riot-http;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header Host $http_host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
location /_matrix/ {
proxy_pass http://matrix-http;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header Host $http_host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
}
```
Так для upstream matrix-http используются именованные порты, а для riot-http – нет.
## Проверки в шаблоне
Переменная _my['env']_ является классом python _os.environ_, что позваоляет нам строить различные проверки, например:
```
{% if my['env'].get('DB_HOST') -%}
host = '{{my['env']['DB_HOST']}}'
{% else -%}
host = 'localhost'
{% endif -%}
```
### Версия 0.7
Surok заполняет словарь my и передает его в шаблон. Surok заполняет словарь my и передает его в шаблон.
``` ```
{ {
@ -29,13 +150,13 @@ Surok заполняет словарь my и передает его в шаб
} }
], ],
"service-with-defined-ports": { "service-with-defined-ports": {
"web": [ "name-of-port0": [
{ {
"name": "f.q.d.n", "name": "f.q.d.n",
"port": 12341 "port": 12341
} }
], ],
"rpc": [ "name-of-port2": [
{ {
"name": "f.q.d.n", "name": "f.q.d.n",
"port": 12342 "port": 12342
@ -58,34 +179,34 @@ upstream matrix-http {
server {{server['name']}}:{{server['port']}} max_fails=3; server {{server['name']}}:{{server['port']}} max_fails=3;
{% endfor %} {% endfor %}
} }
upstream riot-http { upstream riot-http {
hash $remote_addr; hash $remote_addr;
{% for server in my['services']['riot'] %} {% for server in my['services']['riot'] %}
server {{server['name']}}:{{server['port']}} max_fails=3; server {{server['name']}}:{{server['port']}} max_fails=3;
{% endfor %} {% endfor %}
} }
server { server {
listen 10.15.56.157:80; listen 10.15.56.157:80;
server_name matrix.example.com; server_name matrix.example.com;
client_max_body_size 10m; client_max_body_size 10m;
location / { location / {
proxy_pass http://riot-http; proxy_pass http://riot-http;
proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Real-IP $remote_addr;
proxy_set_header Host $http_host; proxy_set_header Host $http_host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
} }
location /_matrix/ { location /_matrix/ {
proxy_pass http://matrix-http; proxy_pass http://matrix-http;
proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Real-IP $remote_addr;
proxy_set_header Host $http_host; proxy_set_header Host $http_host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
} }
} }
``` ```
Так для upstream matrix-http используются именованные порты, а для riot-http – нет. Так для upstream matrix-http используются именованные порты, а для riot-http – нет.

67
surok.py
View File

@ -7,71 +7,42 @@ from os.path import isfile, join
import json import json
import argparse import argparse
from surok.templates import gen from surok.templates import gen
from surok.discovery import resolve from surok.discovery import Discovery
from surok.system import reload_conf from surok.system import reload_conf
from surok.logger import Logger
from surok.config import Config
logger = Logger()
# Load base configurations
surok_conf = '/etc/surok/conf/surok.json'
# Command line arguments # Command line arguments
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument('-c', '--config', help='surok.json path') parser.add_argument('-c', '--config', help='surok.json path')
args = parser.parse_args() args = parser.parse_args()
if args.config:
surok_conf = args.config
# Read config file
f = open(surok_conf, 'r')
conf = json.loads(f.read())
f.close()
# Get app configurations
# Return list of patches to app discovery configuration
def get_configs():
confs = [f for f in listdir(conf['confd']) if isfile(
join(conf['confd'], f))]
return sorted(confs)
# Get Surok App configuration
# Read app conf from file and return dict
def load_app_conf(app):
# Load OS environment to app_conf
f = open(conf['confd'] + '/' + app)
c = json.loads(f.read())
f.close()
c['env'] = os.environ
return c
# Load base configurations
config = Config(args.config if args.config else '/etc/surok/conf/surok.json')
# Main loop # Main loop
########### #
discovery = Discovery()
while 1: while 1:
confs = get_configs() # Update config from discovery object
for app in confs: discovery.update_data()
app_conf = load_app_conf(app) for app in config.apps:
# Will be removed later app_hosts = discovery.resolve(app)
# For old configs
loglevel = 'info'
if 'loglevel' in conf:
loglevel = conf['loglevel']
# Resolve services
app_hosts = resolve(app_conf, conf)
# Populate my dictionary # Populate my dictionary
my = {"services": app_hosts, my = {"services": app_hosts,
"conf_name": app_conf['conf_name']} "conf_name": app['conf_name']}
logger.debug('my=', my)
# Generate config from template # Generate config from template
service_conf = gen(my, app_conf['template']) service_conf = gen(my, app['template'])
reload_conf(service_conf, app_conf, conf, app_hosts) reload_conf(service_conf, app, config, app_hosts)
sleep(conf['wait_time']) sleep(config['wait_time'])

15
surok.spec
View File

@ -1,6 +1,6 @@
Summary: Simple service discovery for Apache Mesos clusters Summary: Simple service discovery for Apache Mesos clusters
Name: surok Name: surok
Version: 0.7.4.3 Version: 0.8
Release: 1 Release: 1
License: BSD License: BSD
Group: admin Group: admin
@ -25,10 +25,12 @@ install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok/logger.py %{buildr
install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok/system.py %{buildroot}/opt/surok/surok install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok/system.py %{buildroot}/opt/surok/surok
install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok/discovery.py %{buildroot}/opt/surok/surok install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok/discovery.py %{buildroot}/opt/surok/surok
install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok/templates.py %{buildroot}/opt/surok/surok install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok/templates.py %{buildroot}/opt/surok/surok
install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok/config.py %{buildroot}/opt/surok/surok
install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok.py %{buildroot}/opt/surok install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok.py %{buildroot}/opt/surok
mkdir -p %{buildroot}/etc/surok/{conf,conf.d,templates} mkdir -p %{buildroot}/etc/surok/{conf,conf.d,templates}
install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/conf/surok.json %{buildroot}/etc/surok/conf install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/conf/surok.json %{buildroot}/etc/surok/conf
install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/conf.d/selfcheck.json %{buildroot}/etc/surok/conf.d install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/conf.d/self_check.json %{buildroot}/etc/surok/conf.d
install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/conf.d/marathon_check.json %{buildroot}/etc/surok/conf.d
install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/templates/selfcheck.jj2 %{buildroot}/etc/surok/templates install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/templates/selfcheck.jj2 %{buildroot}/etc/surok/templates
install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/requriments.txt %{buildroot}/opt/surok install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/requriments.txt %{buildroot}/opt/surok
@ -59,7 +61,11 @@ cd /opt/surok && pip3 install -r requriments.txt
/opt/surok/surok/templates.py /opt/surok/surok/templates.py
/opt/surok/surok/templates.pyc /opt/surok/surok/templates.pyc
/opt/surok/surok/templates.pyo /opt/surok/surok/templates.pyo
/etc/surok/conf.d/selfcheck.json /opt/surok/surok/config.py
/opt/surok/surok/config.pyc
/opt/surok/surok/config.pyo
/etc/surok/conf.d/self_check.json
/etc/surok/conf.d/marathon_check.json
/etc/surok/templates/selfcheck.jj2 /etc/surok/templates/selfcheck.jj2
/opt/surok/requriments.txt /opt/surok/requriments.txt
%defattr(-,root,root,-) %defattr(-,root,root,-)
@ -67,6 +73,9 @@ cd /opt/surok && pip3 install -r requriments.txt
%changelog %changelog
* Tue Feb 7 2017 Denis Zheleztsov <difrex.punk@gmail.com>
- New major release
- Discovery over marathon api
* Mon Nov 14 2016 Denis Zheleztsov <difrex.punk@gmail.com> - * Mon Nov 14 2016 Denis Zheleztsov <difrex.punk@gmail.com> -
- Initial build. - Initial build.

420
surok/config.py Normal file
View File

@ -0,0 +1,420 @@
# Public names
__all__ = ['Config', 'AppConfig']
import hashlib
import importlib
import json
import os
from .logger import *
# Logger link
# Config singleton link
_config_singleton = None
'''
Test values
==================================================
key - key
value - value of key
type_value - type of value
type_par - additional parameters for test
'''
'''
Public Config object
==================================================
.set_config(conf_data) - set config data
Use: conf_data(str type) - path of json config file
conf_data(dict type) - dict with config
.set(key,value) - set config key
.get(key) - get config key
.update_apps() - update apps config data
.apps - Apps object. List of AppConfig oblects
'''
class _ConfigTemplate(dict):
_conf = {}
def _init_conf(self, params):
conf = {}
for k in params.keys():
if params[k].get('params'):
conf[k] = self._init_conf(params[k].get('params'))
else:
if params[k].get('value') is not None:
conf[k] = params[k].get('value')
return conf
def __init__(self, *conf_data):
self._logger = Logger()
if not self._conf:
self._conf = self._init_conf(self._params)
for c in conf_data:
self.set_config(c)
def _set_conf_params(self, oldconf, testconf, params):
conf = oldconf if oldconf else {}
for key in testconf.keys():
resvalue = None
param = params.get(key)
oldvalue = conf.get(key)
testvalue = testconf.get(key)
if param is None:
self._logger.error('Parameter "', key, '" value "', testvalue,
'" type is "', type(testvalue).__name__, '" not found')
else:
type_param = param.get('type')
resvalue = []
if type(testvalue).__name__ != 'list':
testvalue = [testvalue]
for testitem in testvalue:
if self._test_value(key, testitem, param):
if 'dict' in type_param:
if param.get('params'):
res = self._set_conf_params(
oldvalue, testitem, param.get('params'))
if res is not None:
resvalue.append(res)
else:
resvalue.append(testitem)
if 'list' not in type_param:
resvalue = list([None] + resvalue).pop()
if resvalue is not None and 'do' in type_param:
if not self._do_type_set(key, resvalue, param):
self._logger.warning(
'Parameter "', key, '" current "', resvalue, '" type is "', type(resvalue).__name__, '" testing failed')
resvalue = None
if resvalue is not None:
conf[key] = resvalue
return conf
def _test_value(self, key, value, param):
type_param = param.get('type')
type_value = [
x for x in type_param if x in ['str', 'int', 'bool', 'dict']]
if type_value:
if type(value).__name__ not in type_value:
self._logger.error(
'Parameter "', key, '" must be ', type_value,
' types, current "', value, '" (', type(value).__name__, ')')
return False
if 'value' in type_param:
if value not in param.get('values', []):
self._logger.error(
'Value "', value, '" of key "', key, '" unknown')
return False
if 'dir' in type_param:
if not os.path.isdir(value):
self._logger.error('Path "{}" not present'.format(value))
return False
elif 'file' in type_param:
if not os.path.isfile(value):
self._logger.error('File "{}" not present'.format(value))
return False
return True
else:
self._logger.error(
'Type for testing "{}" unknown'.format(type_value))
return False
def set_config(self, conf_data):
conf = {}
if type(conf_data).__name__ == 'str':
try:
self._logger.debug('Open file ', conf_data)
f = open(conf_data, 'r')
json_data = f.read()
f.close()
conf = json.loads(json_data)
except OSError as err:
self._logger.error("OS error: {0}".format(err))
pass
except ValueError as err:
self._logger.error('JSON format error: {0}'.format(err))
pass
except:
self._logger.error('Load main config file failed')
pass
elif type(conf).__name__ == 'dict':
conf = conf_data
else:
return False
self._conf = self._set_conf_params(self._conf, conf, self._params)
self._logger.debug('Conf=', self._conf)
def keys(self):
return self._conf.keys()
def dump(self):
return json.dumps(self._conf, sort_keys=True, indent=2)
def _do_type_set(self, key, value, params):
self._logger.error('_do_type_set handler is not defined')
return False
def hash(self):
return hashlib.sha1(json.dumps(self._conf, sort_keys=True).encode()).hexdigest()
def set(self, key, value):
self._conf[key] = value
def __setitem__(self, key, value):
self.set(key, value)
def get(self, key, default=None):
return self._conf.get(key, default)
def __getitem__(self, key):
return self.get(key)
def __contains__(self, item):
return bool(item in self._conf)
def __len__(self):
return self._conf.__len__()
def __str__(self):
return self._conf.__str__()
def __repr__(self):
return self._conf.__repr__()
class Config(_ConfigTemplate):
_params = {
'marathon': {
'params': {
'force': {
'value': True,
'type': ['bool']
},
'host': {
'value': 'http://marathon.mesos:8080',
'type': ['str']
},
'enabled': {
'value': False,
'type': ['bool']
},
'restart': {
'value': False,
'type': ['bool']
}
},
'type': ['dict']
},
'mesos': {
'params': {
"domain": {
'value': "marathon.mesos",
'type': ['str']
},
"enabled": {
'value': False,
'type': ['bool']
}
},
'type': ['dict']
},
'memcached': {
'params': {
'enabled': {
'value': False,
'type': ['bool']
},
'discovery': {
'params': {
'enabled': {
'value': False,
'type': ['bool']
},
'service': {
'value': 'memcached',
'type': ['str']
},
'group': {
'value': 'system',
'type': ['str']
}
},
'type': ['dict']
},
'hosts': {
'value': ['localhost:11211'],
'type': ['list', 'str']
}
},
'type': ['dict']
},
'version': {
'value': '0.7',
'type': ['str', 'value'],
'values': ['0.7', '0.8']
},
'confd': {
'value': '/etc/surok/conf.d',
'type': ['str', 'dir']
},
'wait_time': {
'value': 20,
'type': ['int']
},
'lock_dir': {
'value': '/var/tmp',
'type': ['str', 'dir']
},
'default_discovery': {
'value': 'mesos_dns',
'type': ['str', 'value'],
'values': ['mesos_dns', 'marathon_api']
},
'loglevel': {
'value': 'info',
'type': ['str', 'do'],
'do': 'set_loglevel'
}
}
def __new__(cls, *args):
global _config_singleton
if _config_singleton is None:
_config_singleton = super(Config, cls).__new__(cls)
return _config_singleton
def __init__(self, *conf_data):
super().__init__(*conf_data)
self.apps = _Apps()
def set_config(self, conf_data):
super().set_config(conf_data)
if self.get('version') == '0.7':
domain = self.get('domain')
if domain is not None and self.get('mesos') is None:
self.set('mesos', {'domain': domain, 'enabled': True})
def _do_type_set(self, key, value, param):
if param.get('do') == 'set_loglevel':
if self._logger.set_level(value):
return True
return False
def update_apps(self):
self.apps.reset()
for app in sorted([os.path.join(self.get('confd'), f) for f in os.listdir(self.get('confd')) if os.path.isfile(os.path.join(self.get('confd'), f))]):
self.apps.set(AppConfig(app))
'''
Private Apps object
==================================================
.get(app) - get _AppConfig object
.set(app) - set _AppConfig object
'''
class _Apps:
_apps = {}
_items = []
def get(self, key):
return self._apps.get(key)
def set(self, app):
self._apps[app.get('conf_name')] = app
def reset(self):
keys = [] + list(self.keys())
for k in keys:
del self._apps[k]
def __getitem__(self, key):
return self.get(key)
def __contains__(self, item):
return bool(item in self._apps)
def __iter__(self):
self._items = sorted(self.keys())
return self
def __next__(self):
if self._items:
return self.get(self._items.pop(0))
raise StopIteration
def keys(self):
return self._apps.keys()
'''
Public AppConfig object
==================================================
.set_config(conf_data) - set config data
Use: conf_data(str type) - path of json config file
conf_data(dict type) - dict with config
.set(key,value) - set config key
.get(key) - get config key
'''
class AppConfig(_ConfigTemplate):
_params = {
'conf_name': {
'type': ['str']
},
'services': {
'params': {
'name': {
'type': ['str']
},
'ports': {
'type': ['list', 'str']
},
'discovery': {
'type': ['str']
},
'group': {
'type': ['str']
}
},
'type': ['list', 'dict']
},
'template': {
'type': ['str', 'file']
},
'dest': {
'type': ['str']
},
'reload_cmd': {
'type': ['str']
},
'discovery': {
'type': ['str']
},
'group': {
'type': ['str']
}
}
def __init__(self, *conf_data):
self._config = Config()
super().__init__(*conf_data)
def set_config(self, conf_data):
super().set_config(conf_data)
self._conf.setdefault(
'discovery', self._config.get('default_discovery'))
self._conf.setdefault('group', self._get_default_group())
if type(conf_data).__name__ == 'str':
self._conf.setdefault('conf_name', os.path.basename(conf_data))
def _get_default_group(self):
env = self._config.get('env', dict(os.environ))
# Check environment variable
if env.get('SUROK_DISCOVERY_GROUP'):
return env['SUROK_DISCOVERY_GROUP']
# Check marathon environment variable
elif env.get('MARATHON_APP_ID'):
return ".".join(env['MARATHON_APP_ID'].split('/')[-2:0:-1])

301
surok/discovery.py
View File

@ -1,95 +1,232 @@
# Public names
__all__ = ['Discovery', 'DiscoveryMesos', 'DiscoveryMarathon']
import dns.resolver import dns.resolver
import dns.query import dns.query
from dns.exception import DNSException import os
from .logger import info, warning, error, debug
import sys import sys
import requests
from dns.exception import DNSException
from .config import *
from .logger import *
# Discovery object
_discovery_singleton = None
# Resolve service from mesos-dns SRV record class DiscoveryTemplate:
# return dict {"servicename": [{"name": "service.f.q.d.n.", "port": 9999}]}
def resolve(app, conf):
hosts = {}
services = app['services']
domain = conf['domain']
for service in services: def __init__(self):
hosts[service['name']] = {} self._config = Config()
self._logger = Logger()
group = get_group(service, app) def enabled(self):
if group is False: return self._config[self._config_section].get('enabled', False)
error('Group is not defined in config, SUROK_DISCOVERY_GROUP and MARATHON_APP_ID')
error('Not in Mesos launch?')
sys.exit(2)
# Port name from app config def update_data(self):
ports = None pass
# Do DNS queries
# Return array:
# ["10.10.10.1", "10.10.10.2"]
def do_query_a(self, fqdn):
servers = []
try: try:
ports = service['ports'] resolver = dns.resolver.Resolver()
for a_rdata in resolver.query(fqdn, 'A'):
servers.append(a_rdata.address)
except DNSException as e:
self._logger.error("Could not resolve ", fqdn)
return servers
# Do DNS queries
# Return array:
# [{"name": "f.q.d.n", "port": 8876, "ip": ["10.10.10.1", "10.10.10.2"]}]
def do_query_srv(self, fqdn):
servers = []
try:
resolver = dns.resolver.Resolver()
resolver.lifetime = 1
resolver.timeout = 1
query = resolver.query(fqdn, 'SRV')
for rdata in query:
info = str(rdata).split()
servers.append({'name': info[3][:-1], 'port': info[2]})
except DNSException as e:
self._logger.warning("Could not resolve ", fqdn)
return servers
class Discovery:
_discoveries = {}
def __new__(cls):
global _discovery_singleton
if _discovery_singleton is None:
_discovery_singleton = super(Discovery, cls).__new__(cls)
return _discovery_singleton
def __init__(self):
self._config = Config()
self._logger = Logger()
if not self._discoveries.get('mesos_dns'):
self._discoveries['mesos_dns'] = DiscoveryMesos()
if not self._discoveries.get('marathon_api'):
self._discoveries['marathon_api'] = DiscoveryMarathon()
def keys(self):
return self._discoveries.keys()
def resolve(self, app):
discovery = app.get('discovery', self._config.get('default_discovery'))
if discovery not in self.keys():
self._logger.warning('Discovery "', discovery, '" is not present')
return {}
if self._discoveries[discovery].enabled():
return self.compatible(self._discoveries[discovery].resolve(app))
else:
self._logger.error('Discovery "', discovery, '" is disabled')
return {}
def update_data(self):
self._config.update_apps()
for d in self.keys():
if self._discoveries[d].enabled():
self._discoveries[d].update_data()
def compatible(self, hosts):
compatible_hosts = {}
if self._config.get('version') == '0.7':
for service in hosts.keys():
for host in hosts[service]:
ports = host.get('tcp', [])
if type(ports).__name__ == 'list':
compatible_hosts[service] = []
for port in ports:
compatible_hosts[service].append(
{'name': host['name'],
'ip': host['ip'],
'port': str(port)})
else:
compatible_hosts[service] = {}
for port in ports.keys():
compatible_host = compatible_hosts[
service].setdefault(port, [])
compatible_host.append({'name': host['name'],
'ip': host['ip'],
'port': ports[port]})
return compatible_hosts
return hosts
class DiscoveryMesos(DiscoveryTemplate):
_config_section = 'mesos'
def resolve(self, app):
hosts = {}
services = app.get('services')
domain = self._config[self._config_section].get('domain')
for service in services:
group = service.get('group', app.get('group'))
if group is None:
self._logger.error('Group for service "{}" of config "{}" not found'.format(
service['name'], app.get('conf_name')))
continue
ports = service.get('ports')
name = service['name']
hosts[name] = {}
serv = hosts[name]
self._logger.debug(
'group=', group, ' ports=', ports, ' name=', name, ' serv=', serv)
for prot in ['tcp', 'udp']:
if ports is not None:
for port_name in ports:
for d in self.do_query_srv('_' + port_name + '._' + name + '.' + group + '._' + prot + '.' + domain):
hostname = d['name']
serv.setdefault(hostname, {'name': hostname,
'ip': self.do_query_a(hostname)})
serv[hostname].setdefault(prot, {})
serv[hostname][prot][port_name] = d['port']
else:
for d in self.do_query_srv('_' + name + '.' + group + '._' + prot + '.' + domain):
hostname = d['name']
if serv.get(hostname) is None:
serv[hostname] = {'name': hostname,
'ip': self.do_query_a(hostname)}
if serv[hostname].get(prot) is None:
serv[hostname][prot] = []
serv[hostname][prot].extend([d['port']])
hosts[name] = list(serv.values())
return hosts
class DiscoveryMarathon(DiscoveryTemplate):
_config_section = 'marathon'
_tasks = []
_ports = {}
def update_data(self):
hostname = self._config[self._config_section].get('host')
try:
ports = {}
for app in requests.get(hostname + '/v2/apps').json()['apps']:
ports[app['id']] = {}
if app.get('container') is not None and app['container'].get('type') == 'DOCKER':
ports[app['id']] = app['container'][
'docker'].get('portMappings', [])
self._ports = ports
except: except:
self._logger.warning(
'Apps (', hostname, '/v2/apps) request from Marathon API is failed')
pass
try:
self._tasks = requests.get(hostname + '/v2/tasks').json()['tasks']
except:
self._logger.warning(
'Tasks (', hostname, '/v2/tasks) request from Marathon API is failed')
pass pass
# This is fast fix for port naming def _test_mask(self, mask, value):
# Will be rewrite later return (mask.endswith('*') and value.startswith(mask[:-1])) or mask == value
fqdn = ''
if ports is not None:
for port_name in ports:
fqdn = '_' + port_name + '.' + '_' + service['name'] + '.' + group + '._tcp.' + domain
hosts[service['name']][port_name] = do_query(fqdn, conf['loglevel'])
else:
fqdn = '_' + service['name'] + '.' + group + '._tcp.' + domain
hosts[service['name']] = do_query(fqdn, conf['loglevel'])
return hosts def resolve(self, app):
hosts = {}
services = app.get('services')
# Do DNS queries if not services:
# Return array: services = [{'name': '*', 'ports': ['*']}]
# [{"name": "f.q.d.n", "port": 8876, "ip": ["10.10.10.1", "10.10.10.2"]}] for service in services:
def do_query(fqdn, loglevel): # Convert xxx.yyy.zzz to /zzz/yyy/xxx/ format
servers = [] group = service.get('group', app.get('group'))
try: if group is None:
resolver = dns.resolver.Resolver() self._logger.error('Group for service "{}" of config "{}" not found'.format(
resolver.lifetime = 1 service['name'], app.get('conf_name')))
resolver.timeout = 1 continue
query = resolver.query(fqdn, 'SRV') group = '/' + '/'.join(group.split('.')[::-1]) + '/'
for rdata in query: service_mask = group + service['name']
info = str(rdata).split() for task in self._tasks:
name = info[3][:-1] if self._test_mask(service_mask, task['appId']):
port = info[2] name = '.'.join(
server = {'name': name, 'port': port, 'ip': []} task['appId'][len(group):].split('/')[::-1])
a_query = resolver.query(name, 'A') hosts[name] = {}
for a_rdata in a_query: serv = hosts[name]
server['ip'].append(a_rdata.address) hostname = task['host']
servers.append(server) for task_port in self._ports[task['appId']]:
except DNSException as e: prot = task_port['protocol']
if loglevel != 'info': port_name = task_port['name']
error("Could not resolve " + fqdn) port = task['ports'][
task['servicePorts'].index(task_port['servicePort'])]
return servers if 'ports' in service:
for port_mask in service['ports']:
if self._test_mask(port_mask, port_name):
# Groups switch serv.setdefault(
# Priority: config, environment, marathon environment hostname, {'name': hostname,
def get_group(service, app): 'ip': self.do_query_a(hostname)})
# Check group in app conf serv[hostname].setdefault(prot, {})
if 'group' in service: serv[hostname][prot][port_name] = port
return service['group'] else:
# Check environment variable serv.setdefault(hostname, {'name': hostname,
elif app['env'].get('SUROK_DISCOVERY_GROUP'): 'ip': self.do_query_a(hostname)})
return app['env']['SUROK_DISCOVERY_GROUP'] serv[hostname].setdefault(prot, [])
# Check marathon environment variable serv[hostname][prot].extend([port])
elif app['env'].get('MARATHON_APP_ID'): hosts[name] = list(serv.values())
group = parse_marathon_app_id(app['env']['MARATHON_APP_ID']) return hosts
return group
else:
return False
# Parse MARATHON_APP_ID
# Return marathon.group
def parse_marathon_app_id(marathon_app_id):
marathon_app_id = marathon_app_id.split('/')
del(marathon_app_id[-1])
marathon_app_id.reverse()
group = ".".join(marathon_app_id)[:-1]
return(group)

94
surok/logger.py
View File

@ -1,36 +1,86 @@
# Public names
__all__ = ['Logger']
import sys import sys
from time import time import json
import time
# Logger singleton link
_logger_singleton = None
'''
Public Logger oblect
==================================================
.set_level(level) - set level messages
level - values 'debug', 'info', 'warning', 'error'
* error - write error message
* warning - write warning and error message
* info - write info, warning and error message
* debug - write all message
.get_level() - get level messages
.error(str) - write error message
.warning(str) - write warning message
.info(str) - write info message
.debug(str)- write error message
'''
def make_message(message): class Logger:
cur_time = str(time()) _loglevel = 'info'
m = '[' + cur_time + '] ' + message['level'] + ': ' + message['raw'] + "\n" _msg_level = {
return m 'debug': 'DEBUG',
'info': 'INFO',
'warning': 'WARNING',
'error': 'ERROR'
}
def __new__(cls, *args):
global _logger_singleton
if _logger_singleton is None:
_logger_singleton = super(Logger, cls).__new__(cls)
return _logger_singleton
def info(message): def __init__(self, *args):
req = {'level': 'INFO', 'raw': message} if args:
m = make_message(req) self.set_level(args[0])
sys.stdout.write(m) def set_level(self, level):
if level in ['debug', 'info', 'warning', 'error']:
self._loglevel = level
return True
else:
self.warning('Log level "', level, '" not valid')
return False
def get_level(self):
return self._loglevel
def warning(message): def _make_message(self, level, message):
req = {'level': 'WARNING', 'raw': message} r = []
m = make_message(req) for m in message:
if type(m).__name__ == 'str':
r.append(m)
else:
r.append(json.dumps(m, sort_keys=True, indent=2))
return '[' + str(time.time()) + '] ' + self._msg_level[level] + ': ' + ''.join(r) + "\n"
sys.stderr.write(m) def debug(self, *message):
if self.get_level() in ['debug']:
self._log2err(self._make_message('debug', message))
def info(self, *message):
if self.get_level() in ['debug', 'info']:
self._log2out(self._make_message('info', message))
def error(message): def warning(self, *message):
req = {'level': 'ERROR', 'raw': message} if self.get_level() in ['debug', 'info', 'warning']:
m = make_message(req) self._log2out(self._make_message('warning', message))
sys.stderr.write(m) def error(self, *message):
self._log2err(self._make_message('error', message))
def _log2err(self, out):
sys.stderr.write(out)
def debug(message): def _log2out(self, out):
req = {'level': 'DEBUG', 'raw': message} sys.stdout.write(out)
m = make_message(req)
sys.stderr.write(m)

45
surok/system.py
View File

@ -1,20 +1,20 @@
import os import os
import sys import sys
import requests import requests
from .discovery import resolve from .discovery import Discovery
from .logger import info, warning, error, debug from .logger import Logger
logger=Logger()
# Get old configuration # Get old configuration
def get_old(name, service_conf): def get_old(name, service_conf):
try: try:
path = '/var/tmp/surok.' + name path = '/var/tmp/surok.' + name
f = open(path, 'r') f = open(path, 'r')
old = f.read() old = f.read()
f.close() f.close()
except Exception as e: except Exception as e:
print(str(e)) logger.error(str(e))
return 0 return 0
if old == service_conf: if old == service_conf:
@ -53,7 +53,7 @@ def write_lock(name, service_conf):
def do_reload(service_conf, app_conf): def do_reload(service_conf, app_conf):
warning('Write new configuration of ' + app_conf['conf_name']) logger.warning('Write new configuration of ' + app_conf['conf_name'])
f = open(app_conf['dest'], 'w') f = open(app_conf['dest'], 'w')
f.write(service_conf) f.write(service_conf)
@ -68,6 +68,7 @@ def do_reload(service_conf, app_conf):
# Discovery memcached servers # Discovery memcached servers
def discovery_memcached(conf): def discovery_memcached(conf):
discovery=Discovery()
memcache = conf['memcached'] memcache = conf['memcached']
app_conf = { app_conf = {
"services": [ "services": [
@ -78,7 +79,7 @@ def discovery_memcached(conf):
] ]
} }
hosts = resolve(app_conf, conf) hosts = discovery.resolve(app_conf)
mc_servers = [] mc_servers = []
for server in hosts[memcache['discovery']['service']]: for server in hosts[memcache['discovery']['service']]:
@ -91,7 +92,8 @@ def discovery_memcached(conf):
# !!! NEED REFACTORING !!! # !!! NEED REFACTORING !!!
def reload_conf(service_conf, app_conf, conf, app_hosts): def reload_conf(service_conf, app_conf, conf, app_hosts):
# Check marathon enabled in configuration # Check marathon enabled in configuration
if conf['marathon']['enabled'] is True: if (conf.get('version','0.7')=='0.8' and conf['marathon'].get('restart',False)) or (
conf.get('version','0.7')=='0.7' and conf['marathon'].get('enable',False)):
if get_old(app_conf['conf_name'], service_conf) != 1: if get_old(app_conf['conf_name'], service_conf) != 1:
restart_self_in_marathon(conf['marathon']) restart_self_in_marathon(conf['marathon'])
@ -105,47 +107,38 @@ def reload_conf(service_conf, app_conf, conf, app_hosts):
mc_hosts = None mc_hosts = None
if conf['memcached']['discovery']['enabled'] is True: if conf['memcached']['discovery']['enabled'] is True:
mc_hosts = discovery_memcached(conf) mc_hosts = discovery_memcached(conf)
info('Discovered memcached hosts: ' + str(mc_hosts)) logger.info('Discovered memcached hosts: ' + str(mc_hosts))
else: else:
mc_hosts = conf['memcached']['hosts'] mc_hosts = conf['memcached']['hosts']
try: try:
mc = memcache.Client(mc_hosts) mc = memcache.Client(mc_hosts)
if get_old_from_memcache(mc, app_conf['conf_name'], app_hosts) != 1: if get_old_from_memcache(mc, app_conf['conf_name'], app_hosts) != 1:
stdout = do_reload(service_conf, app_conf) stdout = do_reload(service_conf, app_conf)
info(stdout) logger.info(stdout)
return True return True
except Exception as e: except Exception as e:
error('Cannot connect to memcached: ' + str(e)) logger.error('Cannot connect to memcached: ' + str(e))
else: else:
warning('DEPRECATED main conf file. Please use new syntax!') logger.warning('DEPRECATED main conf file. Please use new syntax!')
# End of memcache block # End of memcache block
####################### #######################
if get_old(app_conf['conf_name'], service_conf) != 1: if get_old(app_conf['conf_name'], service_conf) != 1:
stdout = do_reload(service_conf, app_conf) stdout = do_reload(service_conf, app_conf)
info(stdout) logger.info(stdout)
return True return True
else: else:
if conf['loglevel'] == 'debug': logger.debug('Same config ' + app_conf['conf_name'] + ' Skip reload')
debug('Same config ' + app_conf['conf_name'] + ' Skip reload')
return False return False
# Do POST request to marathon API # Do POST request to marathon API
# /v2/apps//app/name/restart # /v2/apps//app/name/restart
def restart_self_in_marathon(marathon): def restart_self_in_marathon(marathon):
host = marathon['host']
# Check MARATHON_APP_ID environment varible # Check MARATHON_APP_ID environment varible
if os.environ.get('MARATHON_APP_ID') is not True: if not os.environ.get('MARATHON_APP_ID',False):
error('Cannot find MARATHON_APP_ID. Not in Mesos?') logger.error('Cannot find MARATHON_APP_ID. Not in Mesos?')
sys.exit(2) sys.exit(2)
app_id = os.environ['MARATHON_APP_ID']
uri = 'http://' + host + '/v2/apps/' + app_id + '/restart'
# Ok. In this step we made restart request to Marathon # Ok. In this step we made restart request to Marathon
if marathon['force'] is True: r = requests.post('http://'+marathon['host']+'/v2/apps/'+os.environ['MARATHON_APP_ID']+'/restart',
r = requests.post(uri, data = {'force': 'true'}) data={'force': marathon.get('force',False)})
else:
r = requests.post(uri, data = {'force': 'false'})

13
surok_fedora.spec
View File

@ -1,6 +1,6 @@
Summary: Simple service discovery for Apache Mesos clusters Summary: Simple service discovery for Apache Mesos clusters
Name: surok Name: surok
Version: 0.7.4.3 Version: 0.8
Release: 1.fc24 Release: 1.fc24
License: BSD License: BSD
Group: admin Group: admin
@ -25,10 +25,12 @@ install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok/logger.py %{buildr
install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok/system.py %{buildroot}/opt/surok/surok install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok/system.py %{buildroot}/opt/surok/surok
install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok/discovery.py %{buildroot}/opt/surok/surok install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok/discovery.py %{buildroot}/opt/surok/surok
install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok/templates.py %{buildroot}/opt/surok/surok install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok/templates.py %{buildroot}/opt/surok/surok
install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok/config.py %{buildroot}/opt/surok/surok
install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok.py %{buildroot}/opt/surok install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok.py %{buildroot}/opt/surok
mkdir -p %{buildroot}/etc/surok/{conf,conf.d,templates} mkdir -p %{buildroot}/etc/surok/{conf,conf.d,templates}
install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/conf/surok.json %{buildroot}/etc/surok/conf install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/conf/surok.json %{buildroot}/etc/surok/conf
install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/conf.d/selfcheck.json %{buildroot}/etc/surok/conf.d install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/conf.d/self_check.json %{buildroot}/etc/surok/conf.d
install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/conf.d/marathon_check.json %{buildroot}/etc/surok/conf.d
install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/templates/selfcheck.jj2 %{buildroot}/etc/surok/templates install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/templates/selfcheck.jj2 %{buildroot}/etc/surok/templates
@ -43,13 +45,18 @@ rm -rf $RPM_BUILD_ROOT
/opt/surok/surok/discovery.py /opt/surok/surok/discovery.py
/opt/surok/surok/system.py /opt/surok/surok/system.py
/opt/surok/surok/templates.py /opt/surok/surok/templates.py
/etc/surok/conf.d/selfcheck.json /opt/surok/surok/config.py
/etc/surok/conf.d/self_check.json
/etc/surok/conf.d/marathon_check.json
/etc/surok/templates/selfcheck.jj2 /etc/surok/templates/selfcheck.jj2
%defattr(-,root,root,-) %defattr(-,root,root,-)
%doc %doc
%changelog %changelog
* Tue Feb 7 2017 Denis Zheleztsov <difrex.punk@gmail.com>
- New major release
- Discovery over marathon api
* Mon Nov 14 2016 Denis Zheleztsov <difrex.punk@gmail.com> - * Mon Nov 14 2016 Denis Zheleztsov <difrex.punk@gmail.com> -
- Initial build. - Initial build.