Update russian docs. Add compatible for version 0.7. Add config module. Fix discovery module. Update test.py
This commit is contained in:
Васильев Евгений Владимирович
parent
f8460d301a
commit
1f09875aa3
1
AUTHORS
1
AUTHORS
@ -1,2 +1,3 @@
|
||||
Denis Zheleztsov <difrex.punk@gmail.com>
|
||||
Denis Ryabyy <vv1r0x@gmail.com>
|
||||
Evgeniy Vasilev <oren-ibc@yandex.ru>
|
||||
|
||||
@ -5,7 +5,7 @@ set -e
|
||||
. functions.sh
|
||||
|
||||
function run_tests() {
|
||||
docker run -ti -v $(pwd)/tests.py:/opt/surok/tests.py \
|
||||
docker run --rm -ti -v $(pwd)/tests.py:/opt/surok/tests.py \
|
||||
-v $(pwd)/tests_entrypoint.sh:/tests_entrypoint.sh \
|
||||
--entrypoint /tests_entrypoint.sh \
|
||||
surok_base:latest
|
||||
|
||||
426
build/tests.py
Normal file → Executable file
426
build/tests.py
Normal file → Executable file
@ -1,80 +1,372 @@
|
||||
#!/usr/bin/python3
|
||||
import unittest
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import surok.config
|
||||
import surok.logger
|
||||
import surok.discovery
|
||||
import hashlib
|
||||
from surok.config import Config
|
||||
|
||||
class TestLoadConfig(unittest.TestCase):
|
||||
class Logger(surok.logger.Logger):
|
||||
_out=''
|
||||
_err=''
|
||||
def _log2err(self,out):
|
||||
self._err+=out
|
||||
|
||||
def test_main_conf(self):
|
||||
# Load base configurations
|
||||
surok_conf = '/etc/surok/conf/surok.json'
|
||||
# Read config file
|
||||
f = open(surok_conf, 'r')
|
||||
conf = json.loads(f.read())
|
||||
f.close()
|
||||
def _log2out(self,out):
|
||||
self._out+=out
|
||||
|
||||
self.assertIn('confd', conf)
|
||||
self.assertTrue(os.path.isdir(conf['confd']))
|
||||
self.assertIn('wait_time', conf)
|
||||
self.assertIn('lock_dir', conf)
|
||||
self.assertTrue(os.path.isdir(conf['lock_dir']))
|
||||
def geterr(self):
|
||||
return self._err
|
||||
|
||||
def getout(self):
|
||||
return self._out
|
||||
|
||||
def reset(self):
|
||||
self._err=''
|
||||
self._out=''
|
||||
|
||||
class DiscoveryTestingTemplate:
|
||||
_testing={}
|
||||
_testing_fqdn_a={
|
||||
"test.zzz0.test":['10.0.0.1','10.1.0.1'],
|
||||
"test.zzz1.test":['10.0.1.1','10.1.1.1'],
|
||||
"test.zzz2.test":['10.0.2.1','10.1.2.1'],
|
||||
"test.zzz3.test":['10.0.3.1','10.1.3.1']
|
||||
}
|
||||
_testing_fqdn_srv={}
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self._orig_logger=surok.logger.Logger()
|
||||
|
||||
def do_query_a(self,fqdn):
|
||||
res=self._testing_fqdn_a.get(fqdn,[])
|
||||
if res:
|
||||
return res
|
||||
else:
|
||||
self._orig_logger.error('Testing FQDN '+fqdn+' not found in test A records')
|
||||
sys.exit(2)
|
||||
|
||||
def do_query_srv(self,fqdn):
|
||||
res=self._testing_fqdn_srv.get(fqdn,[])
|
||||
if res or fqdn.startswith('_tname_e.') or fqdn.find('._udp.'):
|
||||
return res
|
||||
else:
|
||||
self._orig_logger.error('Testing FQDN '+fqdn+' not found in test SRV records')
|
||||
sys.exit(2)
|
||||
|
||||
def update_data(self):
|
||||
class_name=self.__class__.__name__
|
||||
tgen={
|
||||
"name": ["zzz0","zzy0","zzy1","zzz1"],
|
||||
"host": ["test.zzz0.test","test.zzz1.test","test.zzz2.test","test.zzz3.test"],
|
||||
"serv": ["tname_aa","tname_ab","tname_ba","tname_bb"],
|
||||
"ports": [12341,12342,12343,12344],
|
||||
"servicePorts": [21221,21222,21223,21224]
|
||||
}
|
||||
if self._testing.get(class_name,True):
|
||||
if class_name == 'DiscoveryMarathon':
|
||||
_tasks=[]
|
||||
_ports={}
|
||||
for id in (0,1,2,3):
|
||||
ports=[]+tgen['ports']
|
||||
servicePorts=[]+tgen['servicePorts']
|
||||
appId='/'.join(str(tgen['name'][id]+'.xxx.yyy.').split('.')[::-1])
|
||||
_ports[appId]=[]
|
||||
for pid in (0,1,2,3):
|
||||
ports[pid]+=pid*10
|
||||
servicePorts[pid]+=pid*100
|
||||
for prot in ['tcp','udp']:
|
||||
if pid<2 or prot == 'tcp':
|
||||
_ports[appId].append({'containerPort': 0,
|
||||
'hostPort': 0,
|
||||
'labels': {},
|
||||
'name': tgen['serv'][pid],
|
||||
'protocol': prot,
|
||||
'servicePort': servicePorts[pid]})
|
||||
|
||||
_tasks.append({'appId':appId,
|
||||
'host':tgen['host'][id],
|
||||
'ports':ports,
|
||||
'servicePorts':servicePorts})
|
||||
#_tname_a._zzy0.yyy.xxx._tcp.marathon.mesos
|
||||
self._tasks=_tasks
|
||||
self._ports=_ports
|
||||
elif class_name == 'DiscoveryMesos':
|
||||
for id in (0,1,2,3):
|
||||
ports=[]+tgen['ports']
|
||||
for pid in (0,1,2,3):
|
||||
ports[pid]+=pid*10
|
||||
for prot in ['tcp','udp']:
|
||||
if pid<2 or prot == 'tcp':
|
||||
for fqdn in ['_'+tgen['serv'][pid]+'._'+tgen['name'][id]+'.xxx.yyy._'+prot+'.'+self._config['mesos'].get('domain'),
|
||||
'_'+tgen['name'][id]+'.xxx.yyy._'+prot+'.'+self._config['mesos'].get('domain')]:
|
||||
if not self._testing_fqdn_srv.get(fqdn):
|
||||
self._testing_fqdn_srv[fqdn]=[]
|
||||
self._testing_fqdn_srv[fqdn].append({'name':tgen['host'][id],'port':ports[pid]})
|
||||
|
||||
self._testing[class_name]=False
|
||||
|
||||
class DiscoveryMesos(DiscoveryTestingTemplate,surok.discovery.DiscoveryMesos):
|
||||
pass
|
||||
|
||||
class DiscoveryMarathon(DiscoveryTestingTemplate,surok.discovery.DiscoveryMarathon):
|
||||
pass
|
||||
|
||||
class Discovery(surok.discovery.Discovery):
|
||||
_discoveries={}
|
||||
def __init__(self):
|
||||
self._config=Config()
|
||||
self._logger=Logger()
|
||||
if not self._discoveries.get('mesos_dns'):
|
||||
self._discoveries['mesos_dns']=DiscoveryMesos()
|
||||
if not self._discoveries.get('marathon_api'):
|
||||
self._discoveries['marathon_api']=DiscoveryMarathon()
|
||||
|
||||
class Test01_Logger(unittest.TestCase):
|
||||
def test_01_logger_default_level(self):
|
||||
logger = Logger()
|
||||
self.assertEqual(logger.get_level(), 'info')
|
||||
|
||||
def test_02_logger_output_levels(self):
|
||||
message='log message'
|
||||
tests={
|
||||
'debug':{
|
||||
'assertIn':['ERROR: {}','WARNING: {}','INFO: {}','DEBUG: {}'],
|
||||
'assertNotIn':[]
|
||||
},
|
||||
'info':{
|
||||
'assertIn':['ERROR: {}','WARNING: {}','INFO: {}'],
|
||||
'assertNotIn':['DEBUG: {}']
|
||||
},
|
||||
'warning':{
|
||||
'assertIn':['ERROR: {}','WARNING: {}'],
|
||||
'assertNotIn':['INFO: {}','DEBUG: {}']
|
||||
},
|
||||
'error':{
|
||||
'assertIn':['ERROR: {}'],
|
||||
'assertNotIn':['WARNING: {}','INFO: {}','DEBUG: {}']
|
||||
}
|
||||
}
|
||||
logger = Logger()
|
||||
for value01 in tests.keys():
|
||||
logger.reset()
|
||||
logger.set_level(value01)
|
||||
logger.error(message)
|
||||
logger.warning(message)
|
||||
logger.info(message)
|
||||
logger.debug(message)
|
||||
resmessage=logger.geterr()+logger.getout()
|
||||
for test_name in tests[value01].keys():
|
||||
for test_value in tests[value01][test_name]:
|
||||
with self.subTest(msg='Testing Logger for ...', loglevel=value01):
|
||||
test_message=test_value.format(message)
|
||||
eval('self.{}(test_message,resmessage)'.format(test_name))
|
||||
|
||||
class Test02_LoadConfig(unittest.TestCase):
|
||||
|
||||
def test_01_default_values(self):
|
||||
config=Config()
|
||||
with self.subTest(msg='Testing default values for Config.', dump=config.dump()):
|
||||
self.assertEqual(config.get('confd'), '/etc/surok/conf.d')
|
||||
self.assertEqual(config.get('default_discovery'), 'mesos_dns')
|
||||
self.assertEqual(config.get('lock_dir'), '/var/tmp')
|
||||
self.assertEqual(config.get('loglevel'), 'info')
|
||||
self.assertEqual(dict(config.get('marathon',{})).get('enabled'), False)
|
||||
self.assertEqual(dict(config.get('mesos',{})).get('enabled'), False)
|
||||
self.assertEqual(dict(config.get('memcached',{})).get('enabled'), False)
|
||||
self.assertEqual(config.get('version'), '0.7')
|
||||
self.assertEqual(config.get('wait_time'), 20)
|
||||
|
||||
def test_02_main_conf(self):
|
||||
config=Config('/etc/surok/conf/surok.json')
|
||||
with self.subTest(msg='Testing load config for Config.', dump=config.dump()):
|
||||
self.assertEqual(config.hash(), '545c20b322a6ba5fef9c7d2416d80178f26a924b')
|
||||
|
||||
def test_03_apps_conf(self):
|
||||
tests=[
|
||||
{
|
||||
'env':{},
|
||||
'self_check.json':'a4e109b9fec696776fd3df091b607e9c1489748c',
|
||||
'marathon_check.json':'6be7f26d421d4a0a2e7b089184be0c0e3a50f986'
|
||||
},
|
||||
{
|
||||
'env':{'SUROK_DISCOVERY_GROUP':'xxx.yyy'},
|
||||
'self_check.json':'38ab770ff2ba69bf70673288425337ff3c18a807',
|
||||
'marathon_check.json':'7b0cb4eab2d8e0f901cc567df28b17279af21baa'
|
||||
},
|
||||
{
|
||||
'env':{'MARATHON_APP_ID':'/xxx/yyy/zzz'},
|
||||
'self_check.json':'cbd2a15179649d0e06f98bd64e024481a944d65c',
|
||||
'marathon_check.json':'08a382d14285feb1f22b92ba597ecf73d654a2e0'
|
||||
}
|
||||
]
|
||||
config=Config()
|
||||
for test in tests:
|
||||
config.set('env',test['env'])
|
||||
config.update_apps()
|
||||
for app in config.apps:
|
||||
with self.subTest(msg='Testing AppConfig for ...', env=test['env'], conf_name=app.get('conf_name'), dump=app.dump()):
|
||||
self.assertEqual(test[app.get('conf_name')],app.hash())
|
||||
|
||||
def test_04_apps_conf(self):
|
||||
tests={
|
||||
'confd':{
|
||||
'assertEqual': ['/var', '/var/tmp', '/etc/surok/conf.d'],
|
||||
'assertNotEqual': [20, '/var/tmp1', '/etc/surok/conf/surok.json', 1, None, True]
|
||||
},
|
||||
'default_discovery':{
|
||||
'assertEqual':['marathon_api', 'mesos_dns'],
|
||||
'assertNotEqual':[20, 'test', None]
|
||||
},
|
||||
'lock_dir':{
|
||||
'assertEqual':['/var', '/etc/surok/conf.d', '/var/tmp'],
|
||||
'assertNotEqual':[20, '/var/tmp1', '/etc/surok/conf/surok.json', 1, None, True]
|
||||
},
|
||||
'loglevel':{
|
||||
'assertEqual':['error', 'debug', 'info', 'warning'],
|
||||
'assertNotEqual':['errrr', 'DEBUG','warn', 'test', 1, None, True]
|
||||
},
|
||||
'version':{
|
||||
'assertEqual': ['0.7', '0.8'],
|
||||
'assertNotEqual': ['0,7', '07', '0.9', 0.7, 0.8, None]
|
||||
},
|
||||
'wait_time':{
|
||||
'assertEqual': [10, 15, 20],
|
||||
'assertNotEqual': ['10', '15', None, True]
|
||||
}
|
||||
|
||||
}
|
||||
config=Config()
|
||||
for name01 in tests.keys():
|
||||
oldvalue=config.get(name01)
|
||||
for test_name in tests[name01].keys():
|
||||
for value01 in tests[name01][test_name]:
|
||||
config.set_config({name01:value01})
|
||||
test_value=config.get(name01)
|
||||
with self.subTest(msg='Testing Config Change for values...', name=name01, value=value01, test_value=test_value):
|
||||
eval('self.{}(value01, test_value)'.format(test_name))
|
||||
config.set(name01,oldvalue)
|
||||
|
||||
|
||||
class TestLogger(unittest.TestCase):
|
||||
|
||||
def test_debug(self):
|
||||
from surok.logger import Logger
|
||||
m = Logger()
|
||||
self.assertIn('DEBUG', m.testing('debug','log message'))
|
||||
|
||||
def test_info(self):
|
||||
from surok.logger import Logger
|
||||
m = Logger()
|
||||
self.assertIn('INFO', m.testing('info','log message'))
|
||||
|
||||
def test_warning(self):
|
||||
from surok.logger import Logger
|
||||
m = Logger()
|
||||
self.assertIn('WARNING', m.testing('warning','log message'))
|
||||
|
||||
def test_error(self):
|
||||
from surok.logger import Logger
|
||||
m = Logger()
|
||||
self.assertIn('ERROR', m.testing('error','log message'))
|
||||
|
||||
|
||||
class TestMemcachedDiscovery(unittest.TestCase):
|
||||
|
||||
def test_discovery_memcache(self):
|
||||
from surok.system import discovery_memcached
|
||||
from surok.discovery import Discovery
|
||||
# Load base configurations
|
||||
surok_conf = '/etc/surok/conf/surok.json'
|
||||
# Read config file
|
||||
f = open(surok_conf, 'r')
|
||||
conf = json.loads(f.read())
|
||||
f.close()
|
||||
d=Discovery(conf)
|
||||
self.assertEqual(discovery_memcached(conf), [])
|
||||
|
||||
|
||||
class TestGetGroup(unittest.TestCase):
|
||||
|
||||
def test_get_group_from_service(self):
|
||||
from surok.discovery import DiscoveryTemplate
|
||||
d=DiscoveryTemplate({})
|
||||
self.assertEqual('xxx.yyy.zzz',d.get_group({'group':'xxx.yyy.zzz'}, {}))
|
||||
|
||||
def test_get_group_from_env(self):
|
||||
from surok.discovery import DiscoveryTemplate
|
||||
d=DiscoveryTemplate({})
|
||||
self.assertEqual('xxx.yyy.zzz',d.get_group({}, {'env':{'SUROK_DISCOVERY_GROUP':'xxx.yyy.zzz'}}))
|
||||
|
||||
def test_get_group_from_marathon_id(self):
|
||||
from surok.discovery import DiscoveryTemplate
|
||||
d=DiscoveryTemplate({})
|
||||
self.assertEqual('xxx.yyy.zzz',d.get_group({}, {'env':{'MARATHON_APP_ID':'/zzz/yyy/xxx/www'}}))
|
||||
class Test03_Discovery(unittest.TestCase):
|
||||
|
||||
def test_01_discovery(self):
|
||||
tests={
|
||||
'T':{ #mesos_enabled
|
||||
'T':{ #marathon_enabled
|
||||
'0.7':{ #version
|
||||
'mesos_dns':{ #default_discovery
|
||||
'marathon_check.json':'ef55fb10c20df700cb715f4836eadb2d0cfa9cc1', #app['conf_name']
|
||||
'self_check.json':'53b8ddc27e357620f01ea75a7ab827cd90c77446'
|
||||
},
|
||||
'marathon_api':{
|
||||
'marathon_check.json':'ef55fb10c20df700cb715f4836eadb2d0cfa9cc1',
|
||||
'self_check.json':'53b8ddc27e357620f01ea75a7ab827cd90c77446'
|
||||
}
|
||||
},
|
||||
'0.8':{
|
||||
'mesos_dns':{
|
||||
'marathon_check.json':'2016238426eb8ee7df9c4c016b6aecbfdf251a9b',
|
||||
'self_check.json':'b6279a3e8e2fbbc78c6b302ef109bd6e2b456d9f'
|
||||
},
|
||||
'marathon_api':{
|
||||
'marathon_check.json':'2016238426eb8ee7df9c4c016b6aecbfdf251a9b',
|
||||
'self_check.json':'b6279a3e8e2fbbc78c6b302ef109bd6e2b456d9f'
|
||||
}
|
||||
}
|
||||
},
|
||||
'F':{
|
||||
'0.7':{
|
||||
'mesos_dns':{
|
||||
'marathon_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f',
|
||||
'self_check.json':'53b8ddc27e357620f01ea75a7ab827cd90c77446'
|
||||
},
|
||||
'marathon_api':{
|
||||
'marathon_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f',
|
||||
'self_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f'
|
||||
}
|
||||
},
|
||||
'0.8':{
|
||||
'mesos_dns':{
|
||||
'marathon_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f',
|
||||
'self_check.json':'b6279a3e8e2fbbc78c6b302ef109bd6e2b456d9f'
|
||||
},
|
||||
'marathon_api':{
|
||||
'marathon_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f',
|
||||
'self_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f'
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
'F':{
|
||||
'T':{
|
||||
'0.7':{
|
||||
'mesos_dns':{
|
||||
'marathon_check.json':'ef55fb10c20df700cb715f4836eadb2d0cfa9cc1',
|
||||
'self_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f'
|
||||
},
|
||||
'marathon_api':{
|
||||
'marathon_check.json':'ef55fb10c20df700cb715f4836eadb2d0cfa9cc1',
|
||||
'self_check.json':'53b8ddc27e357620f01ea75a7ab827cd90c77446'
|
||||
}
|
||||
},
|
||||
'0.8':{
|
||||
'mesos_dns':{
|
||||
'marathon_check.json':'2016238426eb8ee7df9c4c016b6aecbfdf251a9b',
|
||||
'self_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f'
|
||||
},
|
||||
'marathon_api':{
|
||||
'marathon_check.json':'2016238426eb8ee7df9c4c016b6aecbfdf251a9b',
|
||||
'self_check.json':'b6279a3e8e2fbbc78c6b302ef109bd6e2b456d9f'
|
||||
}
|
||||
}
|
||||
},
|
||||
'F':{
|
||||
'0.7':{
|
||||
'mesos_dns':{
|
||||
'marathon_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f',
|
||||
'self_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f'
|
||||
},
|
||||
'marathon_api':{
|
||||
'marathon_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f',
|
||||
'self_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f'
|
||||
}
|
||||
},
|
||||
'0.8':{
|
||||
'mesos_dns':{
|
||||
'marathon_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f',
|
||||
'self_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f'
|
||||
},
|
||||
'marathon_api':{
|
||||
'marathon_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f',
|
||||
'self_check.json':'bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f'
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
config=Config('/etc/surok/conf/surok.json')
|
||||
config.set('env',{'SUROK_DISCOVERY_GROUP':'xxx.yyy'})
|
||||
discovery=Discovery()
|
||||
for mesos_enabled in tests.keys():
|
||||
for marathon_enabled in tests[mesos_enabled].keys():
|
||||
for version in tests[mesos_enabled][marathon_enabled].keys():
|
||||
for default_discovery in tests[mesos_enabled][marathon_enabled][version].keys():
|
||||
config.set_config({'default_discovery':default_discovery,
|
||||
'mesos':{'enabled':(mesos_enabled=='T')},
|
||||
'marathon':{'enabled':(marathon_enabled=='T')},
|
||||
'version':version})
|
||||
discovery.update_data()
|
||||
for app in config.apps:
|
||||
conf_name=app.get('conf_name')
|
||||
with self.subTest(msg='Testing Discovery for values...', config=config.dump(), conf_name=conf_name):
|
||||
self.assertEqual(hashlib.sha1(json.dumps(discovery.resolve(app), sort_keys=True).encode()).hexdigest(),
|
||||
tests[mesos_enabled][marathon_enabled][version][default_discovery][conf_name])
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
||||
12
conf.d/marathon_check.json
Normal file
12
conf.d/marathon_check.json
Normal file
@ -0,0 +1,12 @@
|
||||
{
|
||||
"services": [
|
||||
{
|
||||
"name": "zzy*",
|
||||
"ports": ["tname_a*"]
|
||||
}
|
||||
],
|
||||
"template": "templates/selfcheck.jj2",
|
||||
"dest": "selfcheck",
|
||||
"reload_cmd": "/bin/echo selfcheck ok",
|
||||
"discovery": "marathon_api"
|
||||
}
|
||||
17
conf.d/self_check.json
Normal file
17
conf.d/self_check.json
Normal file
@ -0,0 +1,17 @@
|
||||
{
|
||||
"services": [
|
||||
{"name":"zzy0",
|
||||
"ports":["tname_aa","tname_ab","tname_ba","tname_bb","tname_d"]
|
||||
},
|
||||
{"name":"zzy1",
|
||||
"ports":["tname_aa","tname_ab","tname_ba","tname_bb"]
|
||||
},
|
||||
{"name":"zzz0",
|
||||
"ports":["tname_aa","tname_bb"]
|
||||
},
|
||||
{"name":"zzz1"}
|
||||
],
|
||||
"template": "templates/selfcheck.jj2",
|
||||
"dest": "selfcheck",
|
||||
"reload_cmd": "/bin/echo selfcheck ok"
|
||||
}
|
||||
@ -1,12 +0,0 @@
|
||||
{
|
||||
"services": [
|
||||
{
|
||||
"name": "",
|
||||
"group": "mesos"
|
||||
}
|
||||
],
|
||||
"conf_name": "selfcheck",
|
||||
"template": "templates/selfcheck.jj2",
|
||||
"dest": "selfcheck",
|
||||
"reload_cmd": "/bin/echo selfcheck ok"
|
||||
}
|
||||
@ -1,16 +1,13 @@
|
||||
{
|
||||
"version":"0.8",
|
||||
"marathon": {
|
||||
"enabled": false,
|
||||
"restart": false,
|
||||
"force": true,
|
||||
"host": "http://marathon.mesos:8080"
|
||||
},
|
||||
"consul": {
|
||||
"enabled": false,
|
||||
"domain": "service.dc1.consul"
|
||||
},
|
||||
"mesos":{
|
||||
"enabled": true,
|
||||
"enabled": false,
|
||||
"domain": "marathon.mesos"
|
||||
},
|
||||
"default_discovery": "mesos_dns",
|
||||
@ -18,7 +15,6 @@
|
||||
"wait_time": 20,
|
||||
"lock_dir": "/var/tmp",
|
||||
"loglevel": "info",
|
||||
"container": false,
|
||||
"memcached": {
|
||||
"enabled": false,
|
||||
"discovery": {
|
||||
|
||||
4
debian/install
vendored
4
debian/install
vendored
@ -1,7 +1,11 @@
|
||||
conf/surok.json etc/surok/conf
|
||||
conf.d/self_check.json etc/surok/conf.d
|
||||
conf.d/marathon_check.json etc/surok/conf.d
|
||||
templates/selfcheck.jj2 etc/surok/templates
|
||||
surok/templates.py opt/surok/surok
|
||||
surok/system.py opt/surok/surok
|
||||
surok/logger.py opt/surok/surok
|
||||
surok/__init__.py opt/surok/surok
|
||||
surok/discovery.py opt/surok/surok
|
||||
surok/config.py opt/surok/surok
|
||||
surok.py opt/surok
|
||||
|
||||
@ -31,6 +31,7 @@ Service discovery for Apache Mesos.
|
||||
|
||||
* Denis Zheleztsov <difrex.punk@gmail.com>
|
||||
* Denis Ryabyy <vv1r0x@gmail.com>
|
||||
* Evgeniy Vasilev <oren-ibc@yandex.ru>
|
||||
|
||||
## LICENSE
|
||||
|
||||
|
||||
@ -1,30 +1,89 @@
|
||||
# Конфигурация Surok (0.7.x)
|
||||
|
||||
**/etc/surok/conf/surok.json**
|
||||
Разберем конфигурационный файл по опциям
|
||||
# Конфигурация Surok (0.8.x)
|
||||
**/etc/surok/conf/surok.json** Разберем конфигурационный файл по опциям
|
||||
```
|
||||
{
|
||||
"version": "0.8"
|
||||
"marathon": {
|
||||
"force": true,
|
||||
"host": "marathon.mesos:8080",
|
||||
"enabled": true
|
||||
"enabled": false,
|
||||
"restart": false,
|
||||
"force": true,
|
||||
"host": "http://marathon.mesos:8080"
|
||||
},
|
||||
"consul": {
|
||||
"enabled": false,
|
||||
"domain": "service.dc1.consul"
|
||||
},
|
||||
"mesos":{
|
||||
"enabled": true,
|
||||
"domain": "marathon.mesos"
|
||||
},
|
||||
"default_discovery": "mesos_dns",
|
||||
"confd": "/etc/surok/conf.d",
|
||||
"domain": "marathon.mesos",
|
||||
"wait_time": 20,
|
||||
"lock_dir": "/var/tmp",
|
||||
"loglevel": "info|debug"
|
||||
"container": true|false
|
||||
"loglevel": "info",
|
||||
"memcached": {
|
||||
"enabled": false,
|
||||
"discovery": {
|
||||
"enabled": false,
|
||||
"service": "memcached",
|
||||
"group": "system"
|
||||
},
|
||||
"hosts": ["localhost:11211"]
|
||||
}
|
||||
}
|
||||
```
|
||||
## Опции файла конфигурации
|
||||
* **version** - *string. Не обязательный. По умолчанию "0.7".*
|
||||
Версия файлов конфигурации, шаблонов. На текущий момент может принимать значения "0.7" или "0.8".
|
||||
* значение "0.7" - файлы конфигурации версии 0.7.х и более ранних
|
||||
* значение "0.8" - файлы конфигурации версии 0.8
|
||||
|
||||
##### версия 0.8
|
||||
* **marathon**, **mesos**, **consul**, **memcached** - *dict/hash. Не обязательный. По умолчанию '{"enable":false}'.*
|
||||
Системы с которыми работает сурок. Если система выключена, то параметры системы и их наличие уже не важны.
|
||||
* **enable** - *boolean. Не обязательный. По умолчанию false.*
|
||||
Доступность системы для использования.
|
||||
|
||||
специфичные параметры:
|
||||
* для Marathon API "marathon"
|
||||
* **force** - *boolean. Не обязательный. По умолчанию true.*
|
||||
Рестарт контейнера с force или нет.
|
||||
* **restart** - *boolean. Не обязательный. По умолчанию false.*
|
||||
Вкл/выкл. рестарта контейнера
|
||||
* **host** - *string. Не обязательный. По умолчанию "http://marathon.mesos:8080".*
|
||||
Адрес Marathon.
|
||||
* для Consul "consul"
|
||||
* **domain** - *string. Обязательный.*
|
||||
Приватный домен Consul
|
||||
* для mesos DNS "mesos"
|
||||
* **domain** - *string. Не обязательный. По умолчанию "marathon.mesos".*
|
||||
Приватный домен Mesos DNS
|
||||
* для Memcached "memcached"
|
||||
* **hosts** -
|
||||
* **discovery** -
|
||||
* **enabled** -
|
||||
* **service** -
|
||||
* **group** -
|
||||
* **default_discovery** - *string. Не обязательный. По умолчанию "mesos_dns".*
|
||||
|
||||
Может принимать значения:
|
||||
* "mesos_dns" - Mesos DNS
|
||||
* "marathon_api"- Marathon API
|
||||
* "consul_dns" - Consul
|
||||
* **confd** - *strig. Обязательный.*
|
||||
Абсолютный путь до директории с конфигурационными файлами приложений.
|
||||
* **wait_time** - *int. Обязательный.*
|
||||
Время в секундах сколько Surok ждет до того, как начать заново делать запросы на обнаружение сервисов.
|
||||
* **lock_dir** - *string. Обязательный.*
|
||||
Абсолютный путь до директории с lock-конфигурациями.
|
||||
* **loglevel** - *string. Не обязательный. По умолчанию "info".*
|
||||
Уровень логирования. Может принимать значения: "debug", "info", "warning", "error"
|
||||
|
||||
##### версия 0.7 и более ранние
|
||||
Особенности для файла конфигурации
|
||||
* **marathon**
|
||||
* **enabled** - boolean. Вкл/выкл. рестарта контейнера. В версии 0.8 переименована в "restart".
|
||||
* **domain** - string. Приватный домен Mesos DNS. В версии 0.8 перемещен в dict "mesos".
|
||||
Обнаружение Mesos DNS включено всегда.
|
||||
|
||||
* marathon(v0.7) - hash. В текущей версии отвечает за перезапуск контейнера. Обнаружение сервисов через Marathon пока недоступно.
|
||||
1. force - boolean. Рестарт контейнера с force или нет.
|
||||
2. host - string. Адрес Marathon.
|
||||
3. enabled - boolean. Вкл/выкл.
|
||||
* confd - strig. Абсолютный путь до директории с конфигурационными файлами приложений.
|
||||
* domain - string. Домен, который обслуживает mesos-dns.
|
||||
* wait_time - int. Время в секундах сколько Surok ждет до того, как начать заново делать запросы на обнаружение сервисов.
|
||||
* lock_dir - string. Абсолютный путь до директории с lock-конфигурациями.
|
||||
* loglevel - string. Уровень логирования.
|
||||
* container(v0.6) - boolean. Определяем внутри или нет контейнера запущен сурок. Меняется логика работы.
|
||||
|
||||
@ -4,6 +4,127 @@
|
||||
|
||||
## Словарь my в шаблоне
|
||||
|
||||
### Версия 0.8
|
||||
Surok заполняет словарь my и передает его в шаблон.
|
||||
```
|
||||
{
|
||||
"services": {
|
||||
"asterisk": [
|
||||
{
|
||||
"name": "nginx.testing-kl92-s0.marathon.mesos.",
|
||||
"ip": [
|
||||
"10.0.0.1",
|
||||
"11.0.0.1"
|
||||
],
|
||||
"tcp": {
|
||||
"rpc":31200,
|
||||
"web":31201,
|
||||
"sip":32000
|
||||
},
|
||||
"udp": {
|
||||
"sip":31201
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "nginx.testing-kl123-s1.marathon.mesos.",
|
||||
"ip": [
|
||||
"10.0.0.2",
|
||||
"11.0.0.2"
|
||||
],
|
||||
"tcp": {
|
||||
"rpc":31210,
|
||||
"web":31211,
|
||||
"sip":32010
|
||||
},
|
||||
"udp": {
|
||||
"sip":31211
|
||||
}
|
||||
}
|
||||
],
|
||||
"email": [
|
||||
{
|
||||
"name": "nginx.testing-kl92-s0.marathon.mesos.",
|
||||
"ip": [
|
||||
"10.0.0.1"
|
||||
],
|
||||
"tcp": {
|
||||
"smtp":31200,
|
||||
"pop":31201
|
||||
}
|
||||
}
|
||||
],
|
||||
"anyport": [
|
||||
{
|
||||
"name": "nginx.testing-kl92-s0.marathon.mesos.",
|
||||
"ip": [
|
||||
"10.0.0.1"
|
||||
],
|
||||
"tcp": [
|
||||
31200,
|
||||
31201
|
||||
]
|
||||
}
|
||||
]
|
||||
"env": {
|
||||
"HOME": "/var/lib/nginx"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Пример реального шаблона
|
||||
|
||||
```
|
||||
upstream matrix-http {
|
||||
hash $remote_addr;
|
||||
{% for server in my['services']['matrix'] %}
|
||||
server {{server['name']}}:{{server['tcp']['http']}} max_fails=3;
|
||||
{% endfor %}
|
||||
}
|
||||
|
||||
upstream riot-http {
|
||||
hash $remote_addr;
|
||||
{% for server in my['services']['riot'] %}
|
||||
server {{server['name']}}:{{server['tcp'][0]}} max_fails=3;
|
||||
{% endfor %}
|
||||
}
|
||||
|
||||
server {
|
||||
listen 10.15.56.157:80;
|
||||
server_name matrix.example.com;
|
||||
|
||||
client_max_body_size 10m;
|
||||
|
||||
location / {
|
||||
proxy_pass http://riot-http;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header Host $http_host;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
}
|
||||
|
||||
location /_matrix/ {
|
||||
proxy_pass http://matrix-http;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header Host $http_host;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
}
|
||||
|
||||
}
|
||||
```
|
||||
Так для upstream matrix-http используются именованные порты, а для riot-http – нет.
|
||||
|
||||
## Проверки в шаблоне
|
||||
|
||||
Переменная _my['env']_ является классом python _os.environ_, что позваоляет нам строить различные проверки, например:
|
||||
|
||||
```
|
||||
{% if my['env'].get('DB_HOST') -%}
|
||||
host = '{{my['env']['DB_HOST']}}'
|
||||
{% else -%}
|
||||
host = 'localhost'
|
||||
{% endif -%}
|
||||
```
|
||||
|
||||
### Версия 0.7
|
||||
Surok заполняет словарь my и передает его в шаблон.
|
||||
```
|
||||
{
|
||||
@ -29,13 +150,13 @@ Surok заполняет словарь my и передает его в шаб
|
||||
}
|
||||
],
|
||||
"service-with-defined-ports": {
|
||||
"web": [
|
||||
"name-of-port0": [
|
||||
{
|
||||
"name": "f.q.d.n",
|
||||
"port": 12341
|
||||
}
|
||||
],
|
||||
"rpc": [
|
||||
"name-of-port2": [
|
||||
{
|
||||
"name": "f.q.d.n",
|
||||
"port": 12342
|
||||
@ -58,34 +179,34 @@ upstream matrix-http {
|
||||
server {{server['name']}}:{{server['port']}} max_fails=3;
|
||||
{% endfor %}
|
||||
}
|
||||
|
||||
|
||||
upstream riot-http {
|
||||
hash $remote_addr;
|
||||
{% for server in my['services']['riot'] %}
|
||||
server {{server['name']}}:{{server['port']}} max_fails=3;
|
||||
{% endfor %}
|
||||
}
|
||||
|
||||
|
||||
server {
|
||||
listen 10.15.56.157:80;
|
||||
server_name matrix.example.com;
|
||||
|
||||
|
||||
client_max_body_size 10m;
|
||||
|
||||
|
||||
location / {
|
||||
proxy_pass http://riot-http;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header Host $http_host;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
}
|
||||
|
||||
|
||||
location /_matrix/ {
|
||||
proxy_pass http://matrix-http;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header Host $http_host;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
```
|
||||
Так для upstream matrix-http используются именованные порты, а для riot-http – нет.
|
||||
|
||||
54
surok.py
54
surok.py
@ -10,45 +10,17 @@ from surok.templates import gen
|
||||
from surok.discovery import Discovery
|
||||
from surok.system import reload_conf
|
||||
from surok.logger import Logger
|
||||
from surok.config import Config
|
||||
|
||||
logger=Logger()
|
||||
# Load base configurations
|
||||
surok_conf = '/etc/surok/conf/surok.json'
|
||||
|
||||
# Command line arguments
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('-c', '--config', help='surok.json path')
|
||||
args = parser.parse_args()
|
||||
if args.config:
|
||||
surok_conf = args.config
|
||||
|
||||
# Read config file
|
||||
f = open(surok_conf, 'r')
|
||||
conf = json.loads(f.read())
|
||||
f.close()
|
||||
|
||||
|
||||
# Get app configurations
|
||||
# Return list of patches to app discovery configuration
|
||||
def get_configs():
|
||||
confs = [f for f in listdir(conf['confd']) if isfile(
|
||||
join(conf['confd'], f))]
|
||||
return sorted(confs)
|
||||
|
||||
|
||||
# Get Surok App configuration
|
||||
# Read app conf from file and return dict
|
||||
def load_app_conf(app):
|
||||
# Load OS environment to app_conf
|
||||
f = open(conf['confd'] + '/' + app)
|
||||
c = json.loads(f.read())
|
||||
f.close()
|
||||
|
||||
c['env'] = os.environ
|
||||
|
||||
return c
|
||||
|
||||
logger.set_level(conf.get('loglevel','info'))
|
||||
# Load base configurations
|
||||
config=Config(args.config if args.config else '/etc/surok/conf/surok.json')
|
||||
|
||||
# Main loop
|
||||
###########
|
||||
@ -56,29 +28,21 @@ logger.set_level(conf.get('loglevel','info'))
|
||||
discovery=Discovery()
|
||||
|
||||
while 1:
|
||||
confs = get_configs()
|
||||
|
||||
# Update config from discovery object
|
||||
discovery.set_config(conf)
|
||||
|
||||
# Update discovery data
|
||||
discovery.update_data()
|
||||
for app in config.apps:
|
||||
|
||||
for app in confs:
|
||||
app_conf = load_app_conf(app)
|
||||
|
||||
# Resolve services
|
||||
app_hosts = discovery.resolve(app_conf)
|
||||
app_hosts = discovery.resolve(app)
|
||||
|
||||
# Populate my dictionary
|
||||
my = {"services": app_hosts,
|
||||
"conf_name": app_conf['conf_name']}
|
||||
"conf_name": app['conf_name']}
|
||||
|
||||
logger.debug('my=',my)
|
||||
|
||||
# Generate config from template
|
||||
service_conf = gen(my, app_conf['template'])
|
||||
service_conf = gen(my, app['template'])
|
||||
|
||||
reload_conf(service_conf, app_conf, conf, app_hosts)
|
||||
reload_conf(service_conf, app, config, app_hosts)
|
||||
|
||||
sleep(conf['wait_time'])
|
||||
sleep(config['wait_time'])
|
||||
|
||||
10
surok.spec
10
surok.spec
@ -25,10 +25,12 @@ install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok/logger.py %{buildr
|
||||
install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok/system.py %{buildroot}/opt/surok/surok
|
||||
install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok/discovery.py %{buildroot}/opt/surok/surok
|
||||
install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok/templates.py %{buildroot}/opt/surok/surok
|
||||
install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok/config.py %{buildroot}/opt/surok/surok
|
||||
install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok.py %{buildroot}/opt/surok
|
||||
mkdir -p %{buildroot}/etc/surok/{conf,conf.d,templates}
|
||||
install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/conf/surok.json %{buildroot}/etc/surok/conf
|
||||
install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/conf.d/selfcheck.json %{buildroot}/etc/surok/conf.d
|
||||
install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/conf.d/self_check.json %{buildroot}/etc/surok/conf.d
|
||||
install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/conf.d/marathon_check.json %{buildroot}/etc/surok/conf.d
|
||||
install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/templates/selfcheck.jj2 %{buildroot}/etc/surok/templates
|
||||
install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/requriments.txt %{buildroot}/opt/surok
|
||||
|
||||
@ -59,7 +61,11 @@ cd /opt/surok && pip3 install -r requriments.txt
|
||||
/opt/surok/surok/templates.py
|
||||
/opt/surok/surok/templates.pyc
|
||||
/opt/surok/surok/templates.pyo
|
||||
/etc/surok/conf.d/selfcheck.json
|
||||
/opt/surok/surok/config.py
|
||||
/opt/surok/surok/config.pyc
|
||||
/opt/surok/surok/config.pyo
|
||||
/etc/surok/conf.d/self_check.json
|
||||
/etc/surok/conf.d/marathon_check.json
|
||||
/etc/surok/templates/selfcheck.jj2
|
||||
/opt/surok/requriments.txt
|
||||
%defattr(-,root,root,-)
|
||||
|
||||
410
surok/config.py
Normal file
410
surok/config.py
Normal file
@ -0,0 +1,410 @@
|
||||
# Public names
|
||||
__all__ = ['Config', 'AppConfig']
|
||||
|
||||
import hashlib
|
||||
import importlib
|
||||
import json
|
||||
import os
|
||||
from .logger import *
|
||||
|
||||
# Logger link
|
||||
|
||||
# Config singleton link
|
||||
_config_singleton = None
|
||||
|
||||
'''
|
||||
Test values
|
||||
==================================================
|
||||
key - key
|
||||
value - value of key
|
||||
type_value - type of value
|
||||
type_par - additional parameters for test
|
||||
|
||||
'''
|
||||
|
||||
|
||||
|
||||
'''
|
||||
Public Config object
|
||||
==================================================
|
||||
.set_config(conf_data) - set config data
|
||||
Use: conf_data(str type) - path of json config file
|
||||
conf_data(dict type) - dict with config
|
||||
.set(key,value) - set config key
|
||||
.get(key) - get config key
|
||||
.update_apps() - update apps config data
|
||||
.apps - Apps object. List of AppConfig oblects
|
||||
'''
|
||||
|
||||
class _ConfigTemplate(dict):
|
||||
_conf={}
|
||||
def _init_conf(self,params):
|
||||
conf={}
|
||||
for k in params.keys():
|
||||
if params[k].get('params'):
|
||||
conf[k]=self._init_conf(params[k].get('params'))
|
||||
else:
|
||||
if params[k].get('value') is not None:
|
||||
conf[k]=params[k].get('value')
|
||||
return conf
|
||||
|
||||
def __init__(self, *conf_data):
|
||||
self._logger = Logger()
|
||||
if not self._conf:
|
||||
self._conf = self._init_conf(self._params)
|
||||
for c in conf_data:
|
||||
self.set_config(c)
|
||||
|
||||
def _set_conf_params(self,oldconf,testconf,params):
|
||||
conf = oldconf if oldconf else {}
|
||||
for key in testconf.keys():
|
||||
resvalue=None
|
||||
param=params.get(key)
|
||||
oldvalue=conf.get(key)
|
||||
testvalue=testconf.get(key)
|
||||
if param is None:
|
||||
self._logger.error('Parameter "', key, '" value "', testvalue, '" type is "', type(testvalue).__name__, '" not found')
|
||||
else:
|
||||
type_param=param.get('type')
|
||||
resvalue=[]
|
||||
if type(testvalue).__name__ != 'list':
|
||||
testvalue=[testvalue]
|
||||
for testitem in testvalue:
|
||||
if self._test_value(key, testitem, param):
|
||||
if 'dict' in type_param:
|
||||
if param.get('params'):
|
||||
res=self._set_conf_params(oldvalue,testitem,param.get('params'))
|
||||
if res is not None:
|
||||
resvalue.append(res)
|
||||
else:
|
||||
resvalue.append(testitem)
|
||||
if 'list' not in type_param:
|
||||
resvalue=list([None]+resvalue).pop()
|
||||
if resvalue is not None and 'do' in type_param:
|
||||
if not self._do_type_set(key, resvalue, param):
|
||||
self._logger.warning('Parameter "', key, '" current "', resvalue, '" type is "', type(resvalue).__name__, '" testing failed')
|
||||
resvalue = None
|
||||
if resvalue is not None:
|
||||
conf[key]=resvalue
|
||||
return conf
|
||||
|
||||
def _test_value(self, key, value, param):
|
||||
type_param=param.get('type')
|
||||
type_value=[x for x in type_param if x in ['str', 'int', 'bool', 'dict']]
|
||||
if type_value:
|
||||
if type(value).__name__ not in type_value:
|
||||
self._logger.error('Parameter "', key, '" must be ', type_value,' types, current "', value, '" (',type(value).__name__,')')
|
||||
return False
|
||||
if 'value' in type_param:
|
||||
if value not in param.get('values',[]):
|
||||
self._logger.error('Value "', value, '" of key "', key, '" unknown')
|
||||
return False
|
||||
if 'dir' in type_param:
|
||||
if not os.path.isdir(value):
|
||||
self._logger.error('Path "{}" not present'.format(value))
|
||||
return False
|
||||
elif 'file' in type_param:
|
||||
if not os.path.isfile(value):
|
||||
self._logger.error('File "{}" not present'.format(value))
|
||||
return False
|
||||
return True
|
||||
else:
|
||||
self._logger.error('Type for testing "{}" unknown'.format(type_value))
|
||||
return False
|
||||
|
||||
def set_config(self, conf_data):
|
||||
conf = {}
|
||||
if type(conf_data).__name__ == 'str':
|
||||
try:
|
||||
self._logger.debug('Open file ', conf_data)
|
||||
f = open(conf_data, 'r')
|
||||
json_data=f.read()
|
||||
f.close()
|
||||
conf = json.loads(json_data)
|
||||
except OSError as err:
|
||||
self._logger.error("OS error: {0}".format(err))
|
||||
pass
|
||||
except ValueError as err:
|
||||
self._logger.error('JSON format error: {0}'.format(err))
|
||||
pass
|
||||
except:
|
||||
self._logger.error('Load main config file failed')
|
||||
pass
|
||||
elif type(conf).__name__ == 'dict':
|
||||
conf = conf_data
|
||||
else:
|
||||
return False
|
||||
self._conf=self._set_conf_params(self._conf,conf,self._params)
|
||||
self._logger.debug('Conf=', self._conf)
|
||||
|
||||
def keys(self):
|
||||
return self._conf.keys()
|
||||
|
||||
def dump(self):
|
||||
return json.dumps(self._conf, sort_keys=True, indent=2)
|
||||
|
||||
def _do_type_set(self, key, value, params):
|
||||
self._logger.error('_do_type_set handler is not defined')
|
||||
return False
|
||||
|
||||
def hash(self):
|
||||
return hashlib.sha1(json.dumps(self._conf, sort_keys=True).encode()).hexdigest()
|
||||
|
||||
def set(self, key, value):
|
||||
self._conf[key]=value
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
self.set(key, value)
|
||||
|
||||
def get(self, key, default=None):
|
||||
return self._conf.get(key, default)
|
||||
|
||||
def __getitem__(self, key):
|
||||
return self.get(key)
|
||||
|
||||
def __contains__(self, item):
|
||||
return bool(item in self._conf)
|
||||
|
||||
def __len__(self):
|
||||
return self._conf.__len__()
|
||||
|
||||
def __str__(self):
|
||||
return self._conf.__str__()
|
||||
|
||||
def __repr__(self):
|
||||
return self._conf.__repr__()
|
||||
|
||||
|
||||
class Config(_ConfigTemplate):
|
||||
_params = {
|
||||
'marathon': {
|
||||
'params': {
|
||||
'force': {
|
||||
'value': True,
|
||||
'type': ['bool']
|
||||
},
|
||||
'host': {
|
||||
'value': 'http://marathon.mesos:8080',
|
||||
'type': ['str']
|
||||
},
|
||||
'enabled': {
|
||||
'value': False,
|
||||
'type': ['bool']
|
||||
},
|
||||
'restart': {
|
||||
'value': False,
|
||||
'type': ['bool']
|
||||
}
|
||||
},
|
||||
'type':['dict']
|
||||
},
|
||||
'mesos': {
|
||||
'params': {
|
||||
"domain": {
|
||||
'value': "marathon.mesos",
|
||||
'type': ['str']
|
||||
},
|
||||
"enabled": {
|
||||
'value': False,
|
||||
'type': ['bool']
|
||||
}
|
||||
},
|
||||
'type':['dict']
|
||||
},
|
||||
'memcached': {
|
||||
'params': {
|
||||
'enabled': {
|
||||
'value': False,
|
||||
'type': ['bool']
|
||||
},
|
||||
'discovery': {
|
||||
'params': {
|
||||
'enabled': {
|
||||
'value': False,
|
||||
'type': ['bool']
|
||||
},
|
||||
'service': {
|
||||
'value': 'memcached',
|
||||
'type': ['str']
|
||||
},
|
||||
'group': {
|
||||
'value': 'system',
|
||||
'type': ['str']
|
||||
}
|
||||
},
|
||||
'type':['dict']
|
||||
},
|
||||
'hosts': {
|
||||
'value': ['localhost:11211'],
|
||||
'type': ['list','str']
|
||||
}
|
||||
},
|
||||
'type':['dict']
|
||||
},
|
||||
'version': {
|
||||
'value': '0.7',
|
||||
'type': ['str','value'],
|
||||
'values': ['0.7', '0.8']
|
||||
},
|
||||
'confd': {
|
||||
'value': '/etc/surok/conf.d',
|
||||
'type': ['str','dir']
|
||||
},
|
||||
'wait_time': {
|
||||
'value': 20,
|
||||
'type': ['int']
|
||||
},
|
||||
'lock_dir': {
|
||||
'value': '/var/tmp',
|
||||
'type': ['str','dir']
|
||||
},
|
||||
'default_discovery': {
|
||||
'value': 'mesos_dns',
|
||||
'type': ['str', 'value'],
|
||||
'values': ['mesos_dns', 'marathon_api']
|
||||
},
|
||||
'loglevel': {
|
||||
'value': 'info',
|
||||
'type': ['str','do'],
|
||||
'do':'set_loglevel'
|
||||
}
|
||||
}
|
||||
|
||||
def __new__(cls, *args):
|
||||
global _config_singleton
|
||||
if _config_singleton is None:
|
||||
_config_singleton = super(Config, cls).__new__(cls)
|
||||
return _config_singleton
|
||||
|
||||
def __init__(self, *conf_data):
|
||||
super().__init__( *conf_data)
|
||||
self.apps = _Apps()
|
||||
|
||||
def set_config(self, conf_data):
|
||||
super().set_config(conf_data)
|
||||
if self.get('version') == '0.7':
|
||||
domain = self.get('domain')
|
||||
if domain is not None and self.get('mesos') is None:
|
||||
self.set('mesos', {'domain': domain, 'enabled': True})
|
||||
|
||||
def _do_type_set(self, key, value, param):
|
||||
if param.get('do') == 'set_loglevel':
|
||||
if self._logger.set_level(value):
|
||||
return True
|
||||
return False
|
||||
|
||||
def update_apps(self):
|
||||
self.apps.reset()
|
||||
for app in sorted([os.path.join(self.get('confd'), f) for f in os.listdir(self.get('confd')) if os.path.isfile(os.path.join(self.get('confd'), f))]):
|
||||
self.apps.set(AppConfig(app))
|
||||
|
||||
'''
|
||||
Private Apps object
|
||||
==================================================
|
||||
.get(app) - get _AppConfig object
|
||||
.set(app) - set _AppConfig object
|
||||
'''
|
||||
|
||||
|
||||
class _Apps:
|
||||
_apps = {}
|
||||
_items = []
|
||||
|
||||
def get(self, key):
|
||||
return self._apps.get(key)
|
||||
|
||||
def set(self, app):
|
||||
self._apps[app.get('conf_name')] = app
|
||||
|
||||
def reset(self):
|
||||
keys=[]+list(self.keys())
|
||||
for k in keys:
|
||||
del self._apps[k]
|
||||
|
||||
def __getitem__(self, key):
|
||||
return self.get(key)
|
||||
|
||||
def __contains__(self, item):
|
||||
return bool(item in self._apps)
|
||||
|
||||
def __iter__(self):
|
||||
self._items = sorted(self.keys())
|
||||
return self
|
||||
|
||||
def __next__(self):
|
||||
if self._items:
|
||||
return self.get(self._items.pop(0))
|
||||
raise StopIteration
|
||||
|
||||
def keys(self):
|
||||
return self._apps.keys()
|
||||
|
||||
'''
|
||||
Public AppConfig object
|
||||
==================================================
|
||||
.set_config(conf_data) - set config data
|
||||
Use: conf_data(str type) - path of json config file
|
||||
conf_data(dict type) - dict with config
|
||||
.set(key,value) - set config key
|
||||
.get(key) - get config key
|
||||
'''
|
||||
|
||||
|
||||
class AppConfig(_ConfigTemplate):
|
||||
_params = {
|
||||
'conf_name': {
|
||||
'type': ['str']
|
||||
},
|
||||
'services': {
|
||||
'params': {
|
||||
'name': {
|
||||
'type': ['str']
|
||||
},
|
||||
'ports': {
|
||||
'type': ['list','str']
|
||||
},
|
||||
'discovery': {
|
||||
'type': ['str']
|
||||
},
|
||||
'group': {
|
||||
'type': ['str']
|
||||
}
|
||||
},
|
||||
'type':['list','dict']
|
||||
},
|
||||
'template': {
|
||||
'type': ['str','file']
|
||||
},
|
||||
'dest': {
|
||||
'type': ['str']
|
||||
},
|
||||
'reload_cmd': {
|
||||
'type': ['str']
|
||||
},
|
||||
'discovery': {
|
||||
'type': ['str']
|
||||
},
|
||||
'group': {
|
||||
'type': ['str']
|
||||
}
|
||||
}
|
||||
|
||||
def __init__(self, *conf_data):
|
||||
self._config = Config()
|
||||
super().__init__(*conf_data)
|
||||
|
||||
def set_config(self, conf_data):
|
||||
super().set_config(conf_data)
|
||||
self._conf.setdefault('discovery', self._config.get('default_discovery'))
|
||||
self._conf.setdefault('group', self._get_default_group())
|
||||
if type(conf_data).__name__ == 'str':
|
||||
self._conf.setdefault('conf_name', os.path.basename(conf_data))
|
||||
|
||||
def _get_default_group(self):
|
||||
env=self._config.get('env',dict(os.environ))
|
||||
# Check environment variable
|
||||
if env.get('SUROK_DISCOVERY_GROUP'):
|
||||
return env['SUROK_DISCOVERY_GROUP']
|
||||
# Check marathon environment variable
|
||||
elif env.get('MARATHON_APP_ID'):
|
||||
return ".".join(env['MARATHON_APP_ID'].split('/')[-2:0:-1])
|
||||
@ -1,282 +1,217 @@
|
||||
#Public names
|
||||
__all__=['Discovery','DiscoveryMesos','DiscoveryMarathon']
|
||||
import dns.resolver
|
||||
import dns.query
|
||||
from dns.exception import DNSException
|
||||
from .logger import Logger
|
||||
import os
|
||||
import sys
|
||||
import requests
|
||||
from dns.exception import DNSException
|
||||
from .config import *
|
||||
from .logger import *
|
||||
|
||||
# Default config for Discovery class
|
||||
_config={
|
||||
'default_discovery':'mesos_dns' # Default discovery system
|
||||
}
|
||||
|
||||
# Discoveries objects
|
||||
_discoveries={}
|
||||
|
||||
#Logger
|
||||
logger=Logger()
|
||||
# Discovery object
|
||||
_discovery_singleton=None
|
||||
|
||||
class DiscoveryTemplate:
|
||||
# Default config values for discovery template
|
||||
_config={}
|
||||
_defconfig={'enabled':False}
|
||||
|
||||
def __init__(self,conf):
|
||||
for key in self._defconfig.keys():
|
||||
if key not in self._config.keys():
|
||||
self._config[key]=self._defconfig[key]
|
||||
self.set_config(conf)
|
||||
|
||||
def set_config(self,conf):
|
||||
pass
|
||||
def __init__(self):
|
||||
self._config=Config()
|
||||
self._logger=Logger()
|
||||
|
||||
def enabled(self):
|
||||
return self._config['enabled']
|
||||
return self._config[self._config_section].get('enabled',False)
|
||||
|
||||
def update_data(self):
|
||||
pass
|
||||
|
||||
def get_group(self,service, app):
|
||||
# Check group in app conf
|
||||
if 'group' in service:
|
||||
return service['group']
|
||||
# Do DNS queries
|
||||
# Return array:
|
||||
# ["10.10.10.1", "10.10.10.2"]
|
||||
def do_query_a(self,fqdn):
|
||||
servers = []
|
||||
try:
|
||||
resolver = dns.resolver.Resolver()
|
||||
for a_rdata in resolver.query(fqdn, 'A'):
|
||||
servers.append(a_rdata.address)
|
||||
except DNSException as e:
|
||||
self._logger.error("Could not resolve ",fqdn)
|
||||
return servers
|
||||
|
||||
# Check environment variable
|
||||
elif app['env'].get('SUROK_DISCOVERY_GROUP'):
|
||||
return app['env']['SUROK_DISCOVERY_GROUP']
|
||||
|
||||
# Check marathon environment variable
|
||||
elif app['env'].get('MARATHON_APP_ID'):
|
||||
return ".".join(app['env']['MARATHON_APP_ID'].split('/')[-2:0:-1])
|
||||
|
||||
else:
|
||||
logger.error('Group is not defined in config, SUROK_DISCOVERY_GROUP and MARATHON_APP_ID')
|
||||
logger.error('Not in Mesos launch?')
|
||||
sys.exit(2)
|
||||
# Do DNS queries
|
||||
# Return array:
|
||||
# [{"name": "f.q.d.n", "port": 8876, "ip": ["10.10.10.1", "10.10.10.2"]}]
|
||||
def do_query_srv(self,fqdn):
|
||||
servers = []
|
||||
try:
|
||||
resolver = dns.resolver.Resolver()
|
||||
resolver.lifetime = 1
|
||||
resolver.timeout = 1
|
||||
query = resolver.query(fqdn, 'SRV')
|
||||
for rdata in query:
|
||||
info = str(rdata).split()
|
||||
servers.append({'name': info[3][:-1], 'port': info[2]})
|
||||
except DNSException as e:
|
||||
self._logger.warning("Could not resolve ",fqdn)
|
||||
return servers
|
||||
|
||||
|
||||
class Discovery:
|
||||
def __init__(self,*conf):
|
||||
for __conf in conf:
|
||||
self.set_config(__conf)
|
||||
_discoveries={}
|
||||
def __new__(cls):
|
||||
global _discovery_singleton
|
||||
if _discovery_singleton is None:
|
||||
_discovery_singleton=super(Discovery, cls).__new__(cls)
|
||||
return _discovery_singleton
|
||||
|
||||
def set_config(self,conf):
|
||||
global _discoveries
|
||||
#Get discoveries objects
|
||||
if not _discoveries.get('mesos_dns'):
|
||||
_discoveries['mesos_dns']=DiscoveryMesos(conf)
|
||||
else:
|
||||
_discoveries['mesos_dns'].set_config(conf)
|
||||
def __init__(self):
|
||||
self._config=Config()
|
||||
self._logger=Logger()
|
||||
if not self._discoveries.get('mesos_dns'):
|
||||
self._discoveries['mesos_dns']=DiscoveryMesos()
|
||||
if not self._discoveries.get('marathon_api'):
|
||||
self._discoveries['marathon_api']=DiscoveryMarathon()
|
||||
|
||||
if not _discoveries.get('marathon_api'):
|
||||
_discoveries['marathon_api']=DiscoveryMarathon(conf)
|
||||
else:
|
||||
_discoveries['marathon_api'].set_config(conf)
|
||||
|
||||
if not _discoveries.get('consul_dns'):
|
||||
_discoveries['consul_dns']=DiscoveryConsul(conf)
|
||||
else:
|
||||
_discoveries['consul_dns'].set_config(conf)
|
||||
|
||||
global _config
|
||||
if conf.get('default_discovery'):
|
||||
discovery=conf.get('default_discovery')
|
||||
if discovery in list(_discoveries.keys()):
|
||||
_config['default_discovery']=discovery
|
||||
else:
|
||||
logger.error('Default discovery "'+discovery+'" is not present')
|
||||
logger.debug('Conf=',conf)
|
||||
def keys(self):
|
||||
return self._discoveries.keys()
|
||||
|
||||
def resolve(self,app):
|
||||
__discovery=_config.get('default_discovery')
|
||||
if app.get('discovery'):
|
||||
discovery=app.get('discovery')
|
||||
if discovery in list(_discoveries.keys()):
|
||||
__discovery=discovery
|
||||
else:
|
||||
logger.warning('Discovery "'+discovery+'" is not present')
|
||||
logger.debug('App=',app)
|
||||
return {}
|
||||
if _discoveries[__discovery].enabled():
|
||||
return _discoveries[__discovery].resolve(app)
|
||||
discovery=app.get('discovery',self._config.get('default_discovery'))
|
||||
if discovery not in self.keys():
|
||||
self._logger.warning('Discovery "',discovery,'" is not present')
|
||||
return {}
|
||||
if self._discoveries[discovery].enabled():
|
||||
return self.compatible(self._discoveries[discovery].resolve(app))
|
||||
else:
|
||||
logger.error('Discovery "'+__discovery+'" is disabled')
|
||||
self._logger.error('Discovery "',discovery,'" is disabled')
|
||||
return {}
|
||||
|
||||
def update_data(self):
|
||||
global _discoveries
|
||||
for d in list(_discoveries.keys()):
|
||||
if _discoveries[d].enabled():
|
||||
_discoveries[d].update_data()
|
||||
self._config.update_apps()
|
||||
for d in self.keys():
|
||||
if self._discoveries[d].enabled():
|
||||
self._discoveries[d].update_data()
|
||||
|
||||
def compatible(self,hosts):
|
||||
compatible_hosts={}
|
||||
if self._config.get('version') == '0.7':
|
||||
for service in hosts.keys():
|
||||
for host in hosts[service]:
|
||||
ports=host.get('tcp',[])
|
||||
if type(ports).__name__ == 'list':
|
||||
compatible_hosts[service]=[]
|
||||
for port in ports:
|
||||
compatible_hosts[service].append({'name':host['name'],
|
||||
'ip':host['ip'],
|
||||
'port':str(port)})
|
||||
else:
|
||||
compatible_hosts[service]={}
|
||||
for port in ports.keys():
|
||||
compatible_host=compatible_hosts[service].setdefault(port,[])
|
||||
compatible_host.append({'name':host['name'],
|
||||
'ip':host['ip'],
|
||||
'port':ports[port]})
|
||||
|
||||
return compatible_hosts
|
||||
return hosts
|
||||
|
||||
|
||||
class DiscoveryMesos(DiscoveryTemplate):
|
||||
_config={
|
||||
'domain':'marathon.mesos' # Default domain
|
||||
}
|
||||
|
||||
def set_config(self,conf):
|
||||
# For old version config
|
||||
if conf.get('domain'):
|
||||
self._config['domain']=conf.get('domain')
|
||||
self._config['enabled']=True
|
||||
# For current version config
|
||||
if conf.get('mesos'):
|
||||
_conf=conf['mesos']
|
||||
for p in ['domain','enabled']:
|
||||
if _conf.get(p):
|
||||
self._config[p]=_conf.get(p)
|
||||
|
||||
_config_section='mesos'
|
||||
def resolve(self,app):
|
||||
hosts = {}
|
||||
services = app['services']
|
||||
domain = self._config['domain']
|
||||
services = app.get('services')
|
||||
domain = self._config[self._config_section].get('domain')
|
||||
for service in services:
|
||||
group = self.get_group(service, app)
|
||||
group = service.get('group',app.get('group'))
|
||||
if group is None:
|
||||
self._logger.error('Group for service "{}" of config "{}" not found'.format(service['name'],app.get('conf_name')))
|
||||
continue
|
||||
ports = service.get('ports')
|
||||
name = service['name']
|
||||
hosts[name] = {}
|
||||
serv = hosts[name]
|
||||
if ports is not None:
|
||||
hosts[name] = {}
|
||||
serv = hosts[name]
|
||||
for prot in ['tcp','udp']:
|
||||
self._logger.debug('group=',group,' ports=',ports,' name=',name,' serv=',serv)
|
||||
for prot in ['tcp','udp']:
|
||||
if ports is not None:
|
||||
for port_name in ports:
|
||||
for d in do_query('_'+port_name+'._'+name+'.'+group+'._'+prot+'.'+domain):
|
||||
for d in self.do_query_srv('_'+port_name+'._'+name+'.'+group+'._'+prot+'.'+domain):
|
||||
hostname=d['name']
|
||||
if serv.get(hostname) is None:
|
||||
serv[hostname]={"name":hostname,"ip":d['ip']}
|
||||
if serv[hostname].get(prot) is None:
|
||||
serv[hostname][prot]={}
|
||||
serv[hostname][prot][port_name]=d['port']
|
||||
hosts[name]=list(hosts[name].values())
|
||||
else:
|
||||
hosts[name]=do_query('_'+name+'.'+group+'._tcp.'+domain)
|
||||
|
||||
serv.setdefault(hostname,{'name':hostname,
|
||||
'ip':self.do_query_a(hostname)})
|
||||
serv[hostname].setdefault(prot,{})
|
||||
serv[hostname][prot][port_name]=d['port']
|
||||
else:
|
||||
for d in self.do_query_srv('_'+name+'.'+group+'._'+prot+'.'+domain):
|
||||
hostname=d['name']
|
||||
if serv.get(hostname) is None:
|
||||
serv[hostname]={'name':hostname,
|
||||
'ip':self.do_query_a(hostname)}
|
||||
if serv[hostname].get(prot) is None:
|
||||
serv[hostname][prot]=[]
|
||||
serv[hostname][prot].extend([d['port']])
|
||||
hosts[name]=list(serv.values())
|
||||
return hosts
|
||||
|
||||
|
||||
class DiscoveryMarathon(DiscoveryTemplate):
|
||||
_config={
|
||||
'host':'http://marathon.mesos:8080',
|
||||
'force':True
|
||||
}
|
||||
__tasks = []
|
||||
__ports = {}
|
||||
def set_config(self,conf):
|
||||
# For current version config
|
||||
if conf.get('marathon'):
|
||||
_conf=conf['marathon']
|
||||
for p in ['host','enabled','force']:
|
||||
if _conf.get(p):
|
||||
self._config[p]=_conf.get(p)
|
||||
_config_section='marathon'
|
||||
_tasks = []
|
||||
_ports = {}
|
||||
|
||||
def update_data(self):
|
||||
hostname=self._config[self._config_section].get('host')
|
||||
try:
|
||||
apps = requests.get(self._config['host']+'/v2/apps').json()['apps']
|
||||
ports = {}
|
||||
for app in apps:
|
||||
for app in requests.get(hostname+'/v2/apps').json()['apps']:
|
||||
ports[app['id']] = {}
|
||||
if app.get('container') is not None and app['container']['type'] == 'DOCKER':
|
||||
if app.get('container') is not None and app['container'].get('type') == 'DOCKER':
|
||||
ports[app['id']] = app['container']['docker'].get('portMappings',[])
|
||||
self.__ports=ports
|
||||
self._ports=ports
|
||||
except:
|
||||
logger.warning('Apps ('+self._config['host']+'/v2/apps) request from Marathon API is failed')
|
||||
self._logger.warning('Apps (',hostname,'/v2/apps) request from Marathon API is failed')
|
||||
pass
|
||||
try:
|
||||
self.__tasks = requests.get(self._config['host']+'/v2/tasks').json()['tasks']
|
||||
self._tasks = requests.get(hostname + '/v2/tasks').json()['tasks']
|
||||
except:
|
||||
logger.warning('Tasks ('+self._config['host']+'/v2/tasks) request from Marathon API is failed')
|
||||
self._logger.warning('Tasks (',hostname,'/v2/tasks) request from Marathon API is failed')
|
||||
pass
|
||||
|
||||
def _test_mask(self, mask, value):
|
||||
return (mask.endswith('*') and value.startswith(mask[:-1])) or mask == value
|
||||
|
||||
def resolve(self, app):
|
||||
hosts={}
|
||||
serv_conf = app['services']
|
||||
if not serv_conf:
|
||||
serv_conf = [{'name':'*','ports':['*']}]
|
||||
for serv in serv_conf:
|
||||
# Convert xxx.yyy.zzz to /zzz/yyy/xxx/ format
|
||||
group = '/'.join(['']+self.get_group(serv, app).split('.')[::-1]+[''])
|
||||
mask = group+serv['name']
|
||||
for task in self.__tasks:
|
||||
if (mask.endswith('*') and task['appId'].startswith(mask[:-1])) or task['appId'] == mask:
|
||||
name='.'.join(task['appId'][len(group):].split('/')[::-1])
|
||||
if 'ports' in serv:
|
||||
hosts[name]={}
|
||||
for port in self.__ports[task['appId']]:
|
||||
for pp in serv['ports']:
|
||||
if (pp.endswith('*') and port['name'].startswith(pp[:-1])) or port['name'] == pp:
|
||||
if hosts[name].get(task['host']) is None:
|
||||
hosts[name][task['host']]={'name':task['host'],
|
||||
'ip':do_query_a(task['host'])}
|
||||
if hosts[name][task['host']].get(port['protocol']) is None:
|
||||
hosts[name][task['host']][port['protocol']]={}
|
||||
hosts[name][task['host']][port['protocol']][port['name']]=task['ports'][task['servicePorts'].index(port['servicePort'])]
|
||||
hosts[name]=list(hosts[name].values())
|
||||
else:
|
||||
hosts[name]=[]
|
||||
for port in self.__ports[task['appId']]:
|
||||
hosts[name].append({'name':task['host'],
|
||||
'port':task['ports'][task['servicePorts'].index(port['servicePort'])],
|
||||
'ip':do_query_a(task['host'])})
|
||||
|
||||
return hosts
|
||||
|
||||
|
||||
class DiscoveryConsul(DiscoveryTemplate):
|
||||
_config={
|
||||
'enabled':False,
|
||||
'domain':None
|
||||
}
|
||||
def set_config(self,conf):
|
||||
# For current version config
|
||||
if conf.get('consul'):
|
||||
_conf=conf['consul']
|
||||
for p in ['domain','enabled']:
|
||||
if _conf.get(p):
|
||||
self._config[p]=_conf.get(p)
|
||||
|
||||
def resolve(self,app):
|
||||
hosts = {}
|
||||
services = app['services']
|
||||
domain = self._config['domain']
|
||||
services = app.get('services')
|
||||
if not services:
|
||||
services = [{'name':'*','ports':['*']}]
|
||||
for service in services:
|
||||
name = service['name']
|
||||
hosts[name]=do_query('_'+name+'._tcp.'+domain)
|
||||
# Convert xxx.yyy.zzz to /zzz/yyy/xxx/ format
|
||||
group=service.get('group',app.get('group'))
|
||||
if group is None:
|
||||
self._logger.error('Group for service "{}" of config "{}" not found'.format(service['name'],app.get('conf_name')))
|
||||
continue
|
||||
group = '/'+'/'.join(group.split('.')[::-1])+'/'
|
||||
service_mask = group + service['name']
|
||||
for task in self._tasks:
|
||||
if self._test_mask(service_mask,task['appId']):
|
||||
name='.'.join(task['appId'][len(group):].split('/')[::-1])
|
||||
hosts[name]={}
|
||||
serv = hosts[name]
|
||||
hostname=task['host']
|
||||
for task_port in self._ports[task['appId']]:
|
||||
prot=task_port['protocol']
|
||||
port_name=task_port['name']
|
||||
port=task['ports'][task['servicePorts'].index(task_port['servicePort'])]
|
||||
if 'ports' in service:
|
||||
for port_mask in service['ports']:
|
||||
if self._test_mask(port_mask,port_name):
|
||||
serv.setdefault(hostname,{'name':hostname,
|
||||
'ip':self.do_query_a(hostname)})
|
||||
serv[hostname].setdefault(prot,{})
|
||||
serv[hostname][prot][port_name]=port
|
||||
else:
|
||||
serv.setdefault(hostname,{'name':hostname,
|
||||
'ip':self.do_query_a(hostname)})
|
||||
serv[hostname].setdefault(prot,[])
|
||||
serv[hostname][prot].extend([port])
|
||||
hosts[name]=list(serv.values())
|
||||
return hosts
|
||||
|
||||
|
||||
# Do DNS queries
|
||||
# Return array:
|
||||
# ["10.10.10.1", "10.10.10.2"]
|
||||
def do_query_a(fqdn):
|
||||
servers = []
|
||||
try:
|
||||
resolver = dns.resolver.Resolver()
|
||||
for a_rdata in resolver.query(fqdn, 'A'):
|
||||
servers.append(a_rdata.address)
|
||||
except DNSException as e:
|
||||
logger.error("Could not resolve "+fqdn)
|
||||
|
||||
return servers
|
||||
|
||||
|
||||
# Do DNS queries
|
||||
# Return array:
|
||||
# [{"name": "f.q.d.n", "port": 8876, "ip": ["10.10.10.1", "10.10.10.2"]}]
|
||||
def do_query(fqdn):
|
||||
servers = []
|
||||
try:
|
||||
resolver = dns.resolver.Resolver()
|
||||
resolver.lifetime = 1
|
||||
resolver.timeout = 1
|
||||
query = resolver.query(fqdn, 'SRV')
|
||||
for rdata in query:
|
||||
info = str(rdata).split()
|
||||
name = info[3][:-1]
|
||||
port = info[2]
|
||||
servers.append({'name': name, 'port': port, 'ip': do_query_a(name)})
|
||||
except DNSException as e:
|
||||
logger.error("Could not resolve " + fqdn)
|
||||
|
||||
return servers
|
||||
|
||||
@ -1,51 +1,86 @@
|
||||
# Public names
|
||||
__all__ = ['Logger']
|
||||
|
||||
import sys
|
||||
import json
|
||||
from time import time
|
||||
_loglevel='info'
|
||||
msg_level={'debug':'DEBUG',
|
||||
'info':'INFO',
|
||||
'warning':'WARNING',
|
||||
'error':'ERROR'}
|
||||
import time
|
||||
|
||||
# Logger singleton link
|
||||
_logger_singleton = None
|
||||
|
||||
'''
|
||||
Public Logger oblect
|
||||
==================================================
|
||||
.set_level(level) - set level messages
|
||||
level - values 'debug', 'info', 'warning', 'error'
|
||||
* error - write error message
|
||||
* warning - write warning and error message
|
||||
* info - write info, warning and error message
|
||||
* debug - write all message
|
||||
.get_level() - get level messages
|
||||
.error(str) - write error message
|
||||
.warning(str) - write warning message
|
||||
.info(str) - write info message
|
||||
.debug(str)- write error message
|
||||
'''
|
||||
|
||||
|
||||
class Logger:
|
||||
def __init__(self,*args):
|
||||
_loglevel = 'info'
|
||||
_msg_level = {
|
||||
'debug': 'DEBUG',
|
||||
'info': 'INFO',
|
||||
'warning': 'WARNING',
|
||||
'error': 'ERROR'
|
||||
}
|
||||
|
||||
def __new__(cls, *args):
|
||||
global _logger_singleton
|
||||
if _logger_singleton is None:
|
||||
_logger_singleton = super(Logger, cls).__new__(cls)
|
||||
return _logger_singleton
|
||||
|
||||
def __init__(self, *args):
|
||||
if args:
|
||||
self.set_level(args[0])
|
||||
|
||||
def set_level(self,level):
|
||||
if level in ['debug','info','warning','error']:
|
||||
global _loglevel
|
||||
_loglevel=level
|
||||
def set_level(self, level):
|
||||
if level in ['debug', 'info', 'warning', 'error']:
|
||||
self._loglevel = level
|
||||
return True
|
||||
else:
|
||||
self.warning('Log level "', level, '" not valid')
|
||||
return False
|
||||
|
||||
def get_level(self):
|
||||
return _loglevel
|
||||
return self._loglevel
|
||||
|
||||
def __make_message(self,message):
|
||||
r=[]
|
||||
l=self.get_level()
|
||||
def _make_message(self, level, message):
|
||||
r = []
|
||||
for m in message:
|
||||
if type(m).__name__=='str':
|
||||
if type(m).__name__ == 'str':
|
||||
r.append(m)
|
||||
else:
|
||||
r.append(json.dumps(m,sort_keys=True,indent=2))
|
||||
return '[' + str(time()) + '] ' + msg_level[l] + ': ' + ''.join(r) + "\n"
|
||||
r.append(json.dumps(m, sort_keys=True, indent=2))
|
||||
return '[' + str(time.time()) + '] ' + self._msg_level[level] + ': ' + ''.join(r) + "\n"
|
||||
|
||||
def debug(self,*message):
|
||||
def debug(self, *message):
|
||||
if self.get_level() in ['debug']:
|
||||
sys.stderr.write(self.__make_message(message))
|
||||
self._log2err(self._make_message('debug',message))
|
||||
|
||||
def info(self,*message):
|
||||
if self.get_level() in ['debug','info']:
|
||||
sys.stdout.write(self.__make_message(message))
|
||||
def info(self, *message):
|
||||
if self.get_level() in ['debug', 'info']:
|
||||
self._log2out(self._make_message('info',message))
|
||||
|
||||
def warning(self,*message):
|
||||
if self.get_level() in ['debug','info','warning']:
|
||||
sys.stderr.write(self.__make_message(message))
|
||||
def warning(self, *message):
|
||||
if self.get_level() in ['debug', 'info', 'warning']:
|
||||
self._log2out(self._make_message('warning',message))
|
||||
|
||||
def error(self,*message):
|
||||
sys.stderr.write(self.__make_message(message))
|
||||
def error(self, *message):
|
||||
self._log2err(self._make_message('error',message))
|
||||
|
||||
def testing(self,level,message):
|
||||
self.set_level(level)
|
||||
return self.__make_message(message)
|
||||
def _log2err(self,out):
|
||||
sys.stderr.write(out)
|
||||
|
||||
def _log2out(self,out):
|
||||
sys.stdout.write(out)
|
||||
|
||||
@ -3,6 +3,7 @@ import sys
|
||||
import requests
|
||||
from .discovery import Discovery
|
||||
from .logger import Logger
|
||||
|
||||
logger=Logger()
|
||||
|
||||
# Get old configuration
|
||||
@ -91,7 +92,8 @@ def discovery_memcached(conf):
|
||||
# !!! NEED REFACTORING !!!
|
||||
def reload_conf(service_conf, app_conf, conf, app_hosts):
|
||||
# Check marathon enabled in configuration
|
||||
if conf['marathon'].get('restart',False):
|
||||
if (conf.get('version','0.7')=='0.8' and conf['marathon'].get('restart',False)) or (
|
||||
conf.get('version','0.7')=='0.7' and conf['marathon'].get('enable',False)):
|
||||
if get_old(app_conf['conf_name'], service_conf) != 1:
|
||||
restart_self_in_marathon(conf['marathon'])
|
||||
|
||||
|
||||
@ -25,10 +25,12 @@ install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok/logger.py %{buildr
|
||||
install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok/system.py %{buildroot}/opt/surok/surok
|
||||
install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok/discovery.py %{buildroot}/opt/surok/surok
|
||||
install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok/templates.py %{buildroot}/opt/surok/surok
|
||||
install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok/config.py %{buildroot}/opt/surok/surok
|
||||
install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/surok.py %{buildroot}/opt/surok
|
||||
mkdir -p %{buildroot}/etc/surok/{conf,conf.d,templates}
|
||||
install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/conf/surok.json %{buildroot}/etc/surok/conf
|
||||
install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/conf.d/selfcheck.json %{buildroot}/etc/surok/conf.d
|
||||
install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/conf.d/self_check.json %{buildroot}/etc/surok/conf.d
|
||||
install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/conf.d/marathon_check.json %{buildroot}/etc/surok/conf.d
|
||||
install -p -m 644 /root/rpmbuild/BUILD/surok-%{version}/templates/selfcheck.jj2 %{buildroot}/etc/surok/templates
|
||||
|
||||
|
||||
@ -43,7 +45,9 @@ rm -rf $RPM_BUILD_ROOT
|
||||
/opt/surok/surok/discovery.py
|
||||
/opt/surok/surok/system.py
|
||||
/opt/surok/surok/templates.py
|
||||
/etc/surok/conf.d/selfcheck.json
|
||||
/opt/surok/surok/config.py
|
||||
/etc/surok/conf.d/self_check.json
|
||||
/etc/surok/conf.d/marathon_check.json
|
||||
/etc/surok/templates/selfcheck.jj2
|
||||
%defattr(-,root,root,-)
|
||||
%doc
|
||||
|
||||
Loading…
Reference in New Issue
Block a user