sum7
/
ejabberd-tools
2
1
Fork 0
Code Issues 1 Pull Requests 1 Projects 1 Releases Wiki Activity

Merge branch 'add-ci' into 'master' 

Add ci

See merge request sum7/ejabberd-tools!6
This commit is contained in:
genofire 2020-06-29 19:34:48 +02:00
parent 1256ea7de8 044e0334d2
commit 091b6e525a
11 changed files with 205 additions and 106 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
.gitlab-ci.yml Normal file
View File

@ -0,0 +1,38 @@
image: "python:3.7"
variables:
# force pip cache dir
PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
cache:
paths:
- .cache/pip # pip caching directory
# setup environemnt
before_script:
- python --version
- pip install -r requirements.txt
- pip install flake8 black
stages:
- syntax
- code consistency
- flake8
syntax:
stage: syntax
script:
# flake8 check only breaking errors ie syntax errors
- flake8 --select=E9,F63,F7,F82 --show-source
black:
stage: code consistency
script:
# code consistency
- black . --check
pep8:
stage: flake8
script:
# full flake8 test
- flake8 --show-source

6
README.md
View File

@ -48,7 +48,7 @@ listen:
acl:
api:
user:
user:
- "api_user@magicbroccoli.de"
loopback:
ip:
@ -59,7 +59,7 @@ acl:
api_permissions:
"api access":
from:
- mod_http_api
- mod_http_api
who:
access:
allow:
@ -81,7 +81,7 @@ configuration file should be located at `/etc/ejabberd-metrics.yml`.
#### systemd service
To properly operate the metrics exporter tools, we created some systemd service templates, to simplify the whole
process. If the `ejabberd-metrics.yml` file is not accessible for the user`nobody:nogroup`, it is required to update the
`User` and `Group` definitions inside the service file.
`User` and `Group` definitions inside the service file.
If a virtualenv is used, it is required to update the `Environment=PATH` to include the `venv/bin` directory created
earlier.

7
api.py
View File

@ -9,6 +9,7 @@ class EjabberdApi:
"""
class to interact with the ejabberd rest/ xmlrpc api
"""
def __init__(self, url, login=None, api: str = "rpc"):
# api variables
self._login = login
@ -25,7 +26,7 @@ class EjabberdApi:
@property
def _auth(self) -> (str, None):
if self._login is not None:
return f"{self._login['user']}@{self._login['server']}", self._login['password']
return f"{self._login['user']}@{self._login['server']}", self._login["password"]
return None
def _rest(self, command: str, data) -> dict:
@ -34,7 +35,7 @@ class EjabberdApi:
self.session.auth = self._auth
# post
r = self.session.post('/'.join([self._url, command]), json=data)
r = self.session.post("/".join([self._url, command]), json=data)
# proceed if response is ok
if r.ok:
@ -52,6 +53,6 @@ class EjabberdApi:
return fn(self._login, data)
return fn(data)
except:
except: # noqa: E722
# this needs to be more specific
return {}

10
calls.py
View File

@ -14,8 +14,8 @@ class EjabberdApiCalls(EjabberdApi):
@property
def nodename(self):
if self._login is not None:
node_str = re.compile('The node \'(.*)\'')
status = self.cmd('status', {})
node_str = re.compile("The node '(.*)'")
status = self.cmd("status", {})
# matches
try:
@ -32,8 +32,8 @@ class EjabberdApiCalls(EjabberdApi):
@property
def verstring(self):
if self._login is not None:
ver_str = re.compile('([1-9][0-9.]+(?![.a-z]))\\b')
status = self.cmd('status', {})
ver_str = re.compile("([1-9][0-9.]+(?![.a-z]))\\b")
status = self.cmd("status", {})
# matches
try:
@ -117,7 +117,7 @@ class EjabberdApiCalls(EjabberdApi):
host = "global"
if vhost is not None:
if self.verstring.major >= 19:
host = '.'.join([muc_host, vhost])
host = ".".join([muc_host, vhost])
else:
host = vhost
result = self.cmd("muc_online_rooms", {"host": host})

40
cleanup.py
View File

@ -20,16 +20,16 @@ class EjabberdCleanup(EjabberdApiCalls):
if self.dry:
logging.warning(f"{user}@{host}: dry delete : {reason}")
else:
self.cmd("unregister", {"host": host, "user": user})
self.cmd("unregister", {"host": host, "user": user})
logging.warning(f"{user}@{host}: deleted - {reason}")
def run_user(self, host, user):
if self.skip_by_roster:
roster = self.cmd("get_roster",{"host": host, "user": user})
roster = self.cmd("get_roster", {"host": host, "user": user})
if len(roster) > 0:
logging.debug(f"{user}@{host}: skipped it has a roster")
return
last = self.cmd("get_last",{"host": host, "user": user})
last = self.cmd("get_last", {"host": host, "user": user})
if self.delete_not_login and last["status"] == "Registered but didn't login":
self.delete_user(host, user, "not login")
return
@ -37,14 +37,16 @@ class EjabberdCleanup(EjabberdApiCalls):
last_stamp = last["timestamp"]
lastdate = None
try:
lastdate = datetime.datetime.strptime(last_stamp,"%Y-%m-%dT%H:%M:%SZ")
except:
lastdate = datetime.datetime.strptime(last_stamp, "%Y-%m-%dT%H:%M:%SZ")
except: # noqa: E722
try:
lastdate = datetime.datetime.strptime(last_stamp,"%Y-%m-%dT%H:%M:%S.%fZ")
except:
lastdate = datetime.datetime.strptime(last_stamp, "%Y-%m-%dT%H:%M:%S.%fZ")
except: # noqa: E722
logging.error(f"{user}@{host}: not able to parse '{last_stamp}'")
return
if lastdate != None and lastdate-datetime.datetime.now() > datetime.timedelta(days=self.offline_since_days):
if lastdate is not None and lastdate - datetime.datetime.now() > datetime.timedelta(
days=self.offline_since_days
):
self.delete_user(host, user, f"last seen @ {lastdate}")
return
@ -53,7 +55,7 @@ class EjabberdCleanup(EjabberdApiCalls):
if host in self.ignore_hosts:
continue
logging.info(f"run cleanup for host: {host}")
for user in self.cmd("registered_users",{"host": host}):
for user in self.cmd("registered_users", {"host": host}):
if user in self.ignore_usernames:
continue
self.run_user(host, user)
@ -64,20 +66,20 @@ if __name__ == "__main__":
# load config
config = Config()
if config.get('debug', default=False):
if config.get("debug", default=False):
logging.getLogger().setLevel(logging.DEBUG)
# credentials and parameters
url = config.get('url', default='http://localhost:5280/api')
login = config.get('login', default=None)
api = config.get('api', default='rest')
url = config.get("url", default="http://localhost:5280/api")
login = config.get("login", default=None)
api = config.get("api", default="rest")
# init handler
cleaner = EjabberdCleanup(url, login, api)
cleaner.dry = config.get('cleaner_dry',default=False)
cleaner.ignore_hosts = config.get('cleaner_ignore_hosts',default=[])
cleaner.ignore_usernames = config.get('cleaner_ignore_usernames',default=[])
cleaner.skip_by_roster = config.get('cleaner_skip_by_roster',default=True)
cleaner.delete_not_login = config.get('cleaner_delete_not_login',default=True)
cleaner.offline_since_days = config.get('cleaner_offline_since_days', default=None)
cleaner.dry = config.get("cleaner_dry", default=False)
cleaner.ignore_hosts = config.get("cleaner_ignore_hosts", default=[])
cleaner.ignore_usernames = config.get("cleaner_ignore_usernames", default=[])
cleaner.skip_by_roster = config.get("cleaner_skip_by_roster", default=True)
cleaner.delete_not_login = config.get("cleaner_delete_not_login", default=True)
cleaner.offline_since_days = config.get("cleaner_offline_since_days", default=None)
cleaner.run()

10
config.py
View File

@ -13,11 +13,11 @@ class Config:
def __init__(self):
# class variables
self.content = None
self.conf_file = Path('/etc/ejabberd-metrics.yml')
self.conf_file = Path("/etc/ejabberd-metrics.yml")
# dev config overwrite
if environ.get('ejabberd_metrics_dev'):
self.conf_file = Path('config.yml')
if environ.get("ejabberd_metrics_dev"):
self.conf_file = Path("config.yml")
# read config file
self._read()
@ -27,9 +27,9 @@ class Config:
self._check()
# open file as an iostream
with open(self.conf_file, 'r', encoding='utf-8') as f:
with open(self.conf_file, "r", encoding="utf-8") as f:
try:
self.content = YAML(typ='safe').load(f)
self.content = YAML(typ="safe").load(f)
# catch json decoding errors
except (ParserError, ScannerError) as err:

57
influx.py
View File

@ -21,8 +21,8 @@ class Influx:
@staticmethod
def _rmspace(key: str = None, value: (str, int) = None):
try:
key = key.replace(' ', '\ ')
value = value.replace(' ', '\ ')
key = key.replace(" ", "\ ") # noqa: W605
value = value.replace(" ", "\ ") # noqa: W605
except (TypeError, AttributeError):
pass
@ -36,10 +36,10 @@ class Influx:
# create tag_key=tag_value pairs for all elements and append them to name
for k, v in tags.items():
output += ',{}={}'.format(*self._rmspace(k, v))
output += ",{}={}".format(*self._rmspace(k, v))
# append key=value to name
output += ' {}={}i {}'.format(*self._rmspace(key, value), ts)
output += " {}={}i {}".format(*self._rmspace(key, value), ts)
return output
def write_metrics(self):
@ -47,53 +47,60 @@ class Influx:
# global values
cur_ts = self._timestamp()
data.append(f'ejabberd s2s_in={self._metrics.get_s2s_in()}i {cur_ts}')
data.append(f'ejabberd s2s_out={self._metrics.get_s2s_out()}i {cur_ts}')
data.append(f'ejabberd uptime={self._metrics.get_uptime()}i {cur_ts}')
data.append(f'ejabberd processes={self._metrics.get_processes()}i {cur_ts}')
data.append(f"ejabberd s2s_in={self._metrics.get_s2s_in()}i {cur_ts}")
data.append(f"ejabberd s2s_out={self._metrics.get_s2s_out()}i {cur_ts}")
data.append(f"ejabberd uptime={self._metrics.get_uptime()}i {cur_ts}")
data.append(f"ejabberd processes={self._metrics.get_processes()}i {cur_ts}")
# vhost values
for vhost in self._metrics.get_vhosts():
cur_ts = self._timestamp()
data.append(f'ejabberd,vhost={vhost} registered={self._metrics.get_registered(vhost)}i {cur_ts}')
data.append(f'ejabberd,vhost={vhost} muc={self._metrics.get_muc(vhost)}i {cur_ts}')
data.append(f"ejabberd,vhost={vhost} registered={self._metrics.get_registered(vhost)}i {cur_ts}")
data.append(f"ejabberd,vhost={vhost} muc={self._metrics.get_muc(vhost)}i {cur_ts}")
# vhost statistics on their respective node
for node in self._metrics.get_nodes():
cur_ts = self._timestamp()
for k, v in self._metrics.get_online_by_status(node=node, vhost=vhost).items():
data.append(self._parse('ejabberd_online_status', k, v, cur_ts, {'node': node, 'vhost': vhost}))
data.append(self._parse("ejabberd_online_status", k, v, cur_ts, {"node": node, "vhost": vhost}))
for k, v in self._metrics.get_online_by_client(node=node, vhost=vhost).items():
data.append(self._parse('ejabberd_online_client', k, v, cur_ts, {'node': node, 'vhost': vhost}))
data.append(self._parse("ejabberd_online_client", k, v, cur_ts, {"node": node, "vhost": vhost}))
for k, v in self._metrics.get_online_by_ipversion(node=node, vhost=vhost).items():
data.append(self._parse('ejabberd_online_ipversion', k, v, cur_ts, {'node': node, 'vhost': vhost}))
data.append(self._parse("ejabberd_online_ipversion", k, v, cur_ts, {"node": node, "vhost": vhost}))
for k, v in self._metrics.get_online_by_connection(node=node, vhost=vhost).items():
data.append(self._parse('ejabberd_online_connection', k, v, cur_ts, {'node': node, 'vhost': vhost}))
data.append(self._parse("ejabberd_online_connection", k, v, cur_ts, {"node": node, "vhost": vhost}))
for cl, ipv in self._metrics.get_online_client_by_ipversion(node=node, vhost=vhost).items():
for k, v in ipv.items():
data.append(self._parse('ejabberd_online_client_ipversion', k, v, cur_ts,
{'vhost': vhost, 'node': node, 'ipversion': k, 'client': cl}))
data.append(
self._parse(
"ejabberd_online_client_ipversion",
k,
v,
cur_ts,
{"vhost": vhost, "node": node, "ipversion": k, "client": cl},
)
)
# write output to database
self.client.write_points(data, time_precision='ms', batch_size=10000, protocol='line')
self.client.write_points(data, time_precision="ms", batch_size=10000, protocol="line")
if __name__ == '__main__':
if __name__ == "__main__":
# load config
config = Config()
if config.get('debug', default=False):
if config.get("debug", default=False):
logging.getLogger().setLevel(logging.DEBUG)
# credentials and parameters
url = config.get('url', default='http://localhost:5280/api')
login = config.get('login', default=None)
api = config.get('api', default='rest')
url = config.get("url", default="http://localhost:5280/api")
login = config.get("login", default=None)
api = config.get("api", default="rest")
# config influxdb
influx_host = config.get('influxdb_host', default='localhost')
influx_port = config.get('influxdb_port', default=8086)
influx_dbname = config.get('influxdb_db', default='ejabberd')
influx_host = config.get("influxdb_host", default="localhost")
influx_port = config.get("influxdb_port", default=8086)
influx_dbname = config.get("influxdb_db", default="ejabberd")
# init handler
metrics = EjabberdMetrics(url, login, api)

29
metrics.py
View File

@ -12,7 +12,8 @@ class EjabberdMetrics(EjabberdApiCalls):
"""
class to fetch metrics per xmlrpc
"""
def __init__(self, url, login=None, api="rpc", muc_host: str = 'conference'):
def __init__(self, url, login=None, api="rpc", muc_host: str = "conference"):
# init ejabberd api
super().__init__(url, login, api)
@ -32,7 +33,7 @@ class EjabberdMetrics(EjabberdApiCalls):
"profanity": [],
"Xabber": ["xabber", "xabber-android"],
"ChatSecure": ["chatsecure"],
"Monal": []
"Monal": [],
}
for client, names in clientmap.items():
@ -110,7 +111,7 @@ class EjabberdMetrics(EjabberdApiCalls):
return self.get_online_by("node", vhost=vhost)
def get_online_by_vhost(self, node=None):
return self.get_online_by("jid", parse=lambda jid: jid[jid.find("@")+1:jid.find("/")], node=node)
return self.get_online_by("jid", parse=lambda jid: jid[jid.find("@") + 1 : jid.find("/")], node=node)
def get_online_by_status(self, vhost=None, node=None):
return self.get_online_by("status", vhost=vhost, node=node)
@ -187,11 +188,11 @@ class EjabberdMetrics(EjabberdApiCalls):
return self._uptime
def get_processes(self):
if not hasattr(self, '_processes'):
if not hasattr(self, "_processes"):
self._processes = self.fetch_processes()
return self._processes
def get_vhost_metrics(self, vhost):
def get_vhost_metrics(self, vhost):
data = {
"registered": self.get_registered(vhost),
"muc": self.get_muc(vhost),
@ -199,7 +200,7 @@ class EjabberdMetrics(EjabberdApiCalls):
"online_by_client": self.get_online_by_client(vhost),
"online_by_ipversion": self.get_online_by_ipversion(vhost),
"online_by_connection": self.get_online_by_connection(vhost),
"online_by_node": self.get_online_by_node(vhost)
"online_by_node": self.get_online_by_node(vhost),
}
return data
@ -209,13 +210,13 @@ class EjabberdMetrics(EjabberdApiCalls):
self._nodes = self.fetch_nodes()
return self._nodes
def get_node_metrics(self, node):
def get_node_metrics(self, node):
data = {
"online_by_status": self.get_online_by_status(node=node),
"online_by_client": self.get_online_by_client(node=node),
"online_by_ipversion": self.get_online_by_ipversion(node=node),
"online_by_connection": self.get_online_by_connection(node=node),
"online_by_vhost": self.get_online_by_vhost(node=node)
"online_by_vhost": self.get_online_by_vhost(node=node),
}
return data
@ -225,7 +226,7 @@ class EjabberdMetrics(EjabberdApiCalls):
"registered": self.get_registered(),
"muc": self.get_muc(),
"s2s_in": self.get_s2s_in(),
"s2s_out": self.get_s2s_out(),
"s2s_out": self.get_s2s_out(),
"uptime": self.get_uptime(),
"processes": self.get_processes(),
"online_by_status": self.get_online_by_status(),
@ -233,14 +234,14 @@ class EjabberdMetrics(EjabberdApiCalls):
"online_by_ipversion": self.get_online_by_ipversion(),
"online_by_connection": self.get_online_by_connection(),
"online_by_node": self.get_online_by_node(),
"online_by_vhost": self.get_online_by_vhost()
"online_by_vhost": self.get_online_by_vhost(),
}
vhosts = {}
for host in self.get_vhosts():
vhosts[host] = self.get_vhost_metrics(host)
data["vhosts"] = vhosts
nodes = {}
for node in self.get_nodes():
nodes[node] = self.get_node_metrics(node)
@ -259,9 +260,9 @@ if __name__ == "__main__":
config = Config()
# credentials and parameters
url = config.get('url', default='http://localhost:5280/api')
login = config.get('login', default=None)
api = config.get('api', default='rest')
url = config.get("url", default="http://localhost:5280/api")
login = config.get("login", default=None)
api = config.get("api", default="rest")
# init handler
metrics = EjabberdMetrics(url, login, api)

89
prometheus.py
View File

@ -7,9 +7,7 @@ from socket import AF_INET6
from time import time
from urllib.parse import parse_qs, urlparse
from prometheus_client import (
CollectorRegistry, Gauge, generate_latest, CONTENT_TYPE_LATEST
)
from prometheus_client import CollectorRegistry, Gauge, generate_latest, CONTENT_TYPE_LATEST
from config import Config
from metrics import EjabberdMetrics
@ -17,26 +15,25 @@ from metrics import EjabberdMetrics
class DynamicMetricsHandler(BaseHTTPRequestHandler):
"""HTTP handler that gives metrics from ``core.REGISTRY``."""
def do_GET(self):
params = parse_qs(urlparse(self.path).query)
registry = self.generator(params)
if 'name[]' in params:
registry = registry.restricted_registry(params['name[]'])
if "name[]" in params:
registry = registry.restricted_registry(params["name[]"])
try:
output = generate_latest(registry)
except:
self.send_error(500, 'error generating metric output')
except: # noqa: E722
self.send_error(500, "error generating metric output")
raise
self.send_response(200)
self.send_header('Content-Type', CONTENT_TYPE_LATEST)
self.send_header("Content-Type", CONTENT_TYPE_LATEST)
self.end_headers()
self.wfile.write(output)
@staticmethod
def factory(registry_generator):
DynMetricsHandler = type('MetricsHandler',
(DynamicMetricsHandler, object),
{"generator": registry_generator})
DynMetricsHandler = type("MetricsHandler", (DynamicMetricsHandler, object), {"generator": registry_generator})
return DynMetricsHandler
@ -45,7 +42,7 @@ class Prometheus:
self.ttl = 10
self._last_update = 0
self._metrics = metrics
def handler(self, metrics_handler):
now = time()
if now >= (self._last_update + self.ttl):
@ -54,25 +51,53 @@ class Prometheus:
registry = CollectorRegistry(auto_describe=True)
Gauge('ejabberd_node_s2s_in', 'count of incoming server-to-server connection', registry=registry).set(self._metrics.get_s2s_in())
Gauge('ejabberd_node_s2s_out', 'count of outgoing server-to-server connection', registry=registry).set(self._metrics.get_s2s_out())
Gauge("ejabberd_node_s2s_in", "count of incoming server-to-server connection", registry=registry).set(
self._metrics.get_s2s_in()
)
Gauge("ejabberd_node_s2s_out", "count of outgoing server-to-server connection", registry=registry).set(
self._metrics.get_s2s_out()
)
nodename = self._metrics.nodename
Gauge('ejabberd_node_uptime', 'uptime of ejabberd service', ["node"], registry=registry).labels(nodename).set(self._metrics.get_uptime())
Gauge('ejabberd_node_proccess', 'count of pejabber proccess', ["node"], registry=registry).labels(nodename).set(self._metrics.get_processes())
Gauge("ejabberd_node_uptime", "uptime of ejabberd service", ["node"], registry=registry).labels(nodename).set(
self._metrics.get_uptime()
)
Gauge("ejabberd_node_proccess", "count of pejabber proccess", ["node"], registry=registry).labels(nodename).set(
self._metrics.get_processes()
)
labelnames_vhost = ["vhost"]
registered_vhosts = Gauge('ejabberd_registered_vhosts', 'count of user per vhost', labelnames_vhost, registry=registry)
muc = Gauge('ejabberd_muc', 'count of muc\'s per vhost', labelnames_vhost, registry=registry)
registered_vhosts = Gauge(
"ejabberd_registered_vhosts", "count of user per vhost", labelnames_vhost, registry=registry
)
muc = Gauge("ejabberd_muc", "count of muc's per vhost", labelnames_vhost, registry=registry)
online_vhost_node = Gauge('ejabberd_online_vhost_node', 'count of client connections', ["vhost", "node"], registry=registry)
online_status = Gauge('ejabberd_online_status', 'count of client connections', ["vhost", "node", "status"], registry=registry)
online_connection = Gauge('ejabberd_online_connection', 'count of client connections', ["vhost", "node", "connection"], registry=registry)
online_client = Gauge('ejabberd_online_client', 'count of client software', ["vhost", "node", "client"], registry=registry)
online_ipversion = Gauge('ejabberd_online_ipversion', 'count of client software', ["vhost", "node", "ipversion"], registry=registry)
online_client_ipversion = Gauge('ejabberd_online_client_ipversion', 'count of client software', ["vhost", "node", "client", "ipversion"], registry=registry)
online_vhost_node = Gauge(
"ejabberd_online_vhost_node", "count of client connections", ["vhost", "node"], registry=registry
)
online_status = Gauge(
"ejabberd_online_status", "count of client connections", ["vhost", "node", "status"], registry=registry
)
online_connection = Gauge(
"ejabberd_online_connection",
"count of client connections",
["vhost", "node", "connection"],
registry=registry,
)
online_client = Gauge(
"ejabberd_online_client", "count of client software", ["vhost", "node", "client"], registry=registry
)
online_ipversion = Gauge(
"ejabberd_online_ipversion", "count of client software", ["vhost", "node", "ipversion"], registry=registry
)
online_client_ipversion = Gauge(
"ejabberd_online_client_ipversion",
"count of client software",
["vhost", "node", "client", "ipversion"],
registry=registry,
)
for host in self._metrics.get_vhosts():
labels_vhost = host
@ -108,19 +133,19 @@ class Prometheus:
if __name__ == "__main__":
# load config
config = Config()
if config.get('debug', default=False):
if config.get("debug", default=False):
logging.getLogger().setLevel(logging.DEBUG)
# credentials and parameters
url = config.get('url', default='http://[::1]:5280/api')
login = config.get('login', default=None)
api = config.get('api', default='rest')
url = config.get("url", default="http://[::1]:5280/api")
login = config.get("login", default=None)
api = config.get("api", default="rest")
# config prometheus
prom_addr = config.get('prometheus_address', default='127.0.0.1')
prom_port = config.get('prometheus_port', default=8080)
prom_addr = config.get("prometheus_address", default="127.0.0.1")
prom_port = config.get("prometheus_port", default=8080)
metrics = EjabberdMetrics(url, login, api)
prom = Prometheus(metrics)
prom.ttl = config.get('prometheus_cache_ttl', default=10)
prom.ttl = config.get("prometheus_cache_ttl", default=10)
prom.listen((prom_addr, prom_port))

20
pyproject.toml Normal file
View File

@ -0,0 +1,20 @@
[tool.black]
line-length = 120
target-version = ['py37', 'py38']
include = '\.pyi?$'
exclude = '''
(
/(
\.eggs # exclude a few common directories in the
| \.git # root of the project
| \.hg
| \.mypy_cache
| \.tox
| \.venv
| _build
| buck-out
| build
| dist
)/ # the root of the project
)
'''

5
setup.cfg Normal file
View File

@ -0,0 +1,5 @@
[flake8]
ignore = E501,E203
exclude = .git,__pycache__,.gitlab
max-complexity = 15
max-line-length = 120