Merge branch 'add-ci' into 'master'

Add ci

See merge request sum7/ejabberd-tools!6

.gitlab-ci.yml

@@ -0,0 +1,38 @@
+image: "python:3.7"
+
+variables:
+  # force pip cache dir
+  PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
+
+cache:
+  paths:
+    - .cache/pip # pip caching directory
+
+# setup environemnt
+before_script:
+  - python --version
+  - pip install -r requirements.txt
+  - pip install flake8 black
+
+stages:
+  - syntax
+  - code consistency
+  - flake8
+
+syntax:
+  stage: syntax
+  script:
+    # flake8 check only breaking errors ie syntax errors
+    - flake8 --select=E9,F63,F7,F82 --show-source
+
+black:
+  stage: code consistency
+  script:
+    # code consistency
+    - black . --check
+
+pep8:
+  stage: flake8
+  script:
+    # full flake8 test
+    - flake8 --show-source

api.py

@@ -9,6 +9,7 @@ class EjabberdApi:
     """
     class to interact with the ejabberd rest/ xmlrpc api
     """
+
     def __init__(self, url, login=None, api: str = "rpc"):
         # api variables
         self._login = login
@@ -25,7 +26,7 @@ class EjabberdApi:
     @property
     def _auth(self) -> (str, None):
         if self._login is not None:
-            return f"{self._login['user']}@{self._login['server']}", self._login['password']
+            return f"{self._login['user']}@{self._login['server']}", self._login["password"]
         return None
 
     def _rest(self, command: str, data) -> dict:
@@ -34,7 +35,7 @@ class EjabberdApi:
             self.session.auth = self._auth
 
         # post
-        r = self.session.post('/'.join([self._url, command]), json=data)
+        r = self.session.post("/".join([self._url, command]), json=data)
 
         # proceed if response is ok
         if r.ok:
@@ -52,6 +53,6 @@ class EjabberdApi:
                     return fn(self._login, data)
                 return fn(data)
 
-            except:
+            except:  # noqa: E722
                 # this needs to be more specific
                 return {}

calls.py

@@ -14,8 +14,8 @@ class EjabberdApiCalls(EjabberdApi):
     @property
     def nodename(self):
         if self._login is not None:
-            node_str = re.compile('The node \'(.*)\'')
+            node_str = re.compile("The node '(.*)'")
-            status = self.cmd('status', {})
+            status = self.cmd("status", {})
 
             # matches
             try:
@@ -32,8 +32,8 @@ class EjabberdApiCalls(EjabberdApi):
     @property
     def verstring(self):
         if self._login is not None:
-            ver_str = re.compile('([1-9][0-9.]+(?![.a-z]))\\b')
+            ver_str = re.compile("([1-9][0-9.]+(?![.a-z]))\\b")
-            status = self.cmd('status', {})
+            status = self.cmd("status", {})
 
             # matches
             try:
@@ -117,7 +117,7 @@ class EjabberdApiCalls(EjabberdApi):
         host = "global"
         if vhost is not None:
             if self.verstring.major >= 19:
-                host = '.'.join([muc_host, vhost])
+                host = ".".join([muc_host, vhost])
             else:
                 host = vhost
         result = self.cmd("muc_online_rooms", {"host": host})

cleanup.py

@@ -25,11 +25,11 @@ class EjabberdCleanup(EjabberdApiCalls):
 
     def run_user(self, host, user):
         if self.skip_by_roster:
-            roster = self.cmd("get_roster",{"host":  host, "user": user})
+            roster = self.cmd("get_roster", {"host": host, "user": user})
             if len(roster) > 0:
                 logging.debug(f"{user}@{host}: skipped it has a roster")
                 return
-        last = self.cmd("get_last",{"host":  host, "user": user})
+        last = self.cmd("get_last", {"host": host, "user": user})
         if self.delete_not_login and last["status"] == "Registered but didn't login":
             self.delete_user(host, user, "not login")
             return
@@ -37,14 +37,16 @@ class EjabberdCleanup(EjabberdApiCalls):
             last_stamp = last["timestamp"]
             lastdate = None
             try:
-                lastdate = datetime.datetime.strptime(last_stamp,"%Y-%m-%dT%H:%M:%SZ")
+                lastdate = datetime.datetime.strptime(last_stamp, "%Y-%m-%dT%H:%M:%SZ")
-            except:
+            except:  # noqa: E722
                 try:
-                    lastdate = datetime.datetime.strptime(last_stamp,"%Y-%m-%dT%H:%M:%S.%fZ")
+                    lastdate = datetime.datetime.strptime(last_stamp, "%Y-%m-%dT%H:%M:%S.%fZ")
-                except:
+                except:  # noqa: E722
                     logging.error(f"{user}@{host}: not able to parse '{last_stamp}'")
                     return
-            if lastdate != None and  lastdate-datetime.datetime.now() > datetime.timedelta(days=self.offline_since_days):
+            if lastdate is not None and lastdate - datetime.datetime.now() > datetime.timedelta(
+                days=self.offline_since_days
+            ):
                 self.delete_user(host, user, f"last seen @ {lastdate}")
                 return
 
@@ -53,7 +55,7 @@ class EjabberdCleanup(EjabberdApiCalls):
             if host in self.ignore_hosts:
                 continue
             logging.info(f"run cleanup for host: {host}")
-            for user in self.cmd("registered_users",{"host": host}):
+            for user in self.cmd("registered_users", {"host": host}):
                 if user in self.ignore_usernames:
                     continue
                 self.run_user(host, user)
@@ -64,20 +66,20 @@ if __name__ == "__main__":
 
     # load config
     config = Config()
-    if config.get('debug', default=False):
+    if config.get("debug", default=False):
         logging.getLogger().setLevel(logging.DEBUG)
 
     # credentials and parameters
-    url = config.get('url', default='http://localhost:5280/api')
+    url = config.get("url", default="http://localhost:5280/api")
-    login = config.get('login', default=None)
+    login = config.get("login", default=None)
-    api = config.get('api', default='rest')
+    api = config.get("api", default="rest")
 
     # init handler
     cleaner = EjabberdCleanup(url, login, api)
-    cleaner.dry = config.get('cleaner_dry',default=False)
+    cleaner.dry = config.get("cleaner_dry", default=False)
-    cleaner.ignore_hosts = config.get('cleaner_ignore_hosts',default=[])
+    cleaner.ignore_hosts = config.get("cleaner_ignore_hosts", default=[])
-    cleaner.ignore_usernames = config.get('cleaner_ignore_usernames',default=[])
+    cleaner.ignore_usernames = config.get("cleaner_ignore_usernames", default=[])
-    cleaner.skip_by_roster = config.get('cleaner_skip_by_roster',default=True)
+    cleaner.skip_by_roster = config.get("cleaner_skip_by_roster", default=True)
-    cleaner.delete_not_login = config.get('cleaner_delete_not_login',default=True)
+    cleaner.delete_not_login = config.get("cleaner_delete_not_login", default=True)
-    cleaner.offline_since_days = config.get('cleaner_offline_since_days', default=None)
+    cleaner.offline_since_days = config.get("cleaner_offline_since_days", default=None)
     cleaner.run()

config.py

@@ -13,11 +13,11 @@ class Config:
     def __init__(self):
         # class variables
         self.content = None
-        self.conf_file = Path('/etc/ejabberd-metrics.yml')
+        self.conf_file = Path("/etc/ejabberd-metrics.yml")
 
         # dev config overwrite
-        if environ.get('ejabberd_metrics_dev'):
+        if environ.get("ejabberd_metrics_dev"):
-            self.conf_file = Path('config.yml')
+            self.conf_file = Path("config.yml")
 
         # read config file
         self._read()
@@ -27,9 +27,9 @@ class Config:
         self._check()
 
         # open file as an iostream
-        with open(self.conf_file, 'r', encoding='utf-8') as f:
+        with open(self.conf_file, "r", encoding="utf-8") as f:
             try:
-                self.content = YAML(typ='safe').load(f)
+                self.content = YAML(typ="safe").load(f)
 
             # catch json decoding errors
             except (ParserError, ScannerError) as err:

influx.py

@@ -21,8 +21,8 @@ class Influx:
     @staticmethod
     def _rmspace(key: str = None, value: (str, int) = None):
         try:
-            key = key.replace(' ', '\ ')
+            key = key.replace(" ", "\ ")  # noqa: W605
-            value = value.replace(' ', '\ ')
+            value = value.replace(" ", "\ ")  # noqa: W605
         except (TypeError, AttributeError):
             pass
 
@@ -36,10 +36,10 @@ class Influx:
 
             # create tag_key=tag_value pairs for all elements and append them to name
             for k, v in tags.items():
-                output += ',{}={}'.format(*self._rmspace(k, v))
+                output += ",{}={}".format(*self._rmspace(k, v))
 
         # append key=value to name
-        output += ' {}={}i {}'.format(*self._rmspace(key, value), ts)
+        output += " {}={}i {}".format(*self._rmspace(key, value), ts)
         return output
 
     def write_metrics(self):
@@ -47,53 +47,60 @@ class Influx:
 
         # global values
         cur_ts = self._timestamp()
-        data.append(f'ejabberd s2s_in={self._metrics.get_s2s_in()}i {cur_ts}')
+        data.append(f"ejabberd s2s_in={self._metrics.get_s2s_in()}i {cur_ts}")
-        data.append(f'ejabberd s2s_out={self._metrics.get_s2s_out()}i {cur_ts}')
+        data.append(f"ejabberd s2s_out={self._metrics.get_s2s_out()}i {cur_ts}")
-        data.append(f'ejabberd uptime={self._metrics.get_uptime()}i {cur_ts}')
+        data.append(f"ejabberd uptime={self._metrics.get_uptime()}i {cur_ts}")
-        data.append(f'ejabberd processes={self._metrics.get_processes()}i {cur_ts}')
+        data.append(f"ejabberd processes={self._metrics.get_processes()}i {cur_ts}")
 
         # vhost values
         for vhost in self._metrics.get_vhosts():
             cur_ts = self._timestamp()
-            data.append(f'ejabberd,vhost={vhost} registered={self._metrics.get_registered(vhost)}i {cur_ts}')
+            data.append(f"ejabberd,vhost={vhost} registered={self._metrics.get_registered(vhost)}i {cur_ts}")
-            data.append(f'ejabberd,vhost={vhost} muc={self._metrics.get_muc(vhost)}i {cur_ts}')
+            data.append(f"ejabberd,vhost={vhost} muc={self._metrics.get_muc(vhost)}i {cur_ts}")
 
             # vhost statistics on their respective node
             for node in self._metrics.get_nodes():
                 cur_ts = self._timestamp()
                 for k, v in self._metrics.get_online_by_status(node=node, vhost=vhost).items():
-                    data.append(self._parse('ejabberd_online_status', k, v, cur_ts, {'node': node, 'vhost': vhost}))
+                    data.append(self._parse("ejabberd_online_status", k, v, cur_ts, {"node": node, "vhost": vhost}))
                 for k, v in self._metrics.get_online_by_client(node=node, vhost=vhost).items():
-                    data.append(self._parse('ejabberd_online_client', k, v, cur_ts, {'node': node, 'vhost': vhost}))
+                    data.append(self._parse("ejabberd_online_client", k, v, cur_ts, {"node": node, "vhost": vhost}))
                 for k, v in self._metrics.get_online_by_ipversion(node=node, vhost=vhost).items():
-                    data.append(self._parse('ejabberd_online_ipversion', k, v, cur_ts, {'node': node, 'vhost': vhost}))
+                    data.append(self._parse("ejabberd_online_ipversion", k, v, cur_ts, {"node": node, "vhost": vhost}))
                 for k, v in self._metrics.get_online_by_connection(node=node, vhost=vhost).items():
-                    data.append(self._parse('ejabberd_online_connection', k, v, cur_ts, {'node': node, 'vhost': vhost}))
+                    data.append(self._parse("ejabberd_online_connection", k, v, cur_ts, {"node": node, "vhost": vhost}))
                 for cl, ipv in self._metrics.get_online_client_by_ipversion(node=node, vhost=vhost).items():
                     for k, v in ipv.items():
-                        data.append(self._parse('ejabberd_online_client_ipversion', k, v, cur_ts,
+                        data.append(
-                                                {'vhost': vhost, 'node': node, 'ipversion': k, 'client': cl}))
+                            self._parse(
+                                "ejabberd_online_client_ipversion",
+                                k,
+                                v,
+                                cur_ts,
+                                {"vhost": vhost, "node": node, "ipversion": k, "client": cl},
+                            )
+                        )
 
         # write output to database
-        self.client.write_points(data, time_precision='ms', batch_size=10000, protocol='line')
+        self.client.write_points(data, time_precision="ms", batch_size=10000, protocol="line")
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     # load config
     config = Config()
 
-    if config.get('debug', default=False):
+    if config.get("debug", default=False):
         logging.getLogger().setLevel(logging.DEBUG)
 
     # credentials and parameters
-    url = config.get('url', default='http://localhost:5280/api')
+    url = config.get("url", default="http://localhost:5280/api")
-    login = config.get('login', default=None)
+    login = config.get("login", default=None)
-    api = config.get('api', default='rest')
+    api = config.get("api", default="rest")
 
     # config influxdb
-    influx_host = config.get('influxdb_host', default='localhost')
+    influx_host = config.get("influxdb_host", default="localhost")
-    influx_port = config.get('influxdb_port', default=8086)
+    influx_port = config.get("influxdb_port", default=8086)
-    influx_dbname = config.get('influxdb_db', default='ejabberd')
+    influx_dbname = config.get("influxdb_db", default="ejabberd")
 
     # init handler
     metrics = EjabberdMetrics(url, login, api)

metrics.py

@@ -12,7 +12,8 @@ class EjabberdMetrics(EjabberdApiCalls):
     """
     class to fetch metrics per xmlrpc
     """
-    def __init__(self, url, login=None, api="rpc", muc_host: str = 'conference'):
+
+    def __init__(self, url, login=None, api="rpc", muc_host: str = "conference"):
         # init ejabberd api
         super().__init__(url, login, api)
 
@@ -32,7 +33,7 @@ class EjabberdMetrics(EjabberdApiCalls):
             "profanity": [],
             "Xabber": ["xabber", "xabber-android"],
             "ChatSecure": ["chatsecure"],
-            "Monal": []
+            "Monal": [],
         }
 
         for client, names in clientmap.items():
@@ -110,7 +111,7 @@ class EjabberdMetrics(EjabberdApiCalls):
         return self.get_online_by("node", vhost=vhost)
 
     def get_online_by_vhost(self, node=None):
-        return self.get_online_by("jid", parse=lambda jid: jid[jid.find("@")+1:jid.find("/")], node=node)
+        return self.get_online_by("jid", parse=lambda jid: jid[jid.find("@") + 1 : jid.find("/")], node=node)
 
     def get_online_by_status(self, vhost=None, node=None):
         return self.get_online_by("status", vhost=vhost, node=node)
@@ -187,7 +188,7 @@ class EjabberdMetrics(EjabberdApiCalls):
         return self._uptime
 
     def get_processes(self):
-        if not hasattr(self, '_processes'):
+        if not hasattr(self, "_processes"):
             self._processes = self.fetch_processes()
         return self._processes
 
@@ -199,7 +200,7 @@ class EjabberdMetrics(EjabberdApiCalls):
             "online_by_client": self.get_online_by_client(vhost),
             "online_by_ipversion": self.get_online_by_ipversion(vhost),
             "online_by_connection": self.get_online_by_connection(vhost),
-            "online_by_node": self.get_online_by_node(vhost)
+            "online_by_node": self.get_online_by_node(vhost),
         }
 
         return data
@@ -215,7 +216,7 @@ class EjabberdMetrics(EjabberdApiCalls):
             "online_by_client": self.get_online_by_client(node=node),
             "online_by_ipversion": self.get_online_by_ipversion(node=node),
             "online_by_connection": self.get_online_by_connection(node=node),
-            "online_by_vhost": self.get_online_by_vhost(node=node)
+            "online_by_vhost": self.get_online_by_vhost(node=node),
         }
 
         return data
@@ -233,7 +234,7 @@ class EjabberdMetrics(EjabberdApiCalls):
             "online_by_ipversion": self.get_online_by_ipversion(),
             "online_by_connection": self.get_online_by_connection(),
             "online_by_node": self.get_online_by_node(),
-            "online_by_vhost": self.get_online_by_vhost()
+            "online_by_vhost": self.get_online_by_vhost(),
         }
 
         vhosts = {}
@@ -259,9 +260,9 @@ if __name__ == "__main__":
     config = Config()
 
     # credentials and parameters
-    url = config.get('url', default='http://localhost:5280/api')
+    url = config.get("url", default="http://localhost:5280/api")
-    login = config.get('login', default=None)
+    login = config.get("login", default=None)
-    api = config.get('api', default='rest')
+    api = config.get("api", default="rest")
 
     # init handler
     metrics = EjabberdMetrics(url, login, api)

prometheus.py

@@ -7,9 +7,7 @@ from socket import AF_INET6
 from time import time
 from urllib.parse import parse_qs, urlparse
 
-from prometheus_client import (
+from prometheus_client import CollectorRegistry, Gauge, generate_latest, CONTENT_TYPE_LATEST
-    CollectorRegistry, Gauge, generate_latest, CONTENT_TYPE_LATEST
-)
 
 from config import Config
 from metrics import EjabberdMetrics
@@ -17,26 +15,25 @@ from metrics import EjabberdMetrics
 
 class DynamicMetricsHandler(BaseHTTPRequestHandler):
     """HTTP handler that gives metrics from ``core.REGISTRY``."""
+
     def do_GET(self):
         params = parse_qs(urlparse(self.path).query)
         registry = self.generator(params)
-        if 'name[]' in params:
+        if "name[]" in params:
-            registry = registry.restricted_registry(params['name[]'])
+            registry = registry.restricted_registry(params["name[]"])
         try:
             output = generate_latest(registry)
-        except:
+        except:  # noqa: E722
-            self.send_error(500, 'error generating metric output')
+            self.send_error(500, "error generating metric output")
             raise
         self.send_response(200)
-        self.send_header('Content-Type', CONTENT_TYPE_LATEST)
+        self.send_header("Content-Type", CONTENT_TYPE_LATEST)
         self.end_headers()
         self.wfile.write(output)
 
     @staticmethod
     def factory(registry_generator):
-        DynMetricsHandler = type('MetricsHandler',
+        DynMetricsHandler = type("MetricsHandler", (DynamicMetricsHandler, object), {"generator": registry_generator})
-                                (DynamicMetricsHandler, object),
-                                {"generator": registry_generator})
         return DynMetricsHandler
 
 
@@ -54,25 +51,53 @@ class Prometheus:
 
         registry = CollectorRegistry(auto_describe=True)
 
-        Gauge('ejabberd_node_s2s_in',  'count of incoming server-to-server connection', registry=registry).set(self._metrics.get_s2s_in())
+        Gauge("ejabberd_node_s2s_in", "count of incoming server-to-server connection", registry=registry).set(
-        Gauge('ejabberd_node_s2s_out', 'count of outgoing server-to-server connection', registry=registry).set(self._metrics.get_s2s_out())
+            self._metrics.get_s2s_in()
+        )
+        Gauge("ejabberd_node_s2s_out", "count of outgoing server-to-server connection", registry=registry).set(
+            self._metrics.get_s2s_out()
+        )
 
         nodename = self._metrics.nodename
-        Gauge('ejabberd_node_uptime',  'uptime of ejabberd service', ["node"], registry=registry).labels(nodename).set(self._metrics.get_uptime())
+        Gauge("ejabberd_node_uptime", "uptime of ejabberd service", ["node"], registry=registry).labels(nodename).set(
-        Gauge('ejabberd_node_proccess', 'count of pejabber proccess', ["node"], registry=registry).labels(nodename).set(self._metrics.get_processes())
+            self._metrics.get_uptime()
+        )
+        Gauge("ejabberd_node_proccess", "count of pejabber proccess", ["node"], registry=registry).labels(nodename).set(
+            self._metrics.get_processes()
+        )
 
         labelnames_vhost = ["vhost"]
 
-        registered_vhosts = Gauge('ejabberd_registered_vhosts', 'count of user per vhost', labelnames_vhost, registry=registry)
+        registered_vhosts = Gauge(
-        muc = Gauge('ejabberd_muc', 'count of muc\'s per vhost', labelnames_vhost, registry=registry)
+            "ejabberd_registered_vhosts", "count of user per vhost", labelnames_vhost, registry=registry
+        )
+        muc = Gauge("ejabberd_muc", "count of muc's per vhost", labelnames_vhost, registry=registry)
 
-        online_vhost_node = Gauge('ejabberd_online_vhost_node', 'count of client connections', ["vhost", "node"], registry=registry)
+        online_vhost_node = Gauge(
+            "ejabberd_online_vhost_node", "count of client connections", ["vhost", "node"], registry=registry
+        )
 
-        online_status = Gauge('ejabberd_online_status', 'count of client connections', ["vhost", "node", "status"], registry=registry)
+        online_status = Gauge(
-        online_connection = Gauge('ejabberd_online_connection', 'count of client connections', ["vhost", "node", "connection"], registry=registry)
+            "ejabberd_online_status", "count of client connections", ["vhost", "node", "status"], registry=registry
-        online_client = Gauge('ejabberd_online_client', 'count of client software', ["vhost", "node", "client"], registry=registry)
+        )
-        online_ipversion = Gauge('ejabberd_online_ipversion', 'count of client software', ["vhost", "node", "ipversion"], registry=registry)
+        online_connection = Gauge(
-        online_client_ipversion = Gauge('ejabberd_online_client_ipversion', 'count of client software', ["vhost", "node", "client", "ipversion"], registry=registry)
+            "ejabberd_online_connection",
+            "count of client connections",
+            ["vhost", "node", "connection"],
+            registry=registry,
+        )
+        online_client = Gauge(
+            "ejabberd_online_client", "count of client software", ["vhost", "node", "client"], registry=registry
+        )
+        online_ipversion = Gauge(
+            "ejabberd_online_ipversion", "count of client software", ["vhost", "node", "ipversion"], registry=registry
+        )
+        online_client_ipversion = Gauge(
+            "ejabberd_online_client_ipversion",
+            "count of client software",
+            ["vhost", "node", "client", "ipversion"],
+            registry=registry,
+        )
 
         for host in self._metrics.get_vhosts():
             labels_vhost = host
@@ -108,19 +133,19 @@ class Prometheus:
 if __name__ == "__main__":
     # load config
     config = Config()
-    if config.get('debug', default=False):
+    if config.get("debug", default=False):
         logging.getLogger().setLevel(logging.DEBUG)
 
     # credentials and parameters
-    url = config.get('url', default='http://[::1]:5280/api')
+    url = config.get("url", default="http://[::1]:5280/api")
-    login = config.get('login', default=None)
+    login = config.get("login", default=None)
-    api = config.get('api', default='rest')
+    api = config.get("api", default="rest")
 
     # config prometheus
-    prom_addr = config.get('prometheus_address', default='127.0.0.1')
+    prom_addr = config.get("prometheus_address", default="127.0.0.1")
-    prom_port = config.get('prometheus_port', default=8080)
+    prom_port = config.get("prometheus_port", default=8080)
 
     metrics = EjabberdMetrics(url, login, api)
     prom = Prometheus(metrics)
-    prom.ttl = config.get('prometheus_cache_ttl', default=10)
+    prom.ttl = config.get("prometheus_cache_ttl", default=10)
     prom.listen((prom_addr, prom_port))

pyproject.toml

@@ -0,0 +1,20 @@
+[tool.black]
+line-length = 120
+target-version = ['py37', 'py38']
+include = '\.pyi?$'
+exclude = '''
+(
+  /(
+      \.eggs         # exclude a few common directories in the
+    | \.git          # root of the project
+    | \.hg
+    | \.mypy_cache
+    | \.tox
+    | \.venv
+    | _build
+    | buck-out
+    | build
+    | dist
+  )/                     # the root of the project
+)
+'''

setup.cfg

@@ -0,0 +1,5 @@
+[flake8]
+ignore = E501,E203
+exclude = .git,__pycache__,.gitlab
+max-complexity = 15
+max-line-length = 120