From 38a7b171808c5a1525da5452b338fac0a9bf2c6b Mon Sep 17 00:00:00 2001 From: genofire Date: Wed, 10 Jun 2020 07:22:42 +0200 Subject: [PATCH] add cleanup.py --- api.py | 19 -------- cleanup.py | 83 ++++++++++++++++++++++++++++++++++ control.py | 84 ++++++++++++++++++++++++++++++++++ ejabberd-metrics.yml.default | 20 +++++++++ metrics.py | 87 +++++++----------------------------- 5 files changed, 203 insertions(+), 90 deletions(-) create mode 100755 cleanup.py create mode 100644 control.py mode change 100644 => 100755 metrics.py diff --git a/api.py b/api.py index a83bae7..ab25f0a 100644 --- a/api.py +++ b/api.py @@ -29,25 +29,6 @@ class EjabberdApi: return f"{self._login['user']}@{self._login['server']}", self._login['password'] return None - @property - def verstring(self): - if self._login is not None: - ver_str = re.compile('([1-9][0-9.]+(?![.a-z]))\\b') - status = self.cmd('status', {}) - - # matches - try: - tmp = ver_str.findall(status)[0] - # raise SystemExit code 17 if no status message is received - except TypeError: - raise SystemExit(17) - - # return parsed version string - logging.debug(f"fetch version: {tmp}") - return version.parse(tmp) - - return None - def _rest(self, command: str, data) -> dict: # add authentication header to the session obj if self.session.auth is None: diff --git a/cleanup.py b/cleanup.py new file mode 100755 index 0000000..63947bd --- /dev/null +++ b/cleanup.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +import logging +import datetime +from control import EjabberdCtl + + +class EjabberdCleanup(EjabberdCtl): + def __init__(self, url, login, api): + super().__init__(url, login, api) + self.ignore_hosts = [] + self.ignore_usernames = [] + self.dry = False + self.skip_by_roster_roster = True + self.delete_not_login = True + self.offline_since_days = None + + def delete_user(self, host, user, reason=""): + if self.dry: + logging.warning(f"{user}@{host}: dry delete : {reason}") + else: + self.cmd("unregister", {"host": host, "user": user}) + logging.warning(f"{user}@{host}: deleted - {reason}") + + def run_user(self, host, user): + if self.skip_by_roster: + roster = self.cmd("get_roster",{"host": host, "user": user}) + if len(roster) > 0: + logging.debug(f"{user}@{host}: skipped it has a roster") + return + last = self.cmd("get_last",{"host": host, "user": user}) + if self.delete_not_login and last["status"] == "Registered but didn't login": + self.delete_user(host, user, "not login") + return + if self.offline_since_days is not None: + last_stamp = last["timestamp"] + lastdate = None + try: + lastdate = datetime.datetime.strptime(last_stamp,"%Y-%m-%dT%H:%M:%SZ") + except: + try: + lastdate = datetime.datetime.strptime(last_stamp,"%Y-%m-%dT%H:%M:%S.%fZ") + except: + logging.error(f"{user}@{host}: not able to parse '{last_stamp}'") + return + if lastdate != None and lastdate-datetime.datetime.now() > datetime.timedelta(days=self.offline_since_days): + self.delete_user(host, user, f"last seen @ {lastdate}") + return + + + def run(self): + for host in self.fetch_vhosts(): + if host in self.ignore_hosts: + continue + logging.info(f"run cleanup for host: {host}") + for user in self.cmd("registered_users",{"host": host}): + if user in self.ignore_usernames: + continue + self.run_user(host, user) + +if __name__ == "__main__": + import json + from config import Config + + # load config + config = Config() + if config.get('debug', default=False): + logging.getLogger().setLevel(logging.DEBUG) + + # credentials and parameters + url = config.get('url', default='http://localhost:5280/api') + login = config.get('login', default=None) + api = config.get('api', default='rest') + + # init handler + cleaner = EjabberdCleanup(url, login, api) + cleaner.dry = config.get('cleaner_dry',default=False) + cleaner.ignore_hosts = config.get('cleaner_ignore_hosts',default=[]) + cleaner.ignore_usernames = config.get('cleaner_ignore_usernames',default=[]) + cleaner.skip_by_roster = config.get('cleaner_skip_by_roster',default=True) + cleaner.delete_not_login = config.get('cleaner_delete_not_login',default=True) + cleaner.offline_since_days = config.get('cleaner_offline_since_days', default=None) + cleaner.run() diff --git a/control.py b/control.py new file mode 100644 index 0000000..3bbf3c7 --- /dev/null +++ b/control.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +import re +import logging + +from packaging import version +from api import EjabberdApi + +class EjabberdCtl(EjabberdApi): + + + @property + def verstring(self): + if self._login is not None: + ver_str = re.compile('([1-9][0-9.]+(?![.a-z]))\\b') + status = self.cmd('status', {}) + + # matches + try: + tmp = ver_str.findall(status)[0] + # raise SystemExit code 17 if no status message is received + except TypeError: + raise SystemExit(17) + + # return parsed version string + logging.debug(f"fetch version: {tmp}") + return version.parse(tmp) + + return None + + def fetch_onlineuser(self): + tmp = self.cmd("connected_users_info", {}) + if "connected_users_info" not in tmp: + return tmp + data = [] + for c in tmp["connected_users_info"]: + if "session" not in c: + continue + user = {} + for attrs in c["session"]: + for k, v in attrs.items(): + user[k] = v + data.append(user) + return data + + def fetch_nodes(self): + result = self.cmd("list_cluster", {}) + if "nodes" not in result: + return result + data = [] + for node in result["nodes"]: + data.append(node["node"]) + return data + + def fetch_vhosts(self): + result = self.cmd("registered_vhosts", {}) + if "vhosts" not in result: + return result + data = [] + for vhost in result["vhosts"]: + data.append(vhost["vhost"]) + return data + + def fetch_s2s_in(self): + result = self.cmd("incoming_s2s_number", {}) + if "s2s_incoming" not in result: + return result + return result["s2s_incoming"] + + def fetch_s2s_out(self): + result = self.cmd("outgoing_s2s_number", {}) + if "s2s_outgoing" not in result: + return result + return result["s2s_outgoing"] + + def fetch_registered(self, vhost=None): + if vhost is None: + result = self.cmd("stats", {"name":"registeredusers"}) + if "stat" in result: + return result["stat"] + else: + result = self.cmd("stats_host", {"name":"registeredusers", "host": vhost}) + if "stat" in result: + return result["stat"] diff --git a/ejabberd-metrics.yml.default b/ejabberd-metrics.yml.default index e89d0ef..db1c89a 100644 --- a/ejabberd-metrics.yml.default +++ b/ejabberd-metrics.yml.default @@ -15,6 +15,8 @@ muc_host: "chat" # api configuration # default : rpc api: "rest" +# show debugging log messages +debug: True # influx db configuration influxdb_host: "localhost" @@ -25,3 +27,21 @@ influxdb_db: "example" prometheus_address: "127.0.0.1" prometheus_port: 8080 prometheus_cache_ttl: 10 + +# cleaner for users +cleaner_dry: True # just test run (only loggging output) +# skip remove of user if client has a roster +cleaner_skip_by_roster: True +# delete user if it was only registered but has never login +cleaner_delete_not_login: True +# delete user if the are offline for x days +cleaner_offline_since_days: 7 +# ignore cleanup for host +cleaner_ignore_hosts: + - "meckerspace.de" +# ignore cleanup for username +cleaner_ignore_usernames: + - "admin" + - "abuse" + - "bugs" + - "bot" diff --git a/metrics.py b/metrics.py old mode 100644 new mode 100755 index 2f6b82b..ad23033 --- a/metrics.py +++ b/metrics.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- import ipaddress -from api import EjabberdApi +from control import EjabberdCtl # rfc6052: IPv6 Addressing of IPv4/IPv6 Translators nat64 = ipaddress.ip_network("64:ff9b::/96") @@ -14,7 +14,7 @@ class EjabberdMetrics: """ def __init__(self, url, login=None, api="rpc", muc_host: str = 'conference'): # init ejabberd api - self.api = EjabberdApi(url, login, api) + self.api = EjabberdCtl(url, login, api) self._cmd = self.api.cmd # variables @@ -55,61 +55,6 @@ class EjabberdMetrics: return 4 return addr.version - def fetch_onlineuser(self): - tmp = self._cmd("connected_users_info", {}) - if "connected_users_info" not in tmp: - return tmp - data = [] - for c in tmp["connected_users_info"]: - if "session" not in c: - continue - user = {} - for attrs in c["session"]: - for k, v in attrs.items(): - user[k] = v - data.append(user) - return data - - def fetch_nodes(self): - result = self._cmd("list_cluster", {}) - if "nodes" not in result: - return result - data = [] - for node in result["nodes"]: - data.append(node["node"]) - return data - - def fetch_vhosts(self): - result = self._cmd("registered_vhosts", {}) - if "vhosts" not in result: - return result - data = [] - for vhost in result["vhosts"]: - data.append(vhost["vhost"]) - return data - - def fetch_s2s_in(self): - result = self._cmd("incoming_s2s_number", {}) - if "s2s_incoming" not in result: - return result - return result["s2s_incoming"] - - def fetch_s2s_out(self): - result = self._cmd("outgoing_s2s_number", {}) - if "s2s_outgoing" not in result: - return result - return result["s2s_outgoing"] - - def fetch_registered(self, vhost=None): - if vhost is None: - result = self._cmd("stats", {"name":"registeredusers"}) - if "stat" in result: - return result["stat"] - else: - result = self._cmd("stats_host", {"name":"registeredusers", "host": vhost}) - if "stat" in result: - return result["stat"] - def fetch_muc(self, vhost=None): host = "global" if vhost is not None: @@ -124,15 +69,15 @@ class EjabberdMetrics: def update(self): # nodes - self._nodes = self.fetch_nodes() + self._nodes = self.api.fetch_nodes() # vhosts - self._vhosts = self.fetch_vhosts() + self._vhosts = self.api.fetch_vhosts() # registered if not hasattr(self, "_registered"): self._registered = {} - self._registered[None] = self.fetch_registered() + self._registered[None] = self.api.fetch_registered() # muc if not hasattr(self, "_muc"): @@ -141,20 +86,20 @@ class EjabberdMetrics: # registered + muc for vhost in self._vhosts: - self._registered[vhost] = self.fetch_registered(vhost) + self._registered[vhost] = self.api.fetch_registered(vhost) self._muc[vhost] = self.fetch_muc(vhost) # online user - self._onlineuser = self.fetch_onlineuser() + self._onlineuser = self.api.fetch_onlineuser() # s2s - self._s2s_in = self.fetch_s2s_in() - self._s2s_out = self.fetch_s2s_out() + self._s2s_in = self.api.fetch_s2s_in() + self._s2s_out = self.api.fetch_s2s_out() def get_online_by(self, by="node", parse=None, vhost=None, node=None): parser = parse or (lambda a: a) if not hasattr(self, "_onlineuser"): - self._onlineuser = self.fetch_onlineuser() + self._onlineuser = self.api.fetch_onlineuser() data = {} for conn in self._onlineuser: @@ -192,7 +137,7 @@ class EjabberdMetrics: def get_online_client_by(self, by="ip", parse=None, vhost=None, node=None): parser = parse or self._ipversion if not hasattr(self, "_onlineuser"): - self._onlineuser = self.fetch_onlineuser() + self._onlineuser = self.api.fetch_onlineuser() data = {} for conn in self._onlineuser: @@ -221,7 +166,7 @@ class EjabberdMetrics: if not hasattr(self, "_registered"): self._registered = {} if vhost not in self._registered: - self._registered[vhost] = self.fetch_registered(vhost) + self._registered[vhost] = self.api.fetch_registered(vhost) return self._registered[vhost] def get_muc(self, vhost=None): @@ -233,17 +178,17 @@ class EjabberdMetrics: def get_vhosts(self): if not hasattr(self, "_vhosts"): - self._vhosts = self.fetch_vhosts() + self._vhosts = self.api.fetch_vhosts() return self._vhosts def get_s2s_in(self): if not hasattr(self, "_s2s_in"): - self._s2s_in = self.fetch_s2s_in() + self._s2s_in = self.api.fetch_s2s_in() return self._s2s_in def get_s2s_out(self): if not hasattr(self, "_s2s_out"): - self._s2s_out = self.fetch_s2s_out() + self._s2s_out = self.api.fetch_s2s_out() return self._s2s_out def get_vhost_metrics(self, vhost): @@ -261,7 +206,7 @@ class EjabberdMetrics: def get_nodes(self): if not hasattr(self, "_nodes"): - self._nodes = self.fetch_nodes() + self._nodes = self.api.fetch_nodes() return self._nodes def get_node_metrics(self, node):