add cleanup.py

This commit is contained in:
genofire 2020-06-10 07:22:42 +02:00
parent 893f68497d
commit 38a7b17180
5 changed files with 203 additions and 90 deletions

19
api.py
View File

@ -29,25 +29,6 @@ class EjabberdApi:
return f"{self._login['user']}@{self._login['server']}", self._login['password']
return None
@property
def verstring(self):
if self._login is not None:
ver_str = re.compile('([1-9][0-9.]+(?![.a-z]))\\b')
status = self.cmd('status', {})
# matches
try:
tmp = ver_str.findall(status)[0]
# raise SystemExit code 17 if no status message is received
except TypeError:
raise SystemExit(17)
# return parsed version string
logging.debug(f"fetch version: {tmp}")
return version.parse(tmp)
return None
def _rest(self, command: str, data) -> dict:
# add authentication header to the session obj
if self.session.auth is None:

83
cleanup.py Executable file
View File

@ -0,0 +1,83 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import logging
import datetime
from control import EjabberdCtl
class EjabberdCleanup(EjabberdCtl):
def __init__(self, url, login, api):
super().__init__(url, login, api)
self.ignore_hosts = []
self.ignore_usernames = []
self.dry = False
self.skip_by_roster_roster = True
self.delete_not_login = True
self.offline_since_days = None
def delete_user(self, host, user, reason=""):
if self.dry:
logging.warning(f"{user}@{host}: dry delete : {reason}")
else:
self.cmd("unregister", {"host": host, "user": user})
logging.warning(f"{user}@{host}: deleted - {reason}")
def run_user(self, host, user):
if self.skip_by_roster:
roster = self.cmd("get_roster",{"host": host, "user": user})
if len(roster) > 0:
logging.debug(f"{user}@{host}: skipped it has a roster")
return
last = self.cmd("get_last",{"host": host, "user": user})
if self.delete_not_login and last["status"] == "Registered but didn't login":
self.delete_user(host, user, "not login")
return
if self.offline_since_days is not None:
last_stamp = last["timestamp"]
lastdate = None
try:
lastdate = datetime.datetime.strptime(last_stamp,"%Y-%m-%dT%H:%M:%SZ")
except:
try:
lastdate = datetime.datetime.strptime(last_stamp,"%Y-%m-%dT%H:%M:%S.%fZ")
except:
logging.error(f"{user}@{host}: not able to parse '{last_stamp}'")
return
if lastdate != None and lastdate-datetime.datetime.now() > datetime.timedelta(days=self.offline_since_days):
self.delete_user(host, user, f"last seen @ {lastdate}")
return
def run(self):
for host in self.fetch_vhosts():
if host in self.ignore_hosts:
continue
logging.info(f"run cleanup for host: {host}")
for user in self.cmd("registered_users",{"host": host}):
if user in self.ignore_usernames:
continue
self.run_user(host, user)
if __name__ == "__main__":
import json
from config import Config
# load config
config = Config()
if config.get('debug', default=False):
logging.getLogger().setLevel(logging.DEBUG)
# credentials and parameters
url = config.get('url', default='http://localhost:5280/api')
login = config.get('login', default=None)
api = config.get('api', default='rest')
# init handler
cleaner = EjabberdCleanup(url, login, api)
cleaner.dry = config.get('cleaner_dry',default=False)
cleaner.ignore_hosts = config.get('cleaner_ignore_hosts',default=[])
cleaner.ignore_usernames = config.get('cleaner_ignore_usernames',default=[])
cleaner.skip_by_roster = config.get('cleaner_skip_by_roster',default=True)
cleaner.delete_not_login = config.get('cleaner_delete_not_login',default=True)
cleaner.offline_since_days = config.get('cleaner_offline_since_days', default=None)
cleaner.run()

84
control.py Normal file
View File

@ -0,0 +1,84 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import re
import logging
from packaging import version
from api import EjabberdApi
class EjabberdCtl(EjabberdApi):
@property
def verstring(self):
if self._login is not None:
ver_str = re.compile('([1-9][0-9.]+(?![.a-z]))\\b')
status = self.cmd('status', {})
# matches
try:
tmp = ver_str.findall(status)[0]
# raise SystemExit code 17 if no status message is received
except TypeError:
raise SystemExit(17)
# return parsed version string
logging.debug(f"fetch version: {tmp}")
return version.parse(tmp)
return None
def fetch_onlineuser(self):
tmp = self.cmd("connected_users_info", {})
if "connected_users_info" not in tmp:
return tmp
data = []
for c in tmp["connected_users_info"]:
if "session" not in c:
continue
user = {}
for attrs in c["session"]:
for k, v in attrs.items():
user[k] = v
data.append(user)
return data
def fetch_nodes(self):
result = self.cmd("list_cluster", {})
if "nodes" not in result:
return result
data = []
for node in result["nodes"]:
data.append(node["node"])
return data
def fetch_vhosts(self):
result = self.cmd("registered_vhosts", {})
if "vhosts" not in result:
return result
data = []
for vhost in result["vhosts"]:
data.append(vhost["vhost"])
return data
def fetch_s2s_in(self):
result = self.cmd("incoming_s2s_number", {})
if "s2s_incoming" not in result:
return result
return result["s2s_incoming"]
def fetch_s2s_out(self):
result = self.cmd("outgoing_s2s_number", {})
if "s2s_outgoing" not in result:
return result
return result["s2s_outgoing"]
def fetch_registered(self, vhost=None):
if vhost is None:
result = self.cmd("stats", {"name":"registeredusers"})
if "stat" in result:
return result["stat"]
else:
result = self.cmd("stats_host", {"name":"registeredusers", "host": vhost})
if "stat" in result:
return result["stat"]

View File

@ -15,6 +15,8 @@ muc_host: "chat"
# api configuration
# default : rpc
api: "rest"
# show debugging log messages
debug: True
# influx db configuration
influxdb_host: "localhost"
@ -25,3 +27,21 @@ influxdb_db: "example"
prometheus_address: "127.0.0.1"
prometheus_port: 8080
prometheus_cache_ttl: 10
# cleaner for users
cleaner_dry: True # just test run (only loggging output)
# skip remove of user if client has a roster
cleaner_skip_by_roster: True
# delete user if it was only registered but has never login
cleaner_delete_not_login: True
# delete user if the are offline for x days
cleaner_offline_since_days: 7
# ignore cleanup for host
cleaner_ignore_hosts:
- "meckerspace.de"
# ignore cleanup for username
cleaner_ignore_usernames:
- "admin"
- "abuse"
- "bugs"
- "bot"

87
metrics.py Normal file → Executable file
View File

@ -2,7 +2,7 @@
# -*- coding: utf-8 -*-
import ipaddress
from api import EjabberdApi
from control import EjabberdCtl
# rfc6052: IPv6 Addressing of IPv4/IPv6 Translators
nat64 = ipaddress.ip_network("64:ff9b::/96")
@ -14,7 +14,7 @@ class EjabberdMetrics:
"""
def __init__(self, url, login=None, api="rpc", muc_host: str = 'conference'):
# init ejabberd api
self.api = EjabberdApi(url, login, api)
self.api = EjabberdCtl(url, login, api)
self._cmd = self.api.cmd
# variables
@ -55,61 +55,6 @@ class EjabberdMetrics:
return 4
return addr.version
def fetch_onlineuser(self):
tmp = self._cmd("connected_users_info", {})
if "connected_users_info" not in tmp:
return tmp
data = []
for c in tmp["connected_users_info"]:
if "session" not in c:
continue
user = {}
for attrs in c["session"]:
for k, v in attrs.items():
user[k] = v
data.append(user)
return data
def fetch_nodes(self):
result = self._cmd("list_cluster", {})
if "nodes" not in result:
return result
data = []
for node in result["nodes"]:
data.append(node["node"])
return data
def fetch_vhosts(self):
result = self._cmd("registered_vhosts", {})
if "vhosts" not in result:
return result
data = []
for vhost in result["vhosts"]:
data.append(vhost["vhost"])
return data
def fetch_s2s_in(self):
result = self._cmd("incoming_s2s_number", {})
if "s2s_incoming" not in result:
return result
return result["s2s_incoming"]
def fetch_s2s_out(self):
result = self._cmd("outgoing_s2s_number", {})
if "s2s_outgoing" not in result:
return result
return result["s2s_outgoing"]
def fetch_registered(self, vhost=None):
if vhost is None:
result = self._cmd("stats", {"name":"registeredusers"})
if "stat" in result:
return result["stat"]
else:
result = self._cmd("stats_host", {"name":"registeredusers", "host": vhost})
if "stat" in result:
return result["stat"]
def fetch_muc(self, vhost=None):
host = "global"
if vhost is not None:
@ -124,15 +69,15 @@ class EjabberdMetrics:
def update(self):
# nodes
self._nodes = self.fetch_nodes()
self._nodes = self.api.fetch_nodes()
# vhosts
self._vhosts = self.fetch_vhosts()
self._vhosts = self.api.fetch_vhosts()
# registered
if not hasattr(self, "_registered"):
self._registered = {}
self._registered[None] = self.fetch_registered()
self._registered[None] = self.api.fetch_registered()
# muc
if not hasattr(self, "_muc"):
@ -141,20 +86,20 @@ class EjabberdMetrics:
# registered + muc
for vhost in self._vhosts:
self._registered[vhost] = self.fetch_registered(vhost)
self._registered[vhost] = self.api.fetch_registered(vhost)
self._muc[vhost] = self.fetch_muc(vhost)
# online user
self._onlineuser = self.fetch_onlineuser()
self._onlineuser = self.api.fetch_onlineuser()
# s2s
self._s2s_in = self.fetch_s2s_in()
self._s2s_out = self.fetch_s2s_out()
self._s2s_in = self.api.fetch_s2s_in()
self._s2s_out = self.api.fetch_s2s_out()
def get_online_by(self, by="node", parse=None, vhost=None, node=None):
parser = parse or (lambda a: a)
if not hasattr(self, "_onlineuser"):
self._onlineuser = self.fetch_onlineuser()
self._onlineuser = self.api.fetch_onlineuser()
data = {}
for conn in self._onlineuser:
@ -192,7 +137,7 @@ class EjabberdMetrics:
def get_online_client_by(self, by="ip", parse=None, vhost=None, node=None):
parser = parse or self._ipversion
if not hasattr(self, "_onlineuser"):
self._onlineuser = self.fetch_onlineuser()
self._onlineuser = self.api.fetch_onlineuser()
data = {}
for conn in self._onlineuser:
@ -221,7 +166,7 @@ class EjabberdMetrics:
if not hasattr(self, "_registered"):
self._registered = {}
if vhost not in self._registered:
self._registered[vhost] = self.fetch_registered(vhost)
self._registered[vhost] = self.api.fetch_registered(vhost)
return self._registered[vhost]
def get_muc(self, vhost=None):
@ -233,17 +178,17 @@ class EjabberdMetrics:
def get_vhosts(self):
if not hasattr(self, "_vhosts"):
self._vhosts = self.fetch_vhosts()
self._vhosts = self.api.fetch_vhosts()
return self._vhosts
def get_s2s_in(self):
if not hasattr(self, "_s2s_in"):
self._s2s_in = self.fetch_s2s_in()
self._s2s_in = self.api.fetch_s2s_in()
return self._s2s_in
def get_s2s_out(self):
if not hasattr(self, "_s2s_out"):
self._s2s_out = self.fetch_s2s_out()
self._s2s_out = self.api.fetch_s2s_out()
return self._s2s_out
def get_vhost_metrics(self, vhost):
@ -261,7 +206,7 @@ class EjabberdMetrics:
def get_nodes(self):
if not hasattr(self, "_nodes"):
self._nodes = self.fetch_nodes()
self._nodes = self.api.fetch_nodes()
return self._nodes
def get_node_metrics(self, node):