From 3e8a728fe751cfb02a34e3d72730dc4d6ee1a7cb Mon Sep 17 00:00:00 2001 From: nico wellpott Date: Sat, 27 Mar 2021 00:11:22 +0100 Subject: [PATCH 1/6] Python Package + init basic python package + add console_script blimp --- MANIFEST.in | 3 + main.py | 152 ---------------------------------------- requirements.txt | 5 +- setup.cfg | 39 +++++++++++ setup.py | 13 ++++ src/blimp/__init__.py | 14 ++++ src/blimp/bl_process.py | 54 ++++++++++++++ src/blimp/cli.py | 16 +++++ src/blimp/main.py | 115 ++++++++++++++++++++++++++++++ src/blimp/misc.py | 17 +++++ 10 files changed, 274 insertions(+), 154 deletions(-) create mode 100644 MANIFEST.in delete mode 100644 main.py create mode 100644 setup.cfg create mode 100644 setup.py create mode 100644 src/blimp/__init__.py create mode 100644 src/blimp/bl_process.py create mode 100644 src/blimp/cli.py create mode 100644 src/blimp/main.py create mode 100644 src/blimp/misc.py diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..7133ca7 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,3 @@ +include README.md +include LICENCE.md +include requirements.txt diff --git a/main.py b/main.py deleted file mode 100644 index e39d82b..0000000 --- a/main.py +++ /dev/null @@ -1,152 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -import argparse -import os -import subprocess -import sys - -import requests -from ruamel.yaml import YAML, scalarstring - - -class BlacklistImporter: - def __init__(self, args): - self.outfile = args.outfile - self.dryrun = args.dryrun - self.path = os.path.dirname(__file__) - self.url = "https://raw.githubusercontent.com/JabberSPAM/blacklist/master/blacklist.txt" - self.blacklist = "" - self.change = False - - def request(self): - """ - determine if the download is required - """ - etag_path = "/".join([self.path, ".etag"]) - blacklist_path = "/".join([self.path, "blacklist.txt"]) - - # check if etag header is present if not set local_etag to "" - if os.path.isfile(etag_path): - # catch special case were etag file is present and blacklist.txt is not - if not os.path.isfile(blacklist_path): - local_etag = "" - else: - # if both files are present continue normally - with open(etag_path, "r") as local_file: - local_etag = local_file.read() - else: - local_etag = "" - - with requests.Session() as s: - # head request to check etag - head = s.head(self.url) - etag = head.headers['etag'] - - # if etags match up or if the connection is not possible fall back to local cache - if local_etag == etag or head.status_code != 200: - # if local cache is present overwrite blacklist var - if os.path.isfile(blacklist_path): - with open(blacklist_path, "r", encoding="utf-8") as local_file: - self.blacklist = local_file.read() - - # in any other case request a new file - else: - r = s.get(self.url) - r.encoding = 'utf-8' - local_etag = head.headers['etag'] - self.blacklist = r.content.decode() - - with open(blacklist_path, "w") as local_file: - local_file.write(self.blacklist) - - with open(etag_path, 'w') as local_file: - local_file.write(local_etag) - - def main(self): - # first check if blacklist is updated - self.request() - - # only output the selected outfile - if self.dryrun: - print("outfile selected: %s" % self.outfile) - - # blacklist processing - self.process() - - # reload config if changes have been applied - if self.change: - # catch ejabberdctl missing - if os.path.isfile('/usr/sbin/ejabberdctl'): - subprocess.call(['/usr/sbin/ejabberdctl', 'reload_config'], shell=False) - - # report missing ejabberdctl reload_config - else: - print('/usr/sbin/ejabberdctl was not found', file=sys.stderr) - print('blacklist changes have been applied\nejabberd config was not reloaded', file=sys.stderr) - sys.exit(1) - - def process(self): - """ - function to build and compare the local yaml file to the remote file - if the remote file is different, the local file gets overwritten - """ - # init new YAML variable - local_file = YAML(typ="safe") - - # None catch - if self.outfile is not None: - # prevent FileNotFoundError on first run or file missing - if os.path.isfile(self.outfile): - local_file = local_file.load(open(self.outfile, "r", encoding="utf-8")) - - # blacklist frame - remote_file = { - "acl": { - "spamblacklist": { - "server": [] - } - } - } - - # build the blacklist with the given frame to compare to local blacklist - for entry in self.blacklist.split(): - entry = scalarstring.DoubleQuotedScalarString(entry) - remote_file["acl"]["spamblacklist"]["server"].append(entry) - - yml = YAML() - yml.indent(offset=2) - yml.default_flow_style = False - - # if dry-run true print expected content - if self.dryrun: - yml.dump(remote_file, sys.stdout) - - # only if the local_file and remote_file are unequal write new file - elif local_file != remote_file: - - # prevent FileNotFoundError if self.outfile is not assigned - if self.outfile is None: - print("no outfile assigned", file=sys.stderr) - print(parser.format_help(), file=sys.stderr) - sys.exit(2) - - # proceed to update the defined outfile - elif self.outfile is not None: - self.change = True - yml.dump(remote_file, open(self.outfile, "w")) - - # if that's impossible break and display help message - else: - print(parser.format_help(), file=sys.stderr) - sys.exit(1) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument('-o', '--outfile', help='set path to output file', dest='outfile', default=None) - parser.add_argument('-dr', '--dry-run', help='perform a dry run', action='store_true', dest='dryrun', default=False) - args = parser.parse_args() - - # run - BlacklistImporter(args).main() diff --git a/requirements.txt b/requirements.txt index ac76703..288c070 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ +appdirs==1.4.4 requests==2.25.1 -ruamel.yaml==0.16.13 +ruamel.yaml==0.17.0 ruamel.yaml.clib==0.2.2 -urllib3==1.26.3 +urllib3==1.26.4 diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..920adaf --- /dev/null +++ b/setup.cfg @@ -0,0 +1,39 @@ +[metadata] +name = blimp +version = attr: blimp.__version__ +url = https://github.com/mightyBroccoli/blacklist_importer +licence = GPLv3 +author = Nico Wellpott +author_email = nico@magicbroccoli.de +description = simple tool to download and update the JabberSpam blacklist list. +long_description = file: README.md +long_description_content_type = text/markdown +classifiers = + Development Status :: 4 - Beta + Intended Audience :: System Administrators + Licence :: OSI Approved :: GNU General Public License v3 (GPLv3) + Operating System :: Unix + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + Programming Language :: Python :: Implementation :: CPython + Topic :: Internet :: WWW/HTTP :: Dynamic Content + +[options] +packages = find: +package_dir = = src +include_package_data = true +python_requires = >= 3.7 + +[options.packages.find] +where = src + +[options.entry_points] +console_scripts = + blimp = blimp.cli:cli + +[tool:pytest] +testpaths = tests +filterwarnings = + error diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..36e690b --- /dev/null +++ b/setup.py @@ -0,0 +1,13 @@ +# -*- coding: utf-8 -*- +from setuptools import setup + +setup( + name="Blimp", + install_requires=[ + "appdirs>=1.4", + "requests>=2.25", + "ruamel.yaml>=0.17", + "ruamel.yaml.clib>=0.2", + "urllib3>=1.26" + ] +) diff --git a/src/blimp/__init__.py b/src/blimp/__init__.py new file mode 100644 index 0000000..5486de7 --- /dev/null +++ b/src/blimp/__init__.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# version +__version__ = "0.1" + +# main +from .main import BlacklistImporter + +# modules +from .bl_process import ProcessBlocklist + +# utils +from .misc import * diff --git a/src/blimp/bl_process.py b/src/blimp/bl_process.py new file mode 100644 index 0000000..e1d8c93 --- /dev/null +++ b/src/blimp/bl_process.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +import sys + +from ruamel.yaml import YAML, scalarstring + +from .misc import * + +class ProcessBlocklist: + def process(self, blacklist, outfile, dryrun: bool): + """ + function to build and compare the local yaml file to the remote file + if the remote file is different, the local file gets overwritten + """ + # cheeky none catch + try: + # load local blacklist outfile + if local_file_present(outfile): + with open(outfile, "r", encoding="utf-8") as local_file: + local_blacklist = local_file.read() + + print("step local file") + + except TypeError: + # no local copy use empty one instead + local_blacklist = YAML(typ="safe") + + # blacklist frame + remote_file = {"acl": {"spamblacklist": {"server": []}}} + + # build the blacklist with the given frame to compare to local blacklist + for entry in blacklist.split(): + entry = scalarstring.DoubleQuotedScalarString(entry) + remote_file["acl"]["spamblacklist"]["server"].append(entry) + + yml = YAML() + yml.indent(offset=2) + yml.default_flow_style = False + + # if dry-run true print expected content + if dryrun: + yml.dump(remote_file, sys.stdout) + return + + if local_blacklist == remote_file: + return + + if outfile is None: + print("no outfile assigned", file=sys.stderr) + sys.exit(2) + + # proceed to update the defined outfile + with open(outfile, "w", encoding="utf-8") as new_local_file: + yml.dump(remote_file, new_local_file) diff --git a/src/blimp/cli.py b/src/blimp/cli.py new file mode 100644 index 0000000..b462918 --- /dev/null +++ b/src/blimp/cli.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import argparse + +from .main import BlacklistImporter + + +def cli(): + parser = argparse.ArgumentParser() + parser.add_argument("-out", "--outfile", help="set path to output file", action="store", default=None) + parser.add_argument("-dr", "--dry-run", help="perform a dry run", action="store_true", default=False) + args = parser.parse_args() + + # run + BlacklistImporter(args).main() diff --git a/src/blimp/main.py b/src/blimp/main.py new file mode 100644 index 0000000..5d4c3ac --- /dev/null +++ b/src/blimp/main.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import argparse + +import requests +from appdirs import * + +from .bl_process import ProcessBlocklist +from .misc import * + + +class BlacklistImporter: + def __init__(self, args): + self.outfile = args.outfile + self.dryrun = args.dry_run + self.path = Path(user_cache_dir("blimp")) + self.url = "https://raw.githubusercontent.com/JabberSPAM/blacklist/master/blacklist.txt" + self.blacklist = "" + self.apply_changes = False + + self.etag_path = self.path.joinpath(".etag") + self.blacklist_path = self.path.joinpath("blacklist.txt") + + def cache_dir_check(self): + if not self.path.is_dir(): + Path(self.path).mkdir(parents=True, exist_ok=True) + + def download_required(self, etag) -> bool: + """ + method to determine if a new download should be initiated + :param etag: requests etag object + :return: true if download is required + """ + # always trigger download if any local cache file is missing + if not local_file_present(self.blacklist_path): + return True + + if not local_file_present(self.etag_path): + return True + + # etag file is present but outdated + else: + with open(self.etag_path, "r") as local_file: + local_etag = local_file.read() + + if local_etag != etag: + return True + + return False + + def start_request(self): + """ + determine if the download is required + """ + with requests.Session() as s: + # head request to check etag + head = s.head(self.url) + etag = head.headers["etag"] + + if head.status_code != requests.codes.ok: + return + + if not self.download_required(etag): + with open(self.blacklist_path, "r", encoding="utf-8") as local_file: + self.blacklist = local_file.read() + + else: + r = s.get(self.url) + r.encoding = "utf-8" + local_etag = head.headers["etag"] + self.blacklist = r.content.decode() + + with open(self.blacklist_path, "w", encoding="utf-8") as local_file: + local_file.write(self.blacklist) + + with open(self.etag_path, "w", encoding="utf-8") as local_file: + local_file.write(local_etag) + + def main(self): + # check the cache dir first + self.cache_dir_check() + + # only output the selected outfile + if self.dryrun: + print("outfile selected: %s" % self.outfile) + + # go + self.start_request() + + # blacklist processing + ProcessBlocklist().process(self.blacklist, self.outfile, self.dryrun) + + """# reload config if changes have been applied + if self.change: + # catch ejabberdctl missing + if Path("/usr/sbin/ejabberdctl").is_file(): + subprocess.call(["/usr/sbin/ejabberdctl", "reload_config"], shell=False) + + # report missing ejabberdctl reload_config + else: + print("/usr/sbin/ejabberdctl was not found", file=sys.stderr) + print("blacklist changes have been applied\nejabberd config was not reloaded", file=sys.stderr) + sys.exit(1) +""" + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("-out", "--outfile", help="set path to output file", action="store", default=None) + parser.add_argument("-dr", "--dry-run", help="perform a dry run", action="store_true", default=False) + args = parser.parse_args() + + # run + BlacklistImporter(args).main() diff --git a/src/blimp/misc.py b/src/blimp/misc.py new file mode 100644 index 0000000..d25c8d7 --- /dev/null +++ b/src/blimp/misc.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +from pathlib import Path + + +def local_file_present(somepath) -> bool: + """ + check local etag copy + :return: true if present + """ + if not Path(somepath).is_file(): + return False + + return True + +def asd(): + print("yo") From c9e689b3f3d043933ea953448832c8cc718cd28f Mon Sep 17 00:00:00 2001 From: nico wellpott Date: Sun, 28 Mar 2021 16:39:58 +0200 Subject: [PATCH 2/6] packaging: various smaller fixes and corrections - remove placeholder print statements * replace wildcard import with a direct one * rename main class to blimp * optimized imports --- src/blimp/__init__.py | 6 +++--- src/blimp/bl_process.py | 7 +++++-- src/blimp/cli.py | 4 ++-- src/blimp/main.py | 22 ++++++++-------------- src/blimp/misc.py | 5 +---- 5 files changed, 19 insertions(+), 25 deletions(-) diff --git a/src/blimp/__init__.py b/src/blimp/__init__.py index 5486de7..769e3ce 100644 --- a/src/blimp/__init__.py +++ b/src/blimp/__init__.py @@ -4,11 +4,11 @@ # version __version__ = "0.1" -# main -from .main import BlacklistImporter - # modules from .bl_process import ProcessBlocklist +# main +from .main import Blimp + # utils from .misc import * diff --git a/src/blimp/bl_process.py b/src/blimp/bl_process.py index e1d8c93..2e56275 100644 --- a/src/blimp/bl_process.py +++ b/src/blimp/bl_process.py @@ -6,7 +6,12 @@ from ruamel.yaml import YAML, scalarstring from .misc import * + class ProcessBlocklist: + def __init__(self): + pass + + @classmethod def process(self, blacklist, outfile, dryrun: bool): """ function to build and compare the local yaml file to the remote file @@ -19,8 +24,6 @@ class ProcessBlocklist: with open(outfile, "r", encoding="utf-8") as local_file: local_blacklist = local_file.read() - print("step local file") - except TypeError: # no local copy use empty one instead local_blacklist = YAML(typ="safe") diff --git a/src/blimp/cli.py b/src/blimp/cli.py index b462918..12bd1dc 100644 --- a/src/blimp/cli.py +++ b/src/blimp/cli.py @@ -3,7 +3,7 @@ import argparse -from .main import BlacklistImporter +from .main import Blimp def cli(): @@ -13,4 +13,4 @@ def cli(): args = parser.parse_args() # run - BlacklistImporter(args).main() + Blimp(args).main() diff --git a/src/blimp/main.py b/src/blimp/main.py index 5d4c3ac..4189bbc 100644 --- a/src/blimp/main.py +++ b/src/blimp/main.py @@ -1,16 +1,14 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -import argparse - import requests -from appdirs import * +from appdirs import user_cache_dir from .bl_process import ProcessBlocklist from .misc import * -class BlacklistImporter: +class Blimp: def __init__(self, args): self.outfile = args.outfile self.dryrun = args.dry_run @@ -89,9 +87,10 @@ class BlacklistImporter: self.start_request() # blacklist processing - ProcessBlocklist().process(self.blacklist, self.outfile, self.dryrun) + ProcessBlocklist.process(self.blacklist, self.outfile, self.dryrun) - """# reload config if changes have been applied + """ + # reload config if changes have been applied if self.change: # catch ejabberdctl missing if Path("/usr/sbin/ejabberdctl").is_file(): @@ -102,14 +101,9 @@ class BlacklistImporter: print("/usr/sbin/ejabberdctl was not found", file=sys.stderr) print("blacklist changes have been applied\nejabberd config was not reloaded", file=sys.stderr) sys.exit(1) -""" + """ if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("-out", "--outfile", help="set path to output file", action="store", default=None) - parser.add_argument("-dr", "--dry-run", help="perform a dry run", action="store_true", default=False) - args = parser.parse_args() - - # run - BlacklistImporter(args).main() + from .cli import cli + cli() diff --git a/src/blimp/misc.py b/src/blimp/misc.py index d25c8d7..528ed5b 100644 --- a/src/blimp/misc.py +++ b/src/blimp/misc.py @@ -5,13 +5,10 @@ from pathlib import Path def local_file_present(somepath) -> bool: """ - check local etag copy + check if a given local filepath exists :return: true if present """ if not Path(somepath).is_file(): return False return True - -def asd(): - print("yo") From eda0a709c9bc0eb42f6fccce4e00285dfff60bd9 Mon Sep 17 00:00:00 2001 From: nico wellpott Date: Tue, 6 Apr 2021 11:23:09 +0200 Subject: [PATCH 3/6] cleanup: make flake8 happy again * remove wildcard import * remove unnecessary imports * various smaller whitespace cleanups --- src/blimp/__init__.py | 9 --------- src/blimp/bl_process.py | 2 +- src/blimp/main.py | 20 ++++---------------- 3 files changed, 5 insertions(+), 26 deletions(-) diff --git a/src/blimp/__init__.py b/src/blimp/__init__.py index 769e3ce..19c2b68 100644 --- a/src/blimp/__init__.py +++ b/src/blimp/__init__.py @@ -3,12 +3,3 @@ # version __version__ = "0.1" - -# modules -from .bl_process import ProcessBlocklist - -# main -from .main import Blimp - -# utils -from .misc import * diff --git a/src/blimp/bl_process.py b/src/blimp/bl_process.py index 2e56275..e0db3da 100644 --- a/src/blimp/bl_process.py +++ b/src/blimp/bl_process.py @@ -4,7 +4,7 @@ import sys from ruamel.yaml import YAML, scalarstring -from .misc import * +from .misc import local_file_present class ProcessBlocklist: diff --git a/src/blimp/main.py b/src/blimp/main.py index 4189bbc..fccf9af 100644 --- a/src/blimp/main.py +++ b/src/blimp/main.py @@ -1,11 +1,13 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- +from pathlib import Path + import requests from appdirs import user_cache_dir from .bl_process import ProcessBlocklist -from .misc import * +from .misc import local_file_present class Blimp: @@ -89,21 +91,7 @@ class Blimp: # blacklist processing ProcessBlocklist.process(self.blacklist, self.outfile, self.dryrun) - """ - # reload config if changes have been applied - if self.change: - # catch ejabberdctl missing - if Path("/usr/sbin/ejabberdctl").is_file(): - subprocess.call(["/usr/sbin/ejabberdctl", "reload_config"], shell=False) - - # report missing ejabberdctl reload_config - else: - print("/usr/sbin/ejabberdctl was not found", file=sys.stderr) - print("blacklist changes have been applied\nejabberd config was not reloaded", file=sys.stderr) - sys.exit(1) - """ - if __name__ == "__main__": - from .cli import cli + from .cli import cli cli() From 1b6dd65afe8f155fbc3fb0b0c71cb4775a716d28 Mon Sep 17 00:00:00 2001 From: nico wellpott Date: Tue, 6 Apr 2021 11:24:29 +0200 Subject: [PATCH 4/6] requirements: push ruamel.yaml to 0.17.2 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 288c070..0ee9f3a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ appdirs==1.4.4 requests==2.25.1 -ruamel.yaml==0.17.0 +ruamel.yaml==0.17.2 ruamel.yaml.clib==0.2.2 urllib3==1.26.4 From d410c454297f8530fc94f37ac0f99bc0093f2b89 Mon Sep 17 00:00:00 2001 From: nico wellpott Date: Tue, 6 Apr 2021 11:28:37 +0200 Subject: [PATCH 5/6] flake8: remove cyclic import --- src/blimp/bl_process.py | 1 - src/blimp/misc.py | 1 - 2 files changed, 2 deletions(-) diff --git a/src/blimp/bl_process.py b/src/blimp/bl_process.py index e0db3da..06c9c0f 100644 --- a/src/blimp/bl_process.py +++ b/src/blimp/bl_process.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # -*- coding: utf-8 -*- import sys diff --git a/src/blimp/misc.py b/src/blimp/misc.py index 528ed5b..7242e3f 100644 --- a/src/blimp/misc.py +++ b/src/blimp/misc.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # -*- coding: utf-8 -*- from pathlib import Path From 78015c196afd5f1ae8c3d65348e0059dddcce6c5 Mon Sep 17 00:00:00 2001 From: nico wellpott Date: Tue, 6 Apr 2021 11:29:24 +0200 Subject: [PATCH 6/6] flake8: unnecessary else * modified method to not use an unnecessary else statement --- src/blimp/main.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/blimp/main.py b/src/blimp/main.py index fccf9af..93fdc29 100644 --- a/src/blimp/main.py +++ b/src/blimp/main.py @@ -39,13 +39,12 @@ class Blimp: if not local_file_present(self.etag_path): return True - # etag file is present but outdated - else: - with open(self.etag_path, "r") as local_file: - local_etag = local_file.read() + with open(self.etag_path, "r") as local_file: + local_etag = local_file.read() - if local_etag != etag: - return True + # etag file is present but outdated + if local_etag != etag: + return True return False