Merge branch 'packaging'

This commit is contained in:
nico 2021-04-06 11:32:04 +02:00
commit e54cb3b215
Signed by: mightyBroccoli
GPG Key ID: 1F6495B44DABCE6E
10 changed files with 244 additions and 154 deletions

3
MANIFEST.in Normal file
View File

@ -0,0 +1,3 @@
include README.md
include LICENCE.md
include requirements.txt

152
main.py
View File

@ -1,152 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import argparse
import os
import subprocess
import sys
import requests
from ruamel.yaml import YAML, scalarstring
class BlacklistImporter:
def __init__(self, args):
self.outfile = args.outfile
self.dryrun = args.dryrun
self.path = os.path.dirname(__file__)
self.url = "https://raw.githubusercontent.com/JabberSPAM/blacklist/master/blacklist.txt"
self.blacklist = ""
self.change = False
def request(self):
"""
determine if the download is required
"""
etag_path = "/".join([self.path, ".etag"])
blacklist_path = "/".join([self.path, "blacklist.txt"])
# check if etag header is present if not set local_etag to ""
if os.path.isfile(etag_path):
# catch special case were etag file is present and blacklist.txt is not
if not os.path.isfile(blacklist_path):
local_etag = ""
else:
# if both files are present continue normally
with open(etag_path, "r") as local_file:
local_etag = local_file.read()
else:
local_etag = ""
with requests.Session() as s:
# head request to check etag
head = s.head(self.url)
etag = head.headers['etag']
# if etags match up or if the connection is not possible fall back to local cache
if local_etag == etag or head.status_code != 200:
# if local cache is present overwrite blacklist var
if os.path.isfile(blacklist_path):
with open(blacklist_path, "r", encoding="utf-8") as local_file:
self.blacklist = local_file.read()
# in any other case request a new file
else:
r = s.get(self.url)
r.encoding = 'utf-8'
local_etag = head.headers['etag']
self.blacklist = r.content.decode()
with open(blacklist_path, "w") as local_file:
local_file.write(self.blacklist)
with open(etag_path, 'w') as local_file:
local_file.write(local_etag)
def main(self):
# first check if blacklist is updated
self.request()
# only output the selected outfile
if self.dryrun:
print("outfile selected: %s" % self.outfile)
# blacklist processing
self.process()
# reload config if changes have been applied
if self.change:
# catch ejabberdctl missing
if os.path.isfile('/usr/sbin/ejabberdctl'):
subprocess.call(['/usr/sbin/ejabberdctl', 'reload_config'], shell=False)
# report missing ejabberdctl reload_config
else:
print('/usr/sbin/ejabberdctl was not found', file=sys.stderr)
print('blacklist changes have been applied\nejabberd config was not reloaded', file=sys.stderr)
sys.exit(1)
def process(self):
"""
function to build and compare the local yaml file to the remote file
if the remote file is different, the local file gets overwritten
"""
# init new YAML variable
local_file = YAML(typ="safe")
# None catch
if self.outfile is not None:
# prevent FileNotFoundError on first run or file missing
if os.path.isfile(self.outfile):
local_file = local_file.load(open(self.outfile, "r", encoding="utf-8"))
# blacklist frame
remote_file = {
"acl": {
"spamblacklist": {
"server": []
}
}
}
# build the blacklist with the given frame to compare to local blacklist
for entry in self.blacklist.split():
entry = scalarstring.DoubleQuotedScalarString(entry)
remote_file["acl"]["spamblacklist"]["server"].append(entry)
yml = YAML()
yml.indent(offset=2)
yml.default_flow_style = False
# if dry-run true print expected content
if self.dryrun:
yml.dump(remote_file, sys.stdout)
# only if the local_file and remote_file are unequal write new file
elif local_file != remote_file:
# prevent FileNotFoundError if self.outfile is not assigned
if self.outfile is None:
print("no outfile assigned", file=sys.stderr)
print(parser.format_help(), file=sys.stderr)
sys.exit(2)
# proceed to update the defined outfile
elif self.outfile is not None:
self.change = True
yml.dump(remote_file, open(self.outfile, "w"))
# if that's impossible break and display help message
else:
print(parser.format_help(), file=sys.stderr)
sys.exit(1)
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument('-o', '--outfile', help='set path to output file', dest='outfile', default=None)
parser.add_argument('-dr', '--dry-run', help='perform a dry run', action='store_true', dest='dryrun', default=False)
args = parser.parse_args()
# run
BlacklistImporter(args).main()

View File

@ -1,4 +1,5 @@
appdirs==1.4.4
requests==2.25.1 requests==2.25.1
ruamel.yaml==0.16.13 ruamel.yaml==0.17.2
ruamel.yaml.clib==0.2.2 ruamel.yaml.clib==0.2.2
urllib3==1.26.3 urllib3==1.26.4

39
setup.cfg Normal file
View File

@ -0,0 +1,39 @@
[metadata]
name = blimp
version = attr: blimp.__version__
url = https://github.com/mightyBroccoli/blacklist_importer
licence = GPLv3
author = Nico Wellpott
author_email = nico@magicbroccoli.de
description = simple tool to download and update the JabberSpam blacklist list.
long_description = file: README.md
long_description_content_type = text/markdown
classifiers =
Development Status :: 4 - Beta
Intended Audience :: System Administrators
Licence :: OSI Approved :: GNU General Public License v3 (GPLv3)
Operating System :: Unix
Programming Language :: Python :: 3
Programming Language :: Python :: 3.7
Programming Language :: Python :: 3.8
Programming Language :: Python :: 3.9
Programming Language :: Python :: Implementation :: CPython
Topic :: Internet :: WWW/HTTP :: Dynamic Content
[options]
packages = find:
package_dir = = src
include_package_data = true
python_requires = >= 3.7
[options.packages.find]
where = src
[options.entry_points]
console_scripts =
blimp = blimp.cli:cli
[tool:pytest]
testpaths = tests
filterwarnings =
error

13
setup.py Normal file
View File

@ -0,0 +1,13 @@
# -*- coding: utf-8 -*-
from setuptools import setup
setup(
name="Blimp",
install_requires=[
"appdirs>=1.4",
"requests>=2.25",
"ruamel.yaml>=0.17",
"ruamel.yaml.clib>=0.2",
"urllib3>=1.26"
]
)

5
src/blimp/__init__.py Normal file
View File

@ -0,0 +1,5 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# version
__version__ = "0.1"

56
src/blimp/bl_process.py Normal file
View File

@ -0,0 +1,56 @@
# -*- coding: utf-8 -*-
import sys
from ruamel.yaml import YAML, scalarstring
from .misc import local_file_present
class ProcessBlocklist:
def __init__(self):
pass
@classmethod
def process(self, blacklist, outfile, dryrun: bool):
"""
function to build and compare the local yaml file to the remote file
if the remote file is different, the local file gets overwritten
"""
# cheeky none catch
try:
# load local blacklist outfile
if local_file_present(outfile):
with open(outfile, "r", encoding="utf-8") as local_file:
local_blacklist = local_file.read()
except TypeError:
# no local copy use empty one instead
local_blacklist = YAML(typ="safe")
# blacklist frame
remote_file = {"acl": {"spamblacklist": {"server": []}}}
# build the blacklist with the given frame to compare to local blacklist
for entry in blacklist.split():
entry = scalarstring.DoubleQuotedScalarString(entry)
remote_file["acl"]["spamblacklist"]["server"].append(entry)
yml = YAML()
yml.indent(offset=2)
yml.default_flow_style = False
# if dry-run true print expected content
if dryrun:
yml.dump(remote_file, sys.stdout)
return
if local_blacklist == remote_file:
return
if outfile is None:
print("no outfile assigned", file=sys.stderr)
sys.exit(2)
# proceed to update the defined outfile
with open(outfile, "w", encoding="utf-8") as new_local_file:
yml.dump(remote_file, new_local_file)

16
src/blimp/cli.py Normal file
View File

@ -0,0 +1,16 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import argparse
from .main import Blimp
def cli():
parser = argparse.ArgumentParser()
parser.add_argument("-out", "--outfile", help="set path to output file", action="store", default=None)
parser.add_argument("-dr", "--dry-run", help="perform a dry run", action="store_true", default=False)
args = parser.parse_args()
# run
Blimp(args).main()

96
src/blimp/main.py Normal file
View File

@ -0,0 +1,96 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from pathlib import Path
import requests
from appdirs import user_cache_dir
from .bl_process import ProcessBlocklist
from .misc import local_file_present
class Blimp:
def __init__(self, args):
self.outfile = args.outfile
self.dryrun = args.dry_run
self.path = Path(user_cache_dir("blimp"))
self.url = "https://raw.githubusercontent.com/JabberSPAM/blacklist/master/blacklist.txt"
self.blacklist = ""
self.apply_changes = False
self.etag_path = self.path.joinpath(".etag")
self.blacklist_path = self.path.joinpath("blacklist.txt")
def cache_dir_check(self):
if not self.path.is_dir():
Path(self.path).mkdir(parents=True, exist_ok=True)
def download_required(self, etag) -> bool:
"""
method to determine if a new download should be initiated
:param etag: requests etag object
:return: true if download is required
"""
# always trigger download if any local cache file is missing
if not local_file_present(self.blacklist_path):
return True
if not local_file_present(self.etag_path):
return True
with open(self.etag_path, "r") as local_file:
local_etag = local_file.read()
# etag file is present but outdated
if local_etag != etag:
return True
return False
def start_request(self):
"""
determine if the download is required
"""
with requests.Session() as s:
# head request to check etag
head = s.head(self.url)
etag = head.headers["etag"]
if head.status_code != requests.codes.ok:
return
if not self.download_required(etag):
with open(self.blacklist_path, "r", encoding="utf-8") as local_file:
self.blacklist = local_file.read()
else:
r = s.get(self.url)
r.encoding = "utf-8"
local_etag = head.headers["etag"]
self.blacklist = r.content.decode()
with open(self.blacklist_path, "w", encoding="utf-8") as local_file:
local_file.write(self.blacklist)
with open(self.etag_path, "w", encoding="utf-8") as local_file:
local_file.write(local_etag)
def main(self):
# check the cache dir first
self.cache_dir_check()
# only output the selected outfile
if self.dryrun:
print("outfile selected: %s" % self.outfile)
# go
self.start_request()
# blacklist processing
ProcessBlocklist.process(self.blacklist, self.outfile, self.dryrun)
if __name__ == "__main__":
from .cli import cli
cli()

13
src/blimp/misc.py Normal file
View File

@ -0,0 +1,13 @@
# -*- coding: utf-8 -*-
from pathlib import Path
def local_file_present(somepath) -> bool:
"""
check if a given local filepath exists
:return: true if present
"""
if not Path(somepath).is_file():
return False
return True