From d1789f8d9648be6d4d79bd5c275961e70c3ff59c Mon Sep 17 00:00:00 2001 From: Giovanni Harting <539@idlegandalf.com> Date: Mon, 24 May 2021 21:57:26 +0200 Subject: [PATCH] added type hints; cleaner shutdown; add epoch parsing --- master.py | 55 +++++++++++++++++++++---------------------------------- 1 file changed, 21 insertions(+), 34 deletions(-) diff --git a/master.py b/master.py index 08d3e38..eec2652 100644 --- a/master.py +++ b/master.py @@ -12,7 +12,6 @@ import subprocess import sys import time from multiprocessing import Pool, current_process, Lock, JoinableQueue -from queue import Empty import yaml from humanfriendly import format_timespan @@ -21,6 +20,7 @@ from packaging.version import LegacyVersion regex_pkgver = re.compile(r"^_?pkgver\s*=\s*(.+)$", re.MULTILINE) regex_pkgrel = re.compile(r"^pkgrel\s*=\s*(.+)$", re.MULTILINE) +regex_epoch = re.compile(r"^epoch\s*=\s*(.+)$", re.MULTILINE) regex_march = re.compile(r"(-march=)(.+?) ", re.MULTILINE) regex_validkeys = re.compile(r"^validpgpkeys\+?=\((.*?)\)", re.MULTILINE | re.DOTALL) fp = None @@ -28,7 +28,7 @@ update_last = time.time() repo_lock = Lock() -def build(pkgbuild, repo): +def build(pkgbuild, repo) -> None: start_time = time.time() name = pathlib.Path(pkgbuild).parts[-4] process_name = current_process().name @@ -119,7 +119,7 @@ def run_worker() -> None: os.chdir(sys.path[0]) -def already_running(): +def already_running() -> bool: global fp fp = os.open(f"/tmp/alhp.lock", os.O_WRONLY | os.O_CREAT) @@ -130,7 +130,7 @@ def already_running(): return True -def find_all_files_for_pkg(name, repo): +def find_all_files_for_pkg(name, repo) -> list: searchpath = os.path.join(config["basedir"]["repo"], repo, "os", config["arch"]) + "/" + name + "-*.pkg.*" pkgs = glob.glob(searchpath) @@ -141,7 +141,7 @@ def find_all_files_for_pkg(name, repo): return pkgs -def get_failed_packages(repo): +def get_failed_packages(repo) -> list: if os.path.exists(os.path.join(config["basedir"]["repo"], repo + "_failed.txt")): with open(os.path.join(config["basedir"]["repo"], repo + "_failed.txt")) as p: return p.read().splitlines() @@ -158,7 +158,7 @@ def setup_chroot(): stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout.decode(errors='ignore')) -def setup_makepkg(repo): +def setup_makepkg(repo) -> None: makepkg_repo = os.path.join(config["basedir"]["makepkg"], "makepkg-" + '-'.join(repo.split("-")[1:]) + ".conf") if not os.path.exists(makepkg_repo): @@ -174,7 +174,7 @@ def setup_makepkg(repo): conf.write(c_all) -def import_keys(pkgbuild): +def import_keys(pkgbuild) -> None: with open(pkgbuild, errors='ignore') as pkgb: keys_s = regex_validkeys.findall(pkgb.read()) @@ -196,13 +196,13 @@ def import_keys(pkgbuild): logging.info("[GPG] Imported key %s", k) -def package_exists(name, repo): +def package_exists(name, repo) -> bool: pkgs = find_all_files_for_pkg(name, repo) return len(pkgs) > 0 -def update_svn2git(): +def update_svn2git() -> None: if not os.path.exists(config["basedir"]["upstream"]): pathlib.Path(config["basedir"]["upstream"]).mkdir(parents=True, exist_ok=True) @@ -223,20 +223,23 @@ def update_svn2git(): os.chdir(sys.path[0]) -def parse_pkgbuild(pkgbuild_file): +def parse_pkgbuild(pkgbuild_file) -> LegacyVersion: with open(pkgbuild_file, errors='ignore') as p: pkgbuild_str = p.read() pkgver = regex_pkgver.findall(pkgbuild_str) pkgrel = regex_pkgrel.findall(pkgbuild_str) + epoch = regex_epoch.findall(pkgbuild_str) if not pkgver or not pkgrel: logging.warning("[%s] Failed to parse pkgbuild", pkgbuild_file.split("/")[-4]) return version.parse("") + if epoch: + return LegacyVersion("{}:{}-{}".format(epoch[0], pkgver[0], pkgrel[0])) return LegacyVersion("{}-{}".format(pkgver[0], pkgrel[0])) -def increase_pkgrel(pkgbuild_file): +def increase_pkgrel(pkgbuild_file) -> None: with open(pkgbuild_file, errors='ignore') as p: pkgbuild_str = p.read() @@ -246,13 +249,13 @@ def increase_pkgrel(pkgbuild_file): pkg.write(pkgbuild_str) -def parse_repo(name, repo): +def parse_repo(name, repo) -> LegacyVersion: ver_split = find_all_files_for_pkg(name, repo)[0].split("-") return LegacyVersion(ver_split[-3] + "-" + ver_split[-2]) -def sync_marchs_with_config(): +def sync_marchs_with_config() -> None: repos = [] with os.scandir(config["basedir"]["repo"]) as it: entry: os.DirEntry @@ -280,7 +283,7 @@ def sync_marchs_with_config(): os.remove(os.path.join(config["basedir"]["makepkg"], "makepkg-" + repo + ".conf")) -def fill_queue(): +def fill_queue() -> None: all_pkgbuild = [] for git_dir, git_url in config["svn2git"].items(): @@ -351,32 +354,16 @@ if __name__ == '__main__': while True: try: - du = shutil.disk_usage(config["basedir"]["upstream"]) - if (du[1] / du[0]) > 0.9: - logging.warning("Less then 10% disk space remaining, performing cleanup...") - - while not q.empty(): - try: - q.get(False) - q.task_done() - except Empty: - continue - logging.info("Waiting for remaining queue items to finish...") - q.join() - - logging.info("Cleared Queue, clearing upstream repos...") - - update_svn2git() - logging.info("Cleanup done, refill queue") - fill_queue() - time.sleep(60) - elif time.time() - update_last > 900 and q.qsize() == 0: + if time.time() - update_last > 900 and q.qsize() == 0: update_last = time.time() update_svn2git() fill_queue() + if q.qsize() > 0: + logging.info("New Queue size: %d", q.qsize()) else: time.sleep(60) except KeyboardInterrupt: + repo_lock.acquire() pool.close() pool.terminate() q.close()