added type hints; cleaner shutdown; add epoch parsing

master.py

@@ -12,7 +12,6 @@ import subprocess
 import sys
 import time
 from multiprocessing import Pool, current_process, Lock, JoinableQueue
-from queue import Empty
 
 import yaml
 from humanfriendly import format_timespan
@@ -21,6 +20,7 @@ from packaging.version import LegacyVersion
 
 regex_pkgver = re.compile(r"^_?pkgver\s*=\s*(.+)$", re.MULTILINE)
 regex_pkgrel = re.compile(r"^pkgrel\s*=\s*(.+)$", re.MULTILINE)
+regex_epoch = re.compile(r"^epoch\s*=\s*(.+)$", re.MULTILINE)
 regex_march = re.compile(r"(-march=)(.+?) ", re.MULTILINE)
 regex_validkeys = re.compile(r"^validpgpkeys\+?=\((.*?)\)", re.MULTILINE | re.DOTALL)
 fp = None
@@ -28,7 +28,7 @@ update_last = time.time()
 repo_lock = Lock()
 
 
-def build(pkgbuild, repo):
+def build(pkgbuild, repo) -> None:
     start_time = time.time()
     name = pathlib.Path(pkgbuild).parts[-4]
     process_name = current_process().name
@@ -119,7 +119,7 @@ def run_worker() -> None:
             os.chdir(sys.path[0])
 
 
-def already_running():
+def already_running() -> bool:
     global fp
     fp = os.open(f"/tmp/alhp.lock", os.O_WRONLY | os.O_CREAT)
 
@@ -130,7 +130,7 @@ def already_running():
         return True
 
 
-def find_all_files_for_pkg(name, repo):
+def find_all_files_for_pkg(name, repo) -> list:
     searchpath = os.path.join(config["basedir"]["repo"], repo, "os", config["arch"]) + "/" + name + "-*.pkg.*"
     pkgs = glob.glob(searchpath)
 
@@ -141,7 +141,7 @@ def find_all_files_for_pkg(name, repo):
     return pkgs
 
 
-def get_failed_packages(repo):
+def get_failed_packages(repo) -> list:
     if os.path.exists(os.path.join(config["basedir"]["repo"], repo + "_failed.txt")):
         with open(os.path.join(config["basedir"]["repo"], repo + "_failed.txt")) as p:
             return p.read().splitlines()
@@ -158,7 +158,7 @@ def setup_chroot():
                           stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout.decode(errors='ignore'))
 
 
-def setup_makepkg(repo):
+def setup_makepkg(repo) -> None:
     makepkg_repo = os.path.join(config["basedir"]["makepkg"], "makepkg-" + '-'.join(repo.split("-")[1:]) + ".conf")
 
     if not os.path.exists(makepkg_repo):
@@ -174,7 +174,7 @@ def setup_makepkg(repo):
             conf.write(c_all)
 
 
-def import_keys(pkgbuild):
+def import_keys(pkgbuild) -> None:
     with open(pkgbuild, errors='ignore') as pkgb:
         keys_s = regex_validkeys.findall(pkgb.read())
 
@@ -196,13 +196,13 @@ def import_keys(pkgbuild):
                 logging.info("[GPG] Imported key %s", k)
 
 
-def package_exists(name, repo):
+def package_exists(name, repo) -> bool:
     pkgs = find_all_files_for_pkg(name, repo)
 
     return len(pkgs) > 0
 
 
-def update_svn2git():
+def update_svn2git() -> None:
     if not os.path.exists(config["basedir"]["upstream"]):
         pathlib.Path(config["basedir"]["upstream"]).mkdir(parents=True, exist_ok=True)
 
@@ -223,20 +223,23 @@ def update_svn2git():
             os.chdir(sys.path[0])
 
 
-def parse_pkgbuild(pkgbuild_file):
+def parse_pkgbuild(pkgbuild_file) -> LegacyVersion:
     with open(pkgbuild_file, errors='ignore') as p:
         pkgbuild_str = p.read()
 
     pkgver = regex_pkgver.findall(pkgbuild_str)
     pkgrel = regex_pkgrel.findall(pkgbuild_str)
+    epoch = regex_epoch.findall(pkgbuild_str)
     if not pkgver or not pkgrel:
         logging.warning("[%s] Failed to parse pkgbuild", pkgbuild_file.split("/")[-4])
         return version.parse("")
 
+    if epoch:
+        return LegacyVersion("{}:{}-{}".format(epoch[0], pkgver[0], pkgrel[0]))
     return LegacyVersion("{}-{}".format(pkgver[0], pkgrel[0]))
 
 
-def increase_pkgrel(pkgbuild_file):
+def increase_pkgrel(pkgbuild_file) -> None:
     with open(pkgbuild_file, errors='ignore') as p:
         pkgbuild_str = p.read()
 
@@ -246,13 +249,13 @@ def increase_pkgrel(pkgbuild_file):
         pkg.write(pkgbuild_str)
 
 
-def parse_repo(name, repo):
+def parse_repo(name, repo) -> LegacyVersion:
     ver_split = find_all_files_for_pkg(name, repo)[0].split("-")
 
     return LegacyVersion(ver_split[-3] + "-" + ver_split[-2])
 
 
-def sync_marchs_with_config():
+def sync_marchs_with_config() -> None:
     repos = []
     with os.scandir(config["basedir"]["repo"]) as it:
         entry: os.DirEntry
@@ -280,7 +283,7 @@ def sync_marchs_with_config():
         os.remove(os.path.join(config["basedir"]["makepkg"], "makepkg-" + repo + ".conf"))
 
 
-def fill_queue():
+def fill_queue() -> None:
     all_pkgbuild = []
 
     for git_dir, git_url in config["svn2git"].items():
@@ -351,32 +354,16 @@ if __name__ == '__main__':
 
         while True:
             try:
-                du = shutil.disk_usage(config["basedir"]["upstream"])
+                if time.time() - update_last > 900 and q.qsize() == 0:
-                if (du[1] / du[0]) > 0.9:
-                    logging.warning("Less then 10% disk space remaining, performing cleanup...")
-
-                    while not q.empty():
-                        try:
-                            q.get(False)
-                            q.task_done()
-                        except Empty:
-                            continue
-                    logging.info("Waiting for remaining queue items to finish...")
-                    q.join()
-
-                    logging.info("Cleared Queue, clearing upstream repos...")
-
-                    update_svn2git()
-                    logging.info("Cleanup done, refill queue")
-                    fill_queue()
-                    time.sleep(60)
-                elif time.time() - update_last > 900 and q.qsize() == 0:
                     update_last = time.time()
                     update_svn2git()
                     fill_queue()
+                    if q.qsize() > 0:
+                        logging.info("New Queue size: %d", q.qsize())
                 else:
                     time.sleep(60)
             except KeyboardInterrupt:
+                repo_lock.acquire()
                 pool.close()
                 pool.terminate()
                 q.close()