delete packages failing to build or not meant to build

master.py

@@ -13,15 +13,15 @@ import sys
 import time
 import traceback
 from multiprocessing import Pool, current_process, JoinableQueue, Lock
+from typing import AnyStr
 
 import yaml
 from humanfriendly import format_timespan
 from packaging.version import LegacyVersion
 
-from utils import parse_pkgbuild, parse_pkgbuild_ver, import_keys
+from utils import parse_pkgbuild, parse_pkgbuild_ver, import_keys, increase_pkgrel
 
 regex_pkgver = re.compile(r"^_?pkgver\s*=\s*(.+)$", re.MULTILINE)
-regex_pkgrel = re.compile(r"^pkgrel\s*=\s*(.+)$", re.MULTILINE)
 regex_epoch = re.compile(r"^epoch\s*=\s*(.+)$", re.MULTILINE)
 regex_march = re.compile(r"(-march=)(.+?) ", re.MULTILINE)
 regex_validkeys = re.compile(r"^validpgpkeys\+?=\((.*?)\)", re.MULTILINE | re.DOTALL)
@@ -142,23 +142,39 @@ def already_running() -> bool:
         return True
 
 
-def package_exists(names: list, repo) -> bool:
+def delete_package(parsed_pkgbuild: dict, repo: list[AnyStr]):
-    for name in names:
+    for pkg in parsed_pkgbuild["packages"]:
-        pkgs = find_all_files_for_pkg(name, repo)
+        pkg_f = find_package_files(pkg, repo)
-        if not pkgs:
+
-            return False
+        if pkg_f:
-    return True
+            for f in pkg_f:
+                base = pathlib.Path(f).parent
+                repo = f.split("/")[-3]
+
+                args = ["repo-remove", "-s", "-v", os.path.join(base, repo + ".db.tar.xz"), pkg]
+                r_res = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+                logging.debug("[REPO-REMOVE] %s", r_res.stdout.decode(errors="ignore"))
+                if r_res.returncode:
+                    logging.error("[REPO/%s] Repo action failed: %s", repo, r_res.stdout.decode(errors="ignore"))
+                    continue
+                os.remove(f)
+                os.remove(f + ".sig")
 
 
-def find_all_files_for_pkg(name: str, repo: str) -> list:
+def find_package_files(name: str, repo: list[AnyStr]) -> list[AnyStr]:
     pkgs = []
-    for root, dirs, files in os.walk(os.path.join(config["basedir"]["repo"], repo, "os", config["arch"])):
-        for file in files:
-            res = regex_pkg_repo.findall(file)
-            for r in res:
-                if r == name and not file.endswith(".sig"):
-                    pkgs.append(os.path.join(root, file))
 
+    for t_repo in repo:
+        files: list[str]
+        root: str
+        for root, dirs, files in os.walk(os.path.join(config["basedir"]["repo"], t_repo, "os", config["arch"])):
+            for file in files:
+                if file.endswith(".sig"):
+                    continue
+
+                res = regex_pkg_repo.search(file)
+                if res and res.group(1) == name:
+                    pkgs.append(os.path.join(root, file))
     return pkgs
 
 
@@ -222,19 +238,8 @@ def update_svn2git() -> None:
                                                      stderr=subprocess.STDOUT).stdout.decode(errors="ignore"))
 
 
-def increase_pkgrel(pkgbuild_file) -> None:
-    parsed = parse_pkgbuild(pkgbuild_file)
-    with open(pkgbuild_file, "r+", errors='ignore') as p:
-        pkgbuild_str = p.read()
-        p.truncate(0)
-        p.seek(0, 0)
-
-        pkgbuild_str = regex_pkgrel.sub("pkgrel=" + parsed["pkgrel"] + ".1", pkgbuild_str)
-        p.write(pkgbuild_str)
-
-
 def parse_repo(name, repo) -> LegacyVersion:
-    ver_split = find_all_files_for_pkg(name, repo)[0].split("-")
+    ver_split = find_package_files(name, repo)[0].split("-")
 
     return LegacyVersion(ver_split[-3] + "-" + ver_split[-2])
 
@@ -278,34 +283,37 @@ def fill_queue() -> None:
 
     for pkgbuild in all_pkgbuild:
         path_split = pkgbuild.split("/")
-        parsed = parse_pkgbuild(pkgbuild)
+        parsed_pkgb = parse_pkgbuild(pkgbuild)
 
         # ignore pkgbuild if in trunk, -any package, not in repos, on blacklist, not for current arch
-        if path_split[-2] == "trunk" or path_split[-2].split("-")[0] not in config["repos"] or "any" in parsed["arch"] \
+        if path_split[-2] == "trunk" or path_split[-2].split("-")[0] not in config["repos"] or "any" in parsed_pkgb[
-                or parsed["pkgbase"] in config["blacklist"] or "i686" in path_split[-2]:
+            "arch"] \
-            # TODO: delete packages not to build
+                or parsed_pkgb["pkgbase"] in config["blacklist"] or "i686" in path_split[-2]:
-            pass
+            delete_package(parsed_pkgb, repos)
         else:
             for march in config["march"]:
                 repo = path_split[-2].split("-")[0] + "-" + march
 
-                if parsed["pkgbase"] in get_failed_packages(repo):
+                if parsed_pkgb["pkgbase"] in get_failed_packages(repo):
-                    logging.info("[%s/%s] Skipped due to failing build", repo, parsed["pkgbase"])
+                    logging.info("[%s/%s] Skipped due to failing build", repo, parsed_pkgb["pkgbase"])
+                    delete_package(parsed_pkgb, [repo, ])
                     continue
 
-                ver = parse_pkgbuild_ver(parsed=parsed)
+                ver = parse_pkgbuild_ver(parsed=parsed_pkgb)
-                packages = list(parsed["packages"])
+                packages = list(parsed_pkgb["packages"])
-                if package_exists(packages, repo):
+                pkg_f = find_package_files(packages[0], [repo, ])
+                if pkg_f:
                     logging.debug("[SEMVER] Comparing %s=%s - %s=%s", packages[0], parse_repo(packages[0], repo),
                                   packages[0], ver)
                     rv = parse_repo(packages[0], repo)
                     if rv < ver:
                         q.put((pkgbuild, repo))
-                        logging.info("[%s/%s] Build queued (new version available %s < %s)", repo, parsed["pkgbase"],
+                        logging.info("[%s/%s] Build queued (new version available %s < %s)", repo,
+                                     parsed_pkgb["pkgbase"],
                                      rv, ver)
                 else:
                     q.put((pkgbuild, repo))
-                    logging.info("[%s/%s] Build queued (package not build yet)", repo, parsed["pkgbase"])
+                    logging.info("[%s/%s] Build queued (package not build yet)", repo, parsed_pkgb["pkgbase"])
 
     logging.info("Build queue size: %s", q.qsize())
 

utils.py

@@ -1,11 +1,14 @@
 import logging
 import os
 import pathlib
+import re
 import subprocess
 
 from packaging.version import LegacyVersion
 from srcinfo.parse import parse_srcinfo
 
+regex_pkgrel = re.compile(r"^pkgrel\s*=\s*(.+)$", re.MULTILINE)
+
 
 def import_keys(pkgbuild: str) -> bool:
     parsed = parse_pkgbuild(pkgbuild)
@@ -24,6 +27,17 @@ def import_keys(pkgbuild: str) -> bool:
     return True
 
 
+def increase_pkgrel(pkgbuild_file) -> None:
+    parsed = parse_pkgbuild(pkgbuild_file)
+    with open(pkgbuild_file, "r+", errors='ignore') as p:
+        pkgbuild_str = p.read()
+        p.truncate(0)
+        p.seek(0, 0)
+
+        pkgbuild_str = regex_pkgrel.sub("pkgrel=" + parsed["pkgrel"] + ".1", pkgbuild_str)
+        p.write(pkgbuild_str)
+
+
 def parse_pkgbuild(pkgbuild_file: str) -> dict:
     pkgbuild_path = pathlib.Path(pkgbuild_file)
     os.chdir(pkgbuild_path.parent)