added deprecation notice

README.md

@@ -1,3 +1,7 @@
+# Deprecation notice
+
+This was replaced with [ALHP.GO](https://git.harting.dev/anonfunc/ALHP.GO), please open any issues or PRs there.
+
 # alhp
 
 Build script for archlinux instructionset enabled repos.

config.yaml

@@ -21,9 +21,6 @@ blacklist:
   - pacman
   - tensorflow
   - tensorflow-cuda
-  - brotli
-  - libarchive
-  - libb2
   - gcc
 
 build:

master.py

@@ -13,15 +13,15 @@ import sys
 import time
 import traceback
 from multiprocessing import Pool, current_process, JoinableQueue, Lock
+from typing import AnyStr
 
 import yaml
 from humanfriendly import format_timespan
 from packaging.version import LegacyVersion
 
-from utils import parse_pkgbuild, parse_pkgbuild_ver, import_keys
+from utils import parse_pkgbuild, parse_pkgbuild_ver, import_keys, increase_pkgrel
 
 regex_pkgver = re.compile(r"^_?pkgver\s*=\s*(.+)$", re.MULTILINE)
-regex_pkgrel = re.compile(r"^pkgrel\s*=\s*(.+)$", re.MULTILINE)
 regex_epoch = re.compile(r"^epoch\s*=\s*(.+)$", re.MULTILINE)
 regex_march = re.compile(r"(-march=)(.+?) ", re.MULTILINE)
 regex_validkeys = re.compile(r"^validpgpkeys\+?=\((.*?)\)", re.MULTILINE | re.DOTALL)
@@ -29,12 +29,14 @@ regex_pkg_repo = re.compile(r"^(.*)-.*-.*-(?:x86_64|any)\.pkg\.tar\.zst(?:\.sig)
 fp = None
 update_last = time.time()
 copy_l = Lock()
+failed_l = Lock()
 repos = []
 
 
 def build(pkgbuild: str, repo: str) -> None:
     start_time = time.time()
-    name = pathlib.Path(pkgbuild).parts[-4]
+    parsed = parse_pkgbuild(pkgbuild)
+    name = parsed["pkgbase"]
     process_name = current_process().name
     logging.info("[%s/%s/%s] Build starting (Queue ~= %s)", process_name, repo, name, q.qsize())
 
@@ -45,7 +47,7 @@ def build(pkgbuild: str, repo: str) -> None:
     import_keys(pkgbuild)
 
     # increase pkgrel
-    increase_pkgrel(pkgbuild)
+    increase_pkgrel(pkgbuild, parsed)
 
     # build with devtools
     os.chdir(pathlib.Path(pkgbuild).parent)
@@ -58,8 +60,8 @@ def build(pkgbuild: str, repo: str) -> None:
         logging.warning("[%s/%s/%s] Build failed. Check repo/logs for more information.", process_name, repo, name)
 
         # write packagename to failed list
-        with open(os.path.join(config["basedir"]["repo"], repo + "_failed.txt"), "a") as f:
+        with failed_l, open(os.path.join(config["basedir"]["repo"], repo + "_failed.txt"), "a") as f:
-            f.write(name + "\n")
+            f.write(name + "==" + str(parse_pkgbuild_ver(parsed=parsed)) + "\n")
 
         # write logs
         if not os.path.exists(os.path.join(config["basedir"]["repo"], "logs", repo)):
@@ -96,6 +98,10 @@ def build(pkgbuild: str, repo: str) -> None:
     logging.debug("[GIT] %s", subprocess.run(["git", "clean", "-xdff"], check=False, stdout=subprocess.PIPE,
                                              stderr=subprocess.STDOUT).stdout.decode(errors="ignore"))
 
+    logpath = pathlib.Path(os.path.join(config["basedir"]["repo"], "logs", repo, name + ".log"))
+    if logpath.exists():
+        os.remove(logpath)
+
     logging.info("[%s/%s/%s] Build successful (%s)", process_name, repo, name,
                  format_timespan(time.time() - start_time))
 
@@ -110,7 +116,6 @@ def run_worker() -> None:
             traceback.print_exc()
         finally:
             q.task_done()
-            os.chdir(sys.path[0])
 
 
 def do_repo_work() -> None:
@@ -130,7 +135,6 @@ def do_repo_work() -> None:
         logging.debug("[PACCACHE] %s", p_res.stdout.decode(errors="ignore"))
         if p_res.returncode:
             logging.error("[REPO/%s] Repo cleanup failed: %s", repo, p_res.stdout.decode(errors="ignore"))
-        os.chdir(sys.path[0])
 
 
 def already_running() -> bool:
@@ -144,32 +148,70 @@ def already_running() -> bool:
         return True
 
 
-def package_exists(name, repo) -> bool:
+def delete_package(parsed_pkgbuild: dict, repo: list[AnyStr]):
-    pkgs = find_all_files_for_pkg(name, repo)
+    for pkg in parsed_pkgbuild["packages"]:
+        pkg_f = find_package_files(pkg, repo)
 
-    return len(pkgs) > 0
+        if pkg_f:
+            for f in pkg_f:
+                base = pathlib.Path(f).parent
+                repo = f.split("/")[-4]
+                logging.info("[%s/%s] Deleting package files: %s", repo, parsed_pkgbuild["pkgbase"],
+                             pathlib.Path(f).name)
+
+                args = ["repo-remove", "-s", "-v", os.path.join(base, repo + ".db.tar.xz"), pkg]
+                r_res = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+                logging.debug("[REPO-REMOVE] %s", r_res.stdout.decode(errors="ignore"))
+                if r_res.returncode:
+                    logging.error("[REPO/%s] Repo action failed: %s", repo, r_res.stdout.decode(errors="ignore"))
+                    continue
+                os.remove(f)
+                os.remove(f + ".sig")
 
 
-def find_all_files_for_pkg(name: str, repo: str) -> list:
+def find_package_files(name: str, repo: list[AnyStr]) -> list[AnyStr]:
     pkgs = []
-    for root, dirs, files in os.walk(os.path.join(config["basedir"]["repo"], repo, "os", config["arch"])):
+
-        for file in files:
+    for t_repo in repo:
-            res = regex_pkg_repo.findall(file)
+        files: list[str]
-            for r in res:
+        root: str
-                if r == name:
+        for root, dirs, files in os.walk(os.path.join(config["basedir"]["repo"], t_repo, "os", config["arch"])):
+            for file in files:
+                if file.endswith(".sig"):
+                    continue
+
+                res = regex_pkg_repo.search(file)
+                if res and res.group(1) == name:
                     pkgs.append(os.path.join(root, file))
-
-    for p in pkgs:
-        if p.endswith(".sig"):
-            pkgs.remove(p)
-
     return pkgs
 
 
+def is_package_failed(package: str, ver: LegacyVersion, repo: str):
+    pkgs = get_failed_packages(repo)
+
+    p: str
+    for p in pkgs:
+        s = p.split("==")
+
+        if s[0] == package:
+            if ver > LegacyVersion(s[1]):
+                with failed_l, open(os.path.join(config["basedir"]["repo"], repo + "_failed.txt"), "r+") as f:
+                    d = f.readlines()
+                    f.seek(0)
+                    f.truncate()
+                    for i in d:
+                        if i.strip("\n") != p:
+                            f.write(i)
+                return False
+            else:
+                return True
+    return False
+
+
 def get_failed_packages(repo: str) -> list:
     if os.path.exists(os.path.join(config["basedir"]["repo"], repo + "_failed.txt")):
-        with open(os.path.join(config["basedir"]["repo"], repo + "_failed.txt")) as p:
+        with failed_l, open(os.path.join(config["basedir"]["repo"], repo + "_failed.txt")) as p:
-            return p.read().splitlines()
+            return p.readlines()
     else:
         return []
 
@@ -224,22 +266,10 @@ def update_svn2git() -> None:
                                                      stderr=subprocess.STDOUT).stdout.decode(errors="ignore"))
             logging.debug("[GIT] %s", subprocess.run(["git", "pull"], check=True, stdout=subprocess.PIPE,
                                                      stderr=subprocess.STDOUT).stdout.decode(errors="ignore"))
-            os.chdir(sys.path[0])
-
-
-def increase_pkgrel(pkgbuild_file) -> None:
-    parsed = parse_pkgbuild(pkgbuild_file)
-    with open(pkgbuild_file, "r+", errors='ignore') as p:
-        pkgbuild_str = p.read()
-        p.truncate(0)
-        p.seek(0, 0)
-
-        pkgbuild_str = regex_pkgrel.sub("pkgrel=" + parsed["pkgrel"] + ".1", pkgbuild_str)
-        p.write(pkgbuild_str)
 
 
 def parse_repo(name, repo) -> LegacyVersion:
-    ver_split = find_all_files_for_pkg(name, repo)[0].split("-")
+    ver_split = find_package_files(name, [repo, ])[0].split("-")
 
     return LegacyVersion(ver_split[-3] + "-" + ver_split[-2])
 
@@ -283,33 +313,39 @@ def fill_queue() -> None:
 
     for pkgbuild in all_pkgbuild:
         path_split = pkgbuild.split("/")
-        parsed = parse_pkgbuild(pkgbuild)
+
+        if path_split[-2] == "trunk" or path_split[-2].split("-")[0] not in config["repos"] or "i686" in path_split[-2]:
+            continue
+
+        parsed_pkgb = parse_pkgbuild(pkgbuild)
 
         # ignore pkgbuild if in trunk, -any package, not in repos, on blacklist, not for current arch
-        if path_split[-2] == "trunk" or path_split[-2].split("-")[0] not in config["repos"] or "any" in parsed["arch"] \
+        if "any" in parsed_pkgb["arch"] or parsed_pkgb["pkgbase"] in config["blacklist"]:
-                or parsed["pkgbase"] in config["blacklist"] or "i686" in path_split[-2]:
+            delete_package(parsed_pkgb, repos)
-            # TODO: delete pkgs not build anymore
-            pass
         else:
             for march in config["march"]:
                 repo = path_split[-2].split("-")[0] + "-" + march
+                ver = parse_pkgbuild_ver(parsed=parsed_pkgb)
 
-                for pkgname in list(parsed["packages"]):
+                if is_package_failed(parsed_pkgb["pkgbase"], ver, repo):
-                    if pkgname in get_failed_packages(repo):
+                    logging.info("[%s/%s] Skipped due to failing build", repo, parsed_pkgb["pkgbase"])
-                        logging.info("[%s/%s] Skipped due to failing build", repo, pkgname)
+                    delete_package(parsed_pkgb, [repo, ])
-                        continue
+                    continue
 
-                    if package_exists(pkgname, repo):
+                packages = list(parsed_pkgb["packages"])
-                        logging.debug("[SEMVER] Comparing %s=%s - %s=%s", pkgname, parse_repo(pkgname, repo), pkgname,
+                pkg_f = find_package_files(packages[0], [repo, ])
-                                      parse_pkgbuild_ver(pkgbuild))
+                if pkg_f:
-
+                    logging.debug("[SEMVER] Comparing %s=%s - %s=%s", packages[0], parse_repo(packages[0], repo),
-                    if not package_exists(pkgname, repo):
+                                  packages[0], ver)
+                    rv = parse_repo(packages[0], repo)
+                    if rv < ver:
                         q.put((pkgbuild, repo))
-                        logging.info("[%s/%s] Build queued (package not build yet)", repo, pkgname)
+                        logging.info("[%s/%s] Build queued (new version available %s < %s)", repo,
-                    elif parse_repo(pkgname, repo) < parse_pkgbuild_ver(pkgbuild):
+                                     parsed_pkgb["pkgbase"],
-                        q.put((pkgbuild, repo))
+                                     rv, ver)
-                        logging.info("[%s/%s] Build queued (new version available %s < %s)", repo, pkgname,
+                else:
-                                     parse_repo(pkgname, repo), parse_pkgbuild(pkgbuild))
+                    q.put((pkgbuild, repo))
+                    logging.info("[%s/%s] Build queued (package not build yet)", repo, parsed_pkgb["pkgbase"])
 
     logging.info("Build queue size: %s", q.qsize())
 
@@ -332,6 +368,7 @@ if __name__ == '__main__':
 
     setup_chroot()
     sync_marchs_with_config()
+    do_repo_work()
     update_svn2git()
     q = JoinableQueue()
 
@@ -354,7 +391,7 @@ if __name__ == '__main__':
                     time.sleep(300)
                     do_repo_work()
             except KeyboardInterrupt:
-                with copy_l:
+                with copy_l, failed_l:
                     pool.close()
                     pool.terminate()
                     q.close()

utils.py

@@ -1,12 +1,14 @@
 import logging
 import os
 import pathlib
+import re
 import subprocess
-import sys
 
 from packaging.version import LegacyVersion
 from srcinfo.parse import parse_srcinfo
 
+regex_pkgrel = re.compile(r"^pkgrel\s*=\s*(.+)$", re.MULTILINE)
+
 
 def import_keys(pkgbuild: str) -> bool:
     parsed = parse_pkgbuild(pkgbuild)
@@ -25,22 +27,38 @@ def import_keys(pkgbuild: str) -> bool:
     return True
 
 
+def increase_pkgrel(pkgbuild_file: str, parsed: dict = None) -> None:
+    if not parsed:
+        parsed = parse_pkgbuild(pkgbuild_file)
+    with open(pkgbuild_file, "r+", errors='ignore') as p:
+        pkgbuild_str = p.read()
+        p.seek(0)
+        p.truncate(0)
+
+        pkgbuild_str = regex_pkgrel.sub("pkgrel=" + parsed["pkgrel"] + ".1", pkgbuild_str)
+        p.write(pkgbuild_str)
+
+
 def parse_pkgbuild(pkgbuild_file: str) -> dict:
     pkgbuild_path = pathlib.Path(pkgbuild_file)
     os.chdir(pkgbuild_path.parent)
-    res = subprocess.run(["makepkg", "--printsrcinfo"], check=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    res = subprocess.run(["makepkg", "--printsrcinfo"], check=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-    os.chdir(sys.path[0])
+
+    if res.returncode:
+        logging.warning("[PKGBUILD/%s] makepkg failed: %s", pkgbuild_path.name, res.stdout.decode(errors="ignore"))
+        return {}
 
     (parsed, errors) = parse_srcinfo(res.stdout.decode(errors="ignore"))
 
     if errors:
-        logging.warning("[PKGBUILD] Failed to parse: %s", pkgbuild_path.name)
+        logging.warning("[PKGBUILD/%s] Failed to parse: %s", pkgbuild_path.name, errors)
         return {}
     return parsed
 
 
-def parse_pkgbuild_ver(pkgbuild_file: str) -> LegacyVersion:
+def parse_pkgbuild_ver(pkgbuild_file: str = None, parsed: dict = None) -> LegacyVersion:
-    parsed = parse_pkgbuild(pkgbuild_file)
+    if not parsed:
+        parsed = parse_pkgbuild(pkgbuild_file)
     if "epoch" in parsed:
         return LegacyVersion("{}:{}-{}".format(parsed["epoch"], parsed["pkgver"], parsed["pkgrel"]))
     return LegacyVersion("{}-{}".format(parsed["pkgver"], parsed["pkgrel"]))