changed way signing works

master.py

@@ -23,12 +23,13 @@ regex_pkgrel = re.compile(r"^pkgrel\s*=\s*(.+)$", re.MULTILINE)
 regex_epoch = re.compile(r"^epoch\s*=\s*(.+)$", re.MULTILINE)
 regex_march = re.compile(r"(-march=)(.+?) ", re.MULTILINE)
 regex_validkeys = re.compile(r"^validpgpkeys\+?=\((.*?)\)", re.MULTILINE | re.DOTALL)
+regex_pkg_repo = re.compile(r"^(.*)-.*-.*-(?:x86_64|any)\.pkg\.tar\.zst(?:\.sig)*$", re.MULTILINE)
 fp = None
 update_last = time.time()
 copy_l = Lock()
 
 
-def build(pkgbuild: str, repo: str, todo: dict) -> None:
+def build(pkgbuild: str, repo: str) -> None:
     start_time = time.time()
     name = pathlib.Path(pkgbuild).parts[-4]
     process_name = current_process().name
@@ -89,11 +90,6 @@ def build(pkgbuild: str, repo: str, todo: dict) -> None:
                           os.path.join(config["basedir"]["repo"], repo, "os", config["arch"] + "/"))
             shutil.copy2(pkg, os.path.join(config["basedir"]["repo"], repo, "os", config["arch"] + "/"))
 
-        # repo
-        logging.debug("[%s/%s/%s] Adding packages to todo list: %s", process_name, repo, name,
-                      ", ".join(glob.glob("*.pkg.tar.zst")))
-        todo[repo].extend(glob.glob("*.pkg.tar.zst"))
-
     logging.debug("[GIT] %s", subprocess.run(["git", "clean", "-xdff"], check=False, stdout=subprocess.PIPE,
                                              stderr=subprocess.STDOUT).stdout.decode(errors="ignore"))
 
@@ -101,11 +97,11 @@ def build(pkgbuild: str, repo: str, todo: dict) -> None:
                  format_timespan(time.time() - start_time))
 
 
-def run_worker(todo: dict) -> None:
+def run_worker() -> None:
     os.nice(20)
     while True:
         try:
-            build(*q.get(block=True), todo=todo)
+            build(*q.get(block=True))
         except Exception as e:
             logging.error("Error in worker: %s", e)
         finally:
@@ -114,28 +110,21 @@ def run_worker(todo: dict) -> None:
 
 
 def do_repo_work() -> None:
-    for repo in d:
+    for repo in config["repos"]:
-        if d[repo]:
+        args = ["repo-add", "-s", "-v", "-p", "-n",
-            logging.info("[REPO/%s] Adding %s", repo, ", ".join(d[repo]))
+                os.path.join(config["basedir"]["repo"], repo, "os", config["arch"], repo + ".db.tar.xz"), "*.zst"]
-            os.chdir(os.path.join(config["basedir"]["repo"], repo, "os", config["arch"]))
+        r_res = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-            args = ["repo-add", "-s", "-v",
+        logging.debug("[REPO-ADD] %s", r_res.stdout.decode(errors="ignore"))
-                    os.path.join(config["basedir"]["repo"], repo, "os", config["arch"], repo + ".db.tar.xz")]
+        if r_res.returncode:
-            args.extend(d[repo])
+            logging.error("[REPO/%s] Repo action failed: %s", repo, r_res.stdout.decode(errors="ignore"))
-            r_res = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-            logging.debug("[REPO-ADD] %s", r_res.stdout.decode(errors="ignore"))
-            if r_res.returncode:
-                logging.error("[REPO/%s] Repo action failed: %s", repo, r_res.stdout.decode(errors="ignore"))
 
-            p_res = subprocess.run(
+        p_res = subprocess.run(
-                ["paccache", "-rc", os.path.join(config["basedir"]["repo"], repo, "os", config["arch"]), "-k", "1"],
+            ["paccache", "-rc", os.path.join(config["basedir"]["repo"], repo, "os", config["arch"]), "-k", "1"],
-                stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+            stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-            logging.debug("[PACCACHE] %s", p_res.stdout.decode(errors="ignore"))
+        logging.debug("[PACCACHE] %s", p_res.stdout.decode(errors="ignore"))
-            if p_res.returncode:
+        if p_res.returncode:
-                logging.error("[REPO/%s] Repo cleanup failed: %s", repo, p_res.stdout.decode(errors="ignore"))
+            logging.error("[REPO/%s] Repo cleanup failed: %s", repo, p_res.stdout.decode(errors="ignore"))
-            d[repo][:] = []
+        os.chdir(sys.path[0])
-            os.chdir(sys.path[0])
-        else:
-            logging.debug("[REPO/%s] Nothing to do", repo)
 
 
 def already_running() -> bool:
@@ -150,8 +139,16 @@ def already_running() -> bool:
 
 
 def find_all_files_for_pkg(name: str, repo: str) -> list:
-    searchpath = os.path.join(config["basedir"]["repo"], repo, "os", config["arch"]) + "/" + name + "-*.pkg.*"
+    pkgs = []
-    pkgs = glob.glob(searchpath)
+    for root, dirs, files in os.walk(os.path.join(config["basedir"]["repo"], repo, "os", config["arch"])):
+        for file in files:
+            res = regex_pkg_repo.match(file)
+            if res:
+                if res.group(1) is name:
+                    pkgs.append(os.path.join(root, file))
+
+    # searchpath = os.path.join(config["basedir"]["repo"], repo, "os", config["arch"]) + "/" + name + "-*.pkg.*"
+    # pkgs = glob.glob(searchpath)
 
     for p in pkgs:
         if p.endswith(".sig"):
@@ -302,8 +299,6 @@ def sync_marchs_with_config() -> None:
     logging.info("Repos: %s", repo_quota)
     repos_create = list(set(repo_quota) - set(repos))
     repos_delete = list(set(repos) - set(repo_quota))
-    for repo in repo_quota:
-        d[repo] = m.list()
 
     for repo in repos_create:
         logging.debug("Create repo %s: %s", repo, os.path.join(config["basedir"]["repo"], repo, "os/x86_64"))
@@ -376,36 +371,33 @@ if __name__ == '__main__':
 
     os.nice(5)
 
-    with Manager() as m:
+    setup_chroot()
-        d = m.dict()
+    sync_marchs_with_config()
-        setup_chroot()
+    update_svn2git()
-        sync_marchs_with_config()
+    q = JoinableQueue()
-        update_svn2git()
-        q = JoinableQueue()
 
-        with Pool(config["build"]["worker"], initializer=run_worker, initargs=(d,)) as pool:
+    with Pool(config["build"]["worker"], initializer=run_worker) as pool:
-            fill_queue()
+        fill_queue()
-            signal.signal(signal.SIGINT, signal.default_int_handler)
+        signal.signal(signal.SIGINT, signal.default_int_handler)
 
-            while True:
+        while True:
-                try:
+            try:
-                    if time.time() - update_last > 900 and q.empty():
+                if time.time() - update_last > 900 and q.empty():
-                        logging.info("[SVN2GIT] Waiting for queue to finish...")
+                    logging.info("[SVN2GIT] Waiting for queue to finish...")
-                        q.join()
+                    q.join()
-                        do_repo_work()
+                    update_last = time.time()
-                        update_last = time.time()
+                    update_svn2git()
-                        update_svn2git()
+                    setup_chroot()
-                        setup_chroot()
+                    fill_queue()
-                        fill_queue()
+                    if q.qsize() > 0:
-                        if q.qsize() > 0:
+                        logging.info("[SVN2GIT] New Queue size: %d", q.qsize())
-                            logging.info("[SVN2GIT] New Queue size: %d", q.qsize())
+                else:
-                    else:
+                    time.sleep(300)
-                        time.sleep(60)
+                    do_repo_work()
-                        do_repo_work()
+            except KeyboardInterrupt:
-                except KeyboardInterrupt:
+                with copy_l:
-                    with copy_l:
+                    pool.close()
-                        pool.close()
+                    pool.terminate()
-                        pool.terminate()
+                    q.close()
-                        q.close()
+                    do_repo_work()
-                        do_repo_work()
+                    sys.exit(0)
-                        sys.exit(0)