changed way signing works

master.py

@@ -23,12 +23,13 @@ regex_pkgrel = re.compile(r"^pkgrel\s*=\s*(.+)$", re.MULTILINE)
 regex_epoch = re.compile(r"^epoch\s*=\s*(.+)$", re.MULTILINE)
 regex_march = re.compile(r"(-march=)(.+?) ", re.MULTILINE)
 regex_validkeys = re.compile(r"^validpgpkeys\+?=\((.*?)\)", re.MULTILINE | re.DOTALL)
+regex_pkg_repo = re.compile(r"^(.*)-.*-.*-(?:x86_64|any)\.pkg\.tar\.zst(?:\.sig)*$", re.MULTILINE)
 fp = None
 update_last = time.time()
 copy_l = Lock()
 
 
-def build(pkgbuild: str, repo: str, todo: dict) -> None:
+def build(pkgbuild: str, repo: str) -> None:
     start_time = time.time()
     name = pathlib.Path(pkgbuild).parts[-4]
     process_name = current_process().name
@@ -89,11 +90,6 @@ def build(pkgbuild: str, repo: str, todo: dict) -> None:
                           os.path.join(config["basedir"]["repo"], repo, "os", config["arch"] + "/"))
             shutil.copy2(pkg, os.path.join(config["basedir"]["repo"], repo, "os", config["arch"] + "/"))
 
-        # repo
-        logging.debug("[%s/%s/%s] Adding packages to todo list: %s", process_name, repo, name,
-                      ", ".join(glob.glob("*.pkg.tar.zst")))
-        todo[repo].extend(glob.glob("*.pkg.tar.zst"))
-
     logging.debug("[GIT] %s", subprocess.run(["git", "clean", "-xdff"], check=False, stdout=subprocess.PIPE,
                                              stderr=subprocess.STDOUT).stdout.decode(errors="ignore"))
 
@@ -101,11 +97,11 @@ def build(pkgbuild: str, repo: str, todo: dict) -> None:
                  format_timespan(time.time() - start_time))
 
 
-def run_worker(todo: dict) -> None:
+def run_worker() -> None:
     os.nice(20)
     while True:
         try:
-            build(*q.get(block=True), todo=todo)
+            build(*q.get(block=True))
         except Exception as e:
             logging.error("Error in worker: %s", e)
         finally:
@@ -114,13 +110,9 @@ def run_worker(todo: dict) -> None:
 
 
 def do_repo_work() -> None:
-    for repo in d:
-        if d[repo]:
-            logging.info("[REPO/%s] Adding %s", repo, ", ".join(d[repo]))
-            os.chdir(os.path.join(config["basedir"]["repo"], repo, "os", config["arch"]))
-            args = ["repo-add", "-s", "-v",
-                    os.path.join(config["basedir"]["repo"], repo, "os", config["arch"], repo + ".db.tar.xz")]
-            args.extend(d[repo])
+    for repo in config["repos"]:
+        args = ["repo-add", "-s", "-v", "-p", "-n",
+                os.path.join(config["basedir"]["repo"], repo, "os", config["arch"], repo + ".db.tar.xz"), "*.zst"]
         r_res = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
         logging.debug("[REPO-ADD] %s", r_res.stdout.decode(errors="ignore"))
         if r_res.returncode:
@@ -132,10 +124,7 @@ def do_repo_work() -> None:
         logging.debug("[PACCACHE] %s", p_res.stdout.decode(errors="ignore"))
         if p_res.returncode:
             logging.error("[REPO/%s] Repo cleanup failed: %s", repo, p_res.stdout.decode(errors="ignore"))
-            d[repo][:] = []
         os.chdir(sys.path[0])
-        else:
-            logging.debug("[REPO/%s] Nothing to do", repo)
 
 
 def already_running() -> bool:
@@ -150,8 +139,16 @@ def already_running() -> bool:
 
 
 def find_all_files_for_pkg(name: str, repo: str) -> list:
-    searchpath = os.path.join(config["basedir"]["repo"], repo, "os", config["arch"]) + "/" + name + "-*.pkg.*"
-    pkgs = glob.glob(searchpath)
+    pkgs = []
+    for root, dirs, files in os.walk(os.path.join(config["basedir"]["repo"], repo, "os", config["arch"])):
+        for file in files:
+            res = regex_pkg_repo.match(file)
+            if res:
+                if res.group(1) is name:
+                    pkgs.append(os.path.join(root, file))
+
+    # searchpath = os.path.join(config["basedir"]["repo"], repo, "os", config["arch"]) + "/" + name + "-*.pkg.*"
+    # pkgs = glob.glob(searchpath)
 
     for p in pkgs:
         if p.endswith(".sig"):
@@ -302,8 +299,6 @@ def sync_marchs_with_config() -> None:
     logging.info("Repos: %s", repo_quota)
     repos_create = list(set(repo_quota) - set(repos))
     repos_delete = list(set(repos) - set(repo_quota))
-    for repo in repo_quota:
-        d[repo] = m.list()
 
     for repo in repos_create:
         logging.debug("Create repo %s: %s", repo, os.path.join(config["basedir"]["repo"], repo, "os/x86_64"))
@@ -376,14 +371,12 @@ if __name__ == '__main__':
 
     os.nice(5)
 
-    with Manager() as m:
-        d = m.dict()
     setup_chroot()
     sync_marchs_with_config()
     update_svn2git()
     q = JoinableQueue()
 
-        with Pool(config["build"]["worker"], initializer=run_worker, initargs=(d,)) as pool:
+    with Pool(config["build"]["worker"], initializer=run_worker) as pool:
         fill_queue()
         signal.signal(signal.SIGINT, signal.default_int_handler)
 
@@ -392,7 +385,6 @@ if __name__ == '__main__':
                 if time.time() - update_last > 900 and q.empty():
                     logging.info("[SVN2GIT] Waiting for queue to finish...")
                     q.join()
-                        do_repo_work()
                     update_last = time.time()
                     update_svn2git()
                     setup_chroot()
@@ -400,7 +392,7 @@ if __name__ == '__main__':
                     if q.qsize() > 0:
                         logging.info("[SVN2GIT] New Queue size: %d", q.qsize())
                 else:
-                        time.sleep(60)
+                    time.sleep(300)
                     do_repo_work()
             except KeyboardInterrupt:
                 with copy_l: