From d0dd9661c6e658670c50f80d9800fa1ebe07ef70 Mon Sep 17 00:00:00 2001
From: Giovanni Harting <539@idlegandalf.com>
Date: Fri, 21 May 2021 17:43:30 +0200
Subject: [PATCH] made upstream dirs configurable, clone only depth 1 don't
 exit worker process after exception

---
 config.yaml |  1 +
 master.py   | 33 +++++++++++++++++++--------------
 2 files changed, 20 insertions(+), 14 deletions(-)

diff --git a/config.yaml b/config.yaml
index 7945d20..ec48cfe 100644
--- a/config.yaml
+++ b/config.yaml
@@ -12,6 +12,7 @@ basedir:
   repo: /tmp/repo/
   chroot: /tmp/chroot/
   makepkg: /tmp/makepkg/
+  upstream: /tmp/upstream/
 
 march:
   - x86-64-v3
diff --git a/master.py b/master.py
index e0a6799..35bdc2d 100644
--- a/master.py
+++ b/master.py
@@ -28,7 +28,7 @@ def build(pkgbuild, repo):
     start_time = time.time()
     name = pathlib.Path(pkgbuild).parts[-4]
     process_name = current_process().name
-    logging.info("[%s/%s] Build starting (Queue ~= %s)", repo, name, q.qsize())
+    logging.info("[%s/%s/%s] Build starting (Queue ~= %s)", process_name, repo, name, q.qsize())
 
     # setup makepkg
     setup_makepkg(repo)
@@ -108,7 +108,10 @@ def build(pkgbuild, repo):
 def run_worker() -> None:
     os.nice(20)
     while True:
-        build(*q.get(block=True))
+        try:
+            build(*q.get(block=True))
+        except Exception as e:
+            logging.error("Error in worker: %s", e)
 
 
 def already_running():
@@ -124,15 +127,12 @@ def already_running():
 
 def find_all_files_for_pkg(name, repo):
     searchpath = os.path.join(config["basedir"]["repo"], repo, "os", config["arch"]) + "/" + name + "-*.pkg.*"
-    # logging.debug("[REPO] Search for package %s (path=%s)", name, searchpath)
     pkgs = glob.glob(searchpath)
 
     for p in pkgs:
         if p.endswith(".sig"):
             pkgs.remove(p)
 
-    # logging.debug("[REPO] Found %s", pkgs)
-
     return pkgs
 
 
@@ -202,21 +202,25 @@ def package_exists(name, repo):
     return len(pkgs) > 0
 
 
-def update_git2svn():
+def update_svn2git():
+    if not os.path.exists(config["basedir"]["upstream"]):
+        pathlib.Path(config["basedir"]["upstream"]).mkdir(parents=True, exist_ok=True)
+
     for git_dir, git_url in config["svn2git"].items():
-        if not os.path.exists(git_dir):
+        git_path = os.path.join(config["basedir"]["upstream"], git_dir)
+        if not os.path.exists(git_path):
             logging.debug("[GIT] %s",
-                          subprocess.run(["git", "clone", git_url, git_dir], check=True, stdout=subprocess.PIPE,
-                                         stderr=subprocess.STDOUT).stdout.decode())
+                          subprocess.run(["git", "clone", "--depth=1", git_url, git_path], check=True,
+                                         stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout.decode())
         else:
-            os.chdir(git_dir)
+            os.chdir(git_path)
             logging.debug("[GIT] %s", subprocess.run(["git", "clean", "-xdf"], check=False, stdout=subprocess.PIPE,
                                                      stderr=subprocess.STDOUT).stdout.decode())
             logging.debug("[GIT] %s", subprocess.run(["git", "reset", "--hard"], check=True, stdout=subprocess.PIPE,
                                                      stderr=subprocess.STDOUT).stdout.decode())
             logging.debug("[GIT] %s", subprocess.run(["git", "pull"], check=True, stdout=subprocess.PIPE,
                                                      stderr=subprocess.STDOUT).stdout.decode())
-            os.chdir("..")
+            os.chdir(sys.path[0])
 
 
 def parse_pkgbuild(pkgbuild_file):
@@ -280,7 +284,8 @@ def fill_queue():
     all_pkgbuild = []
 
     for git_dir, git_url in config["svn2git"].items():
-        all_pkgbuild.extend(glob.glob(os.path.join(git_dir) + "/**/PKGBUILD", recursive=True))
+        all_pkgbuild.extend(
+            glob.glob(os.path.join(config["basedir"]["upstream"], git_dir) + "/**/PKGBUILD", recursive=True))
 
     to_delete = []
 
@@ -337,7 +342,7 @@ if __name__ == '__main__':
 
     setup_chroot()
     sync_marchs_with_config()
-    update_git2svn()
+    update_svn2git()
     q = Queue()
 
     with Pool(config["build"]["worker"], initializer=run_worker) as pool:
@@ -348,7 +353,7 @@ if __name__ == '__main__':
             try:
                 if time.time() - update_last > 900 and q.qsize() == 0:
                     update_last = time.time()
-                    update_git2svn()
+                    update_svn2git()
                     fill_queue()
                 else:
                     time.sleep(60)