diff --git a/master.py b/master.py index 8c58a1d..3c16a75 100644 --- a/master.py +++ b/master.py @@ -12,6 +12,7 @@ import subprocess import sys import time from multiprocessing import Pool, Queue, current_process, Lock +from queue import Empty import yaml from humanfriendly import format_timespan @@ -62,7 +63,6 @@ def build(pkgbuild, repo): with open(os.path.join(config["basedir"]["repo"], "logs", repo, name + ".log"), "w") as log: log.write(res.stdout.decode()) - build_cleanup() return # signing @@ -72,7 +72,6 @@ def build(pkgbuild, repo): stderr=subprocess.STDOUT) if s_res.returncode: logging.error("[%s/%s/%s] Signing failed: %s", process_name, repo, name, s_res.stdout.decode()) - build_cleanup() return # copying @@ -92,7 +91,6 @@ def build(pkgbuild, repo): if r_res.returncode: logging.error("[%s/%s/%s] Repo action failed: %s", process_name, repo, name, r_res.stdout.decode()) repo_lock.release() - build_cleanup() return p_res = subprocess.run( @@ -102,11 +100,8 @@ def build(pkgbuild, repo): repo_lock.release() if p_res.returncode: logging.error("[%s/%s/%s] Repo cleanup failed: %s", process_name, repo, name, p_res.stdout.decode()) - build_cleanup() return - # cleanup - build_cleanup() logging.info("[%s/%s/%s] Build successful (%s)", process_name, repo, name, format_timespan(time.time() - start_time)) @@ -118,6 +113,9 @@ def run_worker() -> None: build(*q.get(block=True)) except Exception as e: logging.error("Error in worker: %s", e) + finally: + q.task_done() + os.chdir(sys.path[0]) def already_running(): @@ -150,10 +148,6 @@ def get_failed_packages(repo): return [] -def build_cleanup(): - os.chdir(sys.path[0]) - - def setup_chroot(): if not os.path.exists(os.path.join(config["basedir"]["chroot"], "root")): pathlib.Path(config["basedir"]["chroot"]).mkdir(parents=True, exist_ok=True) @@ -300,7 +294,7 @@ def fill_queue(): # ignore pkgbuild if in trunk, -any package, not in repos, or on blacklist if path_split[-2] == "trunk" or path_split[-2].split("-")[0] not in config["repos"] or "any" in path_split[-2] \ - or path_split[-4] in config["blacklist"] or "core-i686" in path_split[-2]: + or path_split[-4] in config["blacklist"] or "i686" in path_split[-2]: to_delete.append(pkgbuild) final_pkgbuilds = list(set(all_pkgbuild) - set(to_delete)) @@ -357,7 +351,25 @@ if __name__ == '__main__': while True: try: - if time.time() - update_last > 900 and q.qsize() == 0: + du = shutil.disk_usage(config["basedir"]["upstream"]) + if (du[1] / du[0]) > 0.9: + logging.warning("Less then 10% disk space remaining, performing cleanup...") + + while not q.empty(): + try: + q.get(False) + except Empty: + continue + q.task_done() + q.join() + + logging.info("Cleared Queue, clearing upstream repos...") + + update_svn2git() + logging.info("Cleanup done, refill queue") + fill_queue() + time.sleep(60) + elif time.time() - update_last > 900 and q.qsize() == 0: update_last = time.time() update_svn2git() fill_queue()