diff --git a/master.py b/master.py index a77c4c9..c8dfe69 100644 --- a/master.py +++ b/master.py @@ -29,7 +29,7 @@ update_last = time.time() copy_l = Lock() -def build(pkgbuild: str, repo: str, todo: dict) -> None: +def build(pkgbuild: str, repo: str) -> None: start_time = time.time() name = pathlib.Path(pkgbuild).parts[-4] process_name = current_process().name @@ -90,11 +90,6 @@ def build(pkgbuild: str, repo: str, todo: dict) -> None: os.path.join(config["basedir"]["repo"], repo, "os", config["arch"] + "/")) shutil.copy2(pkg, os.path.join(config["basedir"]["repo"], repo, "os", config["arch"] + "/")) - # repo - logging.debug("[%s/%s/%s] Adding packages to todo list: %s", process_name, repo, name, - ", ".join(glob.glob("*.pkg.tar.zst"))) - todo[repo].extend(glob.glob("*.pkg.tar.zst")) - logging.debug("[GIT] %s", subprocess.run(["git", "clean", "-xdff"], check=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout.decode(errors="ignore")) @@ -102,11 +97,11 @@ def build(pkgbuild: str, repo: str, todo: dict) -> None: format_timespan(time.time() - start_time)) -def run_worker(todo: dict) -> None: +def run_worker() -> None: os.nice(20) while True: try: - build(*q.get(block=True), todo=todo) + build(*q.get(block=True)) except Exception as e: logging.error("Error in worker: %s", e) finally: @@ -115,28 +110,21 @@ def run_worker(todo: dict) -> None: def do_repo_work() -> None: - for repo in d: - if d[repo]: - logging.info("[REPO/%s] Adding %s", repo, ", ".join(d[repo])) - os.chdir(os.path.join(config["basedir"]["repo"], repo, "os", config["arch"])) - args = ["repo-add", "-s", "-v", - os.path.join(config["basedir"]["repo"], repo, "os", config["arch"], repo + ".db.tar.xz")] - args.extend(d[repo]) - r_res = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - logging.debug("[REPO-ADD] %s", r_res.stdout.decode(errors="ignore")) - if r_res.returncode: - logging.error("[REPO/%s] Repo action failed: %s", repo, r_res.stdout.decode(errors="ignore")) + for repo in config["repos"]: + args = ["repo-add", "-s", "-v", "-p", "-n", + os.path.join(config["basedir"]["repo"], repo, "os", config["arch"], repo + ".db.tar.xz"), "*.zst"] + r_res = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + logging.debug("[REPO-ADD] %s", r_res.stdout.decode(errors="ignore")) + if r_res.returncode: + logging.error("[REPO/%s] Repo action failed: %s", repo, r_res.stdout.decode(errors="ignore")) - p_res = subprocess.run( - ["paccache", "-rc", os.path.join(config["basedir"]["repo"], repo, "os", config["arch"]), "-k", "1"], - stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - logging.debug("[PACCACHE] %s", p_res.stdout.decode(errors="ignore")) - if p_res.returncode: - logging.error("[REPO/%s] Repo cleanup failed: %s", repo, p_res.stdout.decode(errors="ignore")) - d[repo][:] = [] - os.chdir(sys.path[0]) - else: - logging.debug("[REPO/%s] Nothing to do", repo) + p_res = subprocess.run( + ["paccache", "-rc", os.path.join(config["basedir"]["repo"], repo, "os", config["arch"]), "-k", "1"], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + logging.debug("[PACCACHE] %s", p_res.stdout.decode(errors="ignore")) + if p_res.returncode: + logging.error("[REPO/%s] Repo cleanup failed: %s", repo, p_res.stdout.decode(errors="ignore")) + os.chdir(sys.path[0]) def already_running() -> bool: @@ -311,8 +299,6 @@ def sync_marchs_with_config() -> None: logging.info("Repos: %s", repo_quota) repos_create = list(set(repo_quota) - set(repos)) repos_delete = list(set(repos) - set(repo_quota)) - for repo in repo_quota: - d[repo] = m.list() for repo in repos_create: logging.debug("Create repo %s: %s", repo, os.path.join(config["basedir"]["repo"], repo, "os/x86_64")) @@ -385,36 +371,33 @@ if __name__ == '__main__': os.nice(5) - with Manager() as m: - d = m.dict() - setup_chroot() - sync_marchs_with_config() - update_svn2git() - q = JoinableQueue() + setup_chroot() + sync_marchs_with_config() + update_svn2git() + q = JoinableQueue() - with Pool(config["build"]["worker"], initializer=run_worker, initargs=(d,)) as pool: - fill_queue() - signal.signal(signal.SIGINT, signal.default_int_handler) + with Pool(config["build"]["worker"], initializer=run_worker) as pool: + fill_queue() + signal.signal(signal.SIGINT, signal.default_int_handler) - while True: - try: - if time.time() - update_last > 900 and q.empty(): - logging.info("[SVN2GIT] Waiting for queue to finish...") - q.join() - do_repo_work() - update_last = time.time() - update_svn2git() - setup_chroot() - fill_queue() - if q.qsize() > 0: - logging.info("[SVN2GIT] New Queue size: %d", q.qsize()) - else: - time.sleep(60) - do_repo_work() - except KeyboardInterrupt: - with copy_l: - pool.close() - pool.terminate() - q.close() - do_repo_work() - sys.exit(0) + while True: + try: + if time.time() - update_last > 900 and q.empty(): + logging.info("[SVN2GIT] Waiting for queue to finish...") + q.join() + update_last = time.time() + update_svn2git() + setup_chroot() + fill_queue() + if q.qsize() > 0: + logging.info("[SVN2GIT] New Queue size: %d", q.qsize()) + else: + time.sleep(300) + do_repo_work() + except KeyboardInterrupt: + with copy_l: + pool.close() + pool.terminate() + q.close() + do_repo_work() + sys.exit(0)