#!/usr/bin/env python3 import fcntl import glob import logging.config import os import pathlib import re import shutil import signal import subprocess import sys import time import traceback from multiprocessing import Pool, current_process, JoinableQueue, Lock import yaml from humanfriendly import format_timespan from packaging.version import LegacyVersion from utils import parse_pkgbuild, parse_pkgbuild_ver, import_keys regex_pkgver = re.compile(r"^_?pkgver\s*=\s*(.+)$", re.MULTILINE) regex_pkgrel = re.compile(r"^pkgrel\s*=\s*(.+)$", re.MULTILINE) regex_epoch = re.compile(r"^epoch\s*=\s*(.+)$", re.MULTILINE) regex_march = re.compile(r"(-march=)(.+?) ", re.MULTILINE) regex_validkeys = re.compile(r"^validpgpkeys\+?=\((.*?)\)", re.MULTILINE | re.DOTALL) regex_pkg_repo = re.compile(r"^(.*)-.*-.*-(?:x86_64|any)\.pkg\.tar\.zst(?:\.sig)*$", re.MULTILINE) fp = None update_last = time.time() copy_l = Lock() repos = [] def build(pkgbuild: str, repo: str) -> None: start_time = time.time() name = pathlib.Path(pkgbuild).parts[-4] process_name = current_process().name logging.info("[%s/%s/%s] Build starting (Queue ~= %s)", process_name, repo, name, q.qsize()) # setup makepkg setup_makepkg(repo) # import pgp keys import_keys(pkgbuild) # increase pkgrel increase_pkgrel(pkgbuild) # build with devtools os.chdir(pathlib.Path(pkgbuild).parent) res = subprocess.run( ["makechrootpkg", "-c", "-D", os.path.join(config["basedir"]["makepkg"]), "-l", process_name, "-r", os.path.join(config["basedir"]["chroot"]), "--", "--config", os.path.join(config["basedir"]["makepkg"]) + "makepkg-" + '-'.join( repo.split("-")[1:]) + ".conf"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) if res.returncode: logging.warning("[%s/%s/%s] Build failed. Check repo/logs for more information.", process_name, repo, name) # write packagename to failed list with open(os.path.join(config["basedir"]["repo"], repo + "_failed.txt"), "a") as f: f.write(name + "\n") # write logs if not os.path.exists(os.path.join(config["basedir"]["repo"], "logs", repo)): pathlib.Path(os.path.join(config["basedir"]["repo"], "logs", repo)).mkdir(parents=True, exist_ok=True) with open(os.path.join(config["basedir"]["repo"], "logs", repo, name + ".log"), "w") as log: log.write(res.stdout.decode(errors="ignore")) logging.debug("[GIT] %s", subprocess.run(["git", "clean", "-xdff"], check=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout.decode(errors="ignore")) return # signing pkgs = glob.glob("*.pkg.tar.zst") for pkg in pkgs: s_res = subprocess.run(["gpg", "--batch", "--detach-sign", pkg], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) if s_res.returncode: logging.error("[%s/%s/%s] Signing failed: %s", process_name, repo, name, s_res.stdout.decode(errors="ignore")) logging.debug("[GIT] %s", subprocess.run(["git", "clean", "-xdff"], check=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout.decode(errors="ignore")) return # copying pkgs.extend(glob.glob("*.pkg.tar.zst.sig")) with copy_l: for pkg in pkgs: logging.debug("[%s/%s/%s] Copy %s to %s", process_name, repo, name, pkg, os.path.join(config["basedir"]["repo"], repo, "os", config["arch"] + "/")) shutil.copy2(pkg, os.path.join(config["basedir"]["repo"], repo, "os", config["arch"] + "/")) logging.debug("[GIT] %s", subprocess.run(["git", "clean", "-xdff"], check=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout.decode(errors="ignore")) logging.info("[%s/%s/%s] Build successful (%s)", process_name, repo, name, format_timespan(time.time() - start_time)) def run_worker() -> None: os.nice(20) while True: try: build(*q.get(block=True)) except Exception as e: logging.error("Error in worker: %s", e) traceback.print_exc() finally: q.task_done() os.chdir(sys.path[0]) def do_repo_work() -> None: for repo in repos: pkgs = glob.glob(os.path.join(config["basedir"]["repo"], repo, "os", config["arch"], "*.zst")) args = ["repo-add", "-s", "-v", "-p", "-n", os.path.join(config["basedir"]["repo"], repo, "os", config["arch"], repo + ".db.tar.xz")] args.extend(pkgs) r_res = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) logging.debug("[REPO-ADD] %s", r_res.stdout.decode(errors="ignore")) if r_res.returncode: logging.error("[REPO/%s] Repo action failed: %s", repo, r_res.stdout.decode(errors="ignore")) p_res = subprocess.run( ["paccache", "-rc", os.path.join(config["basedir"]["repo"], repo, "os", config["arch"]), "-k", "1"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) logging.debug("[PACCACHE] %s", p_res.stdout.decode(errors="ignore")) if p_res.returncode: logging.error("[REPO/%s] Repo cleanup failed: %s", repo, p_res.stdout.decode(errors="ignore")) os.chdir(sys.path[0]) def already_running() -> bool: global fp fp = os.open(f"/tmp/alhp.lock", os.O_WRONLY | os.O_CREAT) try: fcntl.lockf(fp, fcntl.LOCK_EX | fcntl.LOCK_NB) return False except OSError: return True def package_exists(names: list, repo) -> bool: for name in names: pkgs = find_all_files_for_pkg(name, repo) if not pkgs: return False return True def find_all_files_for_pkg(name: str, repo: str) -> list: pkgs = [] for root, dirs, files in os.walk(os.path.join(config["basedir"]["repo"], repo, "os", config["arch"])): for file in files: res = regex_pkg_repo.findall(file) for r in res: if r == name: pkgs.append(os.path.join(root, file)) for p in pkgs: if p.endswith(".sig"): pkgs.remove(p) return pkgs def get_failed_packages(repo: str) -> list: if os.path.exists(os.path.join(config["basedir"]["repo"], repo + "_failed.txt")): with open(os.path.join(config["basedir"]["repo"], repo + "_failed.txt")) as p: return p.read().splitlines() else: return [] def setup_chroot() -> None: if not os.path.exists(os.path.join(config["basedir"]["chroot"], "root")): pathlib.Path(config["basedir"]["chroot"]).mkdir(parents=True, exist_ok=True) s = subprocess.run(["mkarchroot", "-C", "/usr/share/devtools/pacman-extra.conf", os.path.join(config["basedir"]["chroot"], "root"), "base-devel"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) logging.debug("[MKCHROOT] %s", s.stdout.decode(errors='ignore')) if s.returncode: logging.fatal("[MKCHROOT] Failed to create root chroot: %s", s.stdout.decode(errors="ignore")) sys.exit(2) else: logging.debug("[NSPAWN] %s", subprocess.run( ["arch-nspawn", os.path.join(config["basedir"]["chroot"], "root"), "pacman", "-Syuu", "--noconfirm"])) def setup_makepkg(repo) -> None: makepkg_repo = os.path.join(config["basedir"]["makepkg"], "makepkg-" + '-'.join(repo.split("-")[1:]) + ".conf") if not os.path.exists(makepkg_repo): pathlib.Path(config["basedir"]["makepkg"]).mkdir(parents=True, exist_ok=True) shutil.copyfile("makepkg.tmpl", makepkg_repo) with open(makepkg_repo) as conf: c_all = conf.read() c_all = c_all.replace("-mtune=generic", "") c_all = c_all.replace("-O2", "-O3") c_all = regex_march.sub(r"\1" + '-'.join(repo.split("-")[1:]), c_all) with open(makepkg_repo, "w") as conf: conf.write(c_all) def update_svn2git() -> None: if not os.path.exists(config["basedir"]["upstream"]): pathlib.Path(config["basedir"]["upstream"]).mkdir(parents=True, exist_ok=True) for git_dir, git_url in config["svn2git"].items(): git_path = os.path.join(config["basedir"]["upstream"], git_dir) if not os.path.exists(git_path): logging.debug("[GIT] %s", subprocess.run(["git", "clone", "--depth=1", git_url, git_path], check=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout.decode( errors="ignore")) else: os.chdir(git_path) logging.debug("[GIT] %s", subprocess.run(["git", "clean", "-xdff"], check=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout.decode(errors="ignore")) logging.debug("[GIT] %s", subprocess.run(["git", "reset", "--hard"], check=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout.decode(errors="ignore")) logging.debug("[GIT] %s", subprocess.run(["git", "pull"], check=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout.decode(errors="ignore")) os.chdir(sys.path[0]) def increase_pkgrel(pkgbuild_file) -> None: parsed = parse_pkgbuild(pkgbuild_file) with open(pkgbuild_file, "r+", errors='ignore') as p: pkgbuild_str = p.read() p.truncate(0) p.seek(0, 0) pkgbuild_str = regex_pkgrel.sub("pkgrel=" + parsed["pkgrel"] + ".1", pkgbuild_str) p.write(pkgbuild_str) def parse_repo(name, repo) -> LegacyVersion: ver_split = find_all_files_for_pkg(name, repo)[0].split("-") return LegacyVersion(ver_split[-3] + "-" + ver_split[-2]) def sync_marchs_with_config() -> None: existing_repos = [] with os.scandir(config["basedir"]["repo"]) as it: entry: os.DirEntry for entry in it: if not entry.name.startswith('logs') and entry.is_dir(): existing_repos.append(entry.name) repo_quota = [] for r, a in ((x, y) for x in config["repos"] for y in config["march"]): repo_quota.append("{}-{}".format(r, a)) logging.info("Repos: %s", repo_quota) global repos repos = repo_quota repos_create = list(set(repo_quota) - set(existing_repos)) repos_delete = list(set(existing_repos) - set(repo_quota)) for repo in repos_create: logging.debug("Create repo %s: %s", repo, os.path.join(config["basedir"]["repo"], repo, "os/x86_64")) pathlib.Path(os.path.join(config["basedir"]["repo"], repo, "os/x86_64")).mkdir(parents=True, exist_ok=True) setup_makepkg(repo) for repo in repos_delete: logging.debug("Delete repo %s: %s", repo, os.path.join(config["basedir"]["repo"], repo)) shutil.rmtree(os.path.join(config["basedir"]["repo"], repo)) os.remove(os.path.join(config["basedir"]["makepkg"], "makepkg-" + repo + ".conf")) def fill_queue() -> None: all_pkgbuild = [] for git_dir, git_url in config["svn2git"].items(): all_pkgbuild.extend( glob.glob(os.path.join(config["basedir"]["upstream"], git_dir) + "/**/PKGBUILD", recursive=True)) for pkgbuild in all_pkgbuild: path_split = pkgbuild.split("/") parsed = parse_pkgbuild(pkgbuild) # ignore pkgbuild if in trunk, -any package, not in repos, on blacklist, not for current arch if path_split[-2] == "trunk" or path_split[-2].split("-")[0] not in config["repos"] or "any" in parsed["arch"] \ or parsed["pkgbase"] in config["blacklist"] or "i686" in path_split[-2]: # TODO: delete packages not to build pass else: for march in config["march"]: repo = path_split[-2].split("-")[0] + "-" + march if parsed["pkgbase"] in get_failed_packages(repo): logging.info("[%s/%s] Skipped due to failing build", repo, parsed["pkgbase"]) continue packages = list(parsed["packages"]) if package_exists(packages, repo): logging.debug("[SEMVER] Comparing %s=%s - %s=%s", packages[0], parse_repo(packages[0], repo), packages[0], parse_pkgbuild_ver(pkgbuild)) if not package_exists(packages, repo): q.put((pkgbuild, repo)) logging.info("[%s/%s] Build queued (package not build yet)", repo, parsed["pkgbase"]) elif parse_repo(packages[0], repo) < parse_pkgbuild_ver(pkgbuild): q.put((pkgbuild, repo)) logging.info("[%s/%s] Build queued (new version available %s < %s)", repo, parsed["pkgbase"], parse_repo(packages[0], repo), parse_pkgbuild(pkgbuild)) logging.info("Build queue size: %s", q.qsize()) if __name__ == '__main__': with open("config.yaml") as c: config = yaml.safe_load(c) logging.config.dictConfig(config["logging"]) logging.getLogger("ALHP") if already_running(): logging.error("Another instance is already running") sys.exit(2) if not os.path.exists(config["basedir"]["repo"]): pathlib.Path(config["basedir"]["repo"]).mkdir(parents=True, exist_ok=True) os.nice(5) setup_chroot() sync_marchs_with_config() update_svn2git() q = JoinableQueue() with Pool(config["build"]["worker"], initializer=run_worker) as pool: fill_queue() signal.signal(signal.SIGINT, signal.default_int_handler) while True: try: if time.time() - update_last > 900 and q.empty(): logging.info("[SVN2GIT] Waiting for queue to finish...") q.join() update_last = time.time() update_svn2git() setup_chroot() fill_queue() if q.qsize() > 0: logging.info("[SVN2GIT] New Queue size: %d", q.qsize()) else: time.sleep(300) do_repo_work() except KeyboardInterrupt: with copy_l: pool.close() pool.terminate() q.close() do_repo_work() sys.exit(0)