From 783575045f417d0db932be48cc596ff9211490dd Mon Sep 17 00:00:00 2001 From: Giovanni Harting <539@idlegandalf.com> Date: Wed, 2 Jun 2021 01:44:23 +0200 Subject: [PATCH] delete packages failing to build or not meant to build --- master.py | 86 ++++++++++++++++++++++++++++++------------------------- utils.py | 14 +++++++++ 2 files changed, 61 insertions(+), 39 deletions(-) diff --git a/master.py b/master.py index 5d6abea..a71ec62 100644 --- a/master.py +++ b/master.py @@ -13,15 +13,15 @@ import sys import time import traceback from multiprocessing import Pool, current_process, JoinableQueue, Lock +from typing import AnyStr import yaml from humanfriendly import format_timespan from packaging.version import LegacyVersion -from utils import parse_pkgbuild, parse_pkgbuild_ver, import_keys +from utils import parse_pkgbuild, parse_pkgbuild_ver, import_keys, increase_pkgrel regex_pkgver = re.compile(r"^_?pkgver\s*=\s*(.+)$", re.MULTILINE) -regex_pkgrel = re.compile(r"^pkgrel\s*=\s*(.+)$", re.MULTILINE) regex_epoch = re.compile(r"^epoch\s*=\s*(.+)$", re.MULTILINE) regex_march = re.compile(r"(-march=)(.+?) ", re.MULTILINE) regex_validkeys = re.compile(r"^validpgpkeys\+?=\((.*?)\)", re.MULTILINE | re.DOTALL) @@ -142,23 +142,39 @@ def already_running() -> bool: return True -def package_exists(names: list, repo) -> bool: - for name in names: - pkgs = find_all_files_for_pkg(name, repo) - if not pkgs: - return False - return True +def delete_package(parsed_pkgbuild: dict, repo: list[AnyStr]): + for pkg in parsed_pkgbuild["packages"]: + pkg_f = find_package_files(pkg, repo) + + if pkg_f: + for f in pkg_f: + base = pathlib.Path(f).parent + repo = f.split("/")[-3] + + args = ["repo-remove", "-s", "-v", os.path.join(base, repo + ".db.tar.xz"), pkg] + r_res = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + logging.debug("[REPO-REMOVE] %s", r_res.stdout.decode(errors="ignore")) + if r_res.returncode: + logging.error("[REPO/%s] Repo action failed: %s", repo, r_res.stdout.decode(errors="ignore")) + continue + os.remove(f) + os.remove(f + ".sig") -def find_all_files_for_pkg(name: str, repo: str) -> list: +def find_package_files(name: str, repo: list[AnyStr]) -> list[AnyStr]: pkgs = [] - for root, dirs, files in os.walk(os.path.join(config["basedir"]["repo"], repo, "os", config["arch"])): - for file in files: - res = regex_pkg_repo.findall(file) - for r in res: - if r == name and not file.endswith(".sig"): - pkgs.append(os.path.join(root, file)) + for t_repo in repo: + files: list[str] + root: str + for root, dirs, files in os.walk(os.path.join(config["basedir"]["repo"], t_repo, "os", config["arch"])): + for file in files: + if file.endswith(".sig"): + continue + + res = regex_pkg_repo.search(file) + if res and res.group(1) == name: + pkgs.append(os.path.join(root, file)) return pkgs @@ -222,19 +238,8 @@ def update_svn2git() -> None: stderr=subprocess.STDOUT).stdout.decode(errors="ignore")) -def increase_pkgrel(pkgbuild_file) -> None: - parsed = parse_pkgbuild(pkgbuild_file) - with open(pkgbuild_file, "r+", errors='ignore') as p: - pkgbuild_str = p.read() - p.truncate(0) - p.seek(0, 0) - - pkgbuild_str = regex_pkgrel.sub("pkgrel=" + parsed["pkgrel"] + ".1", pkgbuild_str) - p.write(pkgbuild_str) - - def parse_repo(name, repo) -> LegacyVersion: - ver_split = find_all_files_for_pkg(name, repo)[0].split("-") + ver_split = find_package_files(name, repo)[0].split("-") return LegacyVersion(ver_split[-3] + "-" + ver_split[-2]) @@ -278,34 +283,37 @@ def fill_queue() -> None: for pkgbuild in all_pkgbuild: path_split = pkgbuild.split("/") - parsed = parse_pkgbuild(pkgbuild) + parsed_pkgb = parse_pkgbuild(pkgbuild) # ignore pkgbuild if in trunk, -any package, not in repos, on blacklist, not for current arch - if path_split[-2] == "trunk" or path_split[-2].split("-")[0] not in config["repos"] or "any" in parsed["arch"] \ - or parsed["pkgbase"] in config["blacklist"] or "i686" in path_split[-2]: - # TODO: delete packages not to build - pass + if path_split[-2] == "trunk" or path_split[-2].split("-")[0] not in config["repos"] or "any" in parsed_pkgb[ + "arch"] \ + or parsed_pkgb["pkgbase"] in config["blacklist"] or "i686" in path_split[-2]: + delete_package(parsed_pkgb, repos) else: for march in config["march"]: repo = path_split[-2].split("-")[0] + "-" + march - if parsed["pkgbase"] in get_failed_packages(repo): - logging.info("[%s/%s] Skipped due to failing build", repo, parsed["pkgbase"]) + if parsed_pkgb["pkgbase"] in get_failed_packages(repo): + logging.info("[%s/%s] Skipped due to failing build", repo, parsed_pkgb["pkgbase"]) + delete_package(parsed_pkgb, [repo, ]) continue - ver = parse_pkgbuild_ver(parsed=parsed) - packages = list(parsed["packages"]) - if package_exists(packages, repo): + ver = parse_pkgbuild_ver(parsed=parsed_pkgb) + packages = list(parsed_pkgb["packages"]) + pkg_f = find_package_files(packages[0], [repo, ]) + if pkg_f: logging.debug("[SEMVER] Comparing %s=%s - %s=%s", packages[0], parse_repo(packages[0], repo), packages[0], ver) rv = parse_repo(packages[0], repo) if rv < ver: q.put((pkgbuild, repo)) - logging.info("[%s/%s] Build queued (new version available %s < %s)", repo, parsed["pkgbase"], + logging.info("[%s/%s] Build queued (new version available %s < %s)", repo, + parsed_pkgb["pkgbase"], rv, ver) else: q.put((pkgbuild, repo)) - logging.info("[%s/%s] Build queued (package not build yet)", repo, parsed["pkgbase"]) + logging.info("[%s/%s] Build queued (package not build yet)", repo, parsed_pkgb["pkgbase"]) logging.info("Build queue size: %s", q.qsize()) diff --git a/utils.py b/utils.py index be00519..3e90638 100644 --- a/utils.py +++ b/utils.py @@ -1,11 +1,14 @@ import logging import os import pathlib +import re import subprocess from packaging.version import LegacyVersion from srcinfo.parse import parse_srcinfo +regex_pkgrel = re.compile(r"^pkgrel\s*=\s*(.+)$", re.MULTILINE) + def import_keys(pkgbuild: str) -> bool: parsed = parse_pkgbuild(pkgbuild) @@ -24,6 +27,17 @@ def import_keys(pkgbuild: str) -> bool: return True +def increase_pkgrel(pkgbuild_file) -> None: + parsed = parse_pkgbuild(pkgbuild_file) + with open(pkgbuild_file, "r+", errors='ignore') as p: + pkgbuild_str = p.read() + p.truncate(0) + p.seek(0, 0) + + pkgbuild_str = regex_pkgrel.sub("pkgrel=" + parsed["pkgrel"] + ".1", pkgbuild_str) + p.write(pkgbuild_str) + + def parse_pkgbuild(pkgbuild_file: str) -> dict: pkgbuild_path = pathlib.Path(pkgbuild_file) os.chdir(pkgbuild_path.parent)