anonfunc/alhp
Archived
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

delete packages failing to build or not meant to build

Browse Source
This commit is contained in:
Giovanni Harting
2021-06-02 01:44:23 +02:00
parent e625fcc4fc
commit 783575045f
2 changed files with 61 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

86
master.py
View File

@@ -13,15 +13,15 @@ import sys
import time
import traceback
from multiprocessing import Pool, current_process, JoinableQueue, Lock
from typing import AnyStr
import yaml
from humanfriendly import format_timespan
from packaging.version import LegacyVersion
from utils import parse_pkgbuild, parse_pkgbuild_ver, import_keys
from utils import parse_pkgbuild, parse_pkgbuild_ver, import_keys, increase_pkgrel
regex_pkgver = re.compile(r"^_?pkgver\s*=\s*(.+)$", re.MULTILINE)
regex_pkgrel = re.compile(r"^pkgrel\s*=\s*(.+)$", re.MULTILINE)
regex_epoch = re.compile(r"^epoch\s*=\s*(.+)$", re.MULTILINE)
regex_march = re.compile(r"(-march=)(.+?) ", re.MULTILINE)
regex_validkeys = re.compile(r"^validpgpkeys\+?=\((.*?)\)", re.MULTILINE | re.DOTALL)
@@ -142,23 +142,39 @@ def already_running() -> bool:
return True
def package_exists(names: list, repo) -> bool:
for name in names:
pkgs = find_all_files_for_pkg(name, repo)
if not pkgs:
return False
return True
def delete_package(parsed_pkgbuild: dict, repo: list[AnyStr]):
for pkg in parsed_pkgbuild["packages"]:
pkg_f = find_package_files(pkg, repo)
if pkg_f:
for f in pkg_f:
base = pathlib.Path(f).parent
repo = f.split("/")[-3]
args = ["repo-remove", "-s", "-v", os.path.join(base, repo + ".db.tar.xz"), pkg]
r_res = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
logging.debug("[REPO-REMOVE] %s", r_res.stdout.decode(errors="ignore"))
if r_res.returncode:
logging.error("[REPO/%s] Repo action failed: %s", repo, r_res.stdout.decode(errors="ignore"))
continue
os.remove(f)
os.remove(f + ".sig")
def find_all_files_for_pkg(name: str, repo: str) -> list:
def find_package_files(name: str, repo: list[AnyStr]) -> list[AnyStr]:
pkgs = []
for root, dirs, files in os.walk(os.path.join(config["basedir"]["repo"], repo, "os", config["arch"])):
for file in files:
res = regex_pkg_repo.findall(file)
for r in res:
if r == name and not file.endswith(".sig"):
pkgs.append(os.path.join(root, file))
for t_repo in repo:
files: list[str]
root: str
for root, dirs, files in os.walk(os.path.join(config["basedir"]["repo"], t_repo, "os", config["arch"])):
for file in files:
if file.endswith(".sig"):
continue
res = regex_pkg_repo.search(file)
if res and res.group(1) == name:
pkgs.append(os.path.join(root, file))
return pkgs
@@ -222,19 +238,8 @@ def update_svn2git() -> None:
stderr=subprocess.STDOUT).stdout.decode(errors="ignore"))
def increase_pkgrel(pkgbuild_file) -> None:
parsed = parse_pkgbuild(pkgbuild_file)
with open(pkgbuild_file, "r+", errors='ignore') as p:
pkgbuild_str = p.read()
p.truncate(0)
p.seek(0, 0)
pkgbuild_str = regex_pkgrel.sub("pkgrel=" + parsed["pkgrel"] + ".1", pkgbuild_str)
p.write(pkgbuild_str)
def parse_repo(name, repo) -> LegacyVersion:
ver_split = find_all_files_for_pkg(name, repo)[0].split("-")
ver_split = find_package_files(name, repo)[0].split("-")
return LegacyVersion(ver_split[-3] + "-" + ver_split[-2])
@@ -278,34 +283,37 @@ def fill_queue() -> None:
for pkgbuild in all_pkgbuild:
path_split = pkgbuild.split("/")
parsed = parse_pkgbuild(pkgbuild)
parsed_pkgb = parse_pkgbuild(pkgbuild)
# ignore pkgbuild if in trunk, -any package, not in repos, on blacklist, not for current arch
if path_split[-2] == "trunk" or path_split[-2].split("-")[0] not in config["repos"] or "any" in parsed["arch"] \
or parsed["pkgbase"] in config["blacklist"] or "i686" in path_split[-2]:
# TODO: delete packages not to build
pass
if path_split[-2] == "trunk" or path_split[-2].split("-")[0] not in config["repos"] or "any" in parsed_pkgb[
"arch"] \
or parsed_pkgb["pkgbase"] in config["blacklist"] or "i686" in path_split[-2]:
delete_package(parsed_pkgb, repos)
else:
for march in config["march"]:
repo = path_split[-2].split("-")[0] + "-" + march
if parsed["pkgbase"] in get_failed_packages(repo):
logging.info("[%s/%s] Skipped due to failing build", repo, parsed["pkgbase"])
if parsed_pkgb["pkgbase"] in get_failed_packages(repo):
logging.info("[%s/%s] Skipped due to failing build", repo, parsed_pkgb["pkgbase"])
delete_package(parsed_pkgb, [repo, ])
continue
ver = parse_pkgbuild_ver(parsed=parsed)
packages = list(parsed["packages"])
if package_exists(packages, repo):
ver = parse_pkgbuild_ver(parsed=parsed_pkgb)
packages = list(parsed_pkgb["packages"])
pkg_f = find_package_files(packages[0], [repo, ])
if pkg_f:
logging.debug("[SEMVER] Comparing %s=%s - %s=%s", packages[0], parse_repo(packages[0], repo),
packages[0], ver)
rv = parse_repo(packages[0], repo)
if rv < ver:
q.put((pkgbuild, repo))
logging.info("[%s/%s] Build queued (new version available %s < %s)", repo, parsed["pkgbase"],
logging.info("[%s/%s] Build queued (new version available %s < %s)", repo,
parsed_pkgb["pkgbase"],
rv, ver)
else:
q.put((pkgbuild, repo))
logging.info("[%s/%s] Build queued (package not build yet)", repo, parsed["pkgbase"])
logging.info("[%s/%s] Build queued (package not build yet)", repo, parsed_pkgb["pkgbase"])
logging.info("Build queue size: %s", q.qsize())

14
utils.py
View File

@@ -1,11 +1,14 @@
import logging
import os
import pathlib
import re
import subprocess
from packaging.version import LegacyVersion
from srcinfo.parse import parse_srcinfo
regex_pkgrel = re.compile(r"^pkgrel\s*=\s*(.+)$", re.MULTILINE)
def import_keys(pkgbuild: str) -> bool:
parsed = parse_pkgbuild(pkgbuild)
@@ -24,6 +27,17 @@ def import_keys(pkgbuild: str) -> bool:
return True
def increase_pkgrel(pkgbuild_file) -> None:
parsed = parse_pkgbuild(pkgbuild_file)
with open(pkgbuild_file, "r+", errors='ignore') as p:
pkgbuild_str = p.read()
p.truncate(0)
p.seek(0, 0)
pkgbuild_str = regex_pkgrel.sub("pkgrel=" + parsed["pkgrel"] + ".1", pkgbuild_str)
p.write(pkgbuild_str)
def parse_pkgbuild(pkgbuild_file: str) -> dict:
pkgbuild_path = pathlib.Path(pkgbuild_file)
os.chdir(pkgbuild_path.parent)