anonfunc/alhp
Archived
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

fixed repo maintenance

Browse Source
This commit is contained in:
Giovanni Harting
2021-05-26 17:09:46 +02:00
parent 528f5c387b
commit 137b963969
2 changed files with 80 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
config.yaml
View File

@@ -9,10 +9,10 @@ svn2git:
upstream-community: "https://github.com/archlinux/svntogit-community.git" upstream-community: "https://github.com/archlinux/svntogit-community.git"
basedir: basedir:
repo: /tmp/repo/ repo: /home/harting/repo/
chroot: /tmp/chroot/ chroot: /home/harting/chroot/
makepkg: /tmp/makepkg/ makepkg: /home/harting/makepkg/
upstream: /tmp/upstream/ upstream: /home/harting/upstream/
march: march:
- x86-64-v3 - x86-64-v3
@@ -21,6 +21,9 @@ blacklist:
- pacman - pacman
- tensorflow - tensorflow
- tensorflow-cuda - tensorflow-cuda
- brotli
- libarchive
- libb2
build: build:
worker: 4 worker: 4

136
master.py
View File

@@ -11,7 +11,7 @@ import signal
import subprocess import subprocess
import sys import sys
import time import time
from multiprocessing import Pool, current_process, JoinableQueue, Lock from multiprocessing import Pool, current_process, JoinableQueue, Lock, Manager
import yaml import yaml
from humanfriendly import format_timespan from humanfriendly import format_timespan
@@ -26,11 +26,9 @@ regex_validkeys = re.compile(r"^validpgpkeys\+?=\((.*?)\)", re.MULTILINE | re.DO
fp = None fp = None
update_last = time.time() update_last = time.time()
copy_l = Lock() copy_l = Lock()
to_add = {}
to_add_l = Lock()
def build(pkgbuild, repo) -> None: def build(pkgbuild: str, repo: str, todo: dict) -> None:
start_time = time.time() start_time = time.time()
name = pathlib.Path(pkgbuild).parts[-4] name = pathlib.Path(pkgbuild).parts[-4]
process_name = current_process().name process_name = current_process().name
@@ -64,7 +62,7 @@ def build(pkgbuild, repo) -> None:
pathlib.Path(os.path.join(config["basedir"]["repo"], "logs", repo)).mkdir(parents=True, pathlib.Path(os.path.join(config["basedir"]["repo"], "logs", repo)).mkdir(parents=True,
exist_ok=True) exist_ok=True)
with open(os.path.join(config["basedir"]["repo"], "logs", repo, name + ".log"), "w") as log: with open(os.path.join(config["basedir"]["repo"], "logs", repo, name + ".log"), "w") as log:
log.write(res.stdout.decode()) log.write(res.stdout.decode(errors="ignore"))
return return
@@ -74,7 +72,8 @@ def build(pkgbuild, repo) -> None:
s_res = subprocess.run(["gpg", "--batch", "--detach-sign", pkg], stdout=subprocess.PIPE, s_res = subprocess.run(["gpg", "--batch", "--detach-sign", pkg], stdout=subprocess.PIPE,
stderr=subprocess.STDOUT) stderr=subprocess.STDOUT)
if s_res.returncode: if s_res.returncode:
logging.error("[%s/%s/%s] Signing failed: %s", process_name, repo, name, s_res.stdout.decode()) logging.error("[%s/%s/%s] Signing failed: %s", process_name, repo, name,
s_res.stdout.decode(errors="ignore"))
return return
# copying # copying
@@ -86,18 +85,19 @@ def build(pkgbuild, repo) -> None:
shutil.copy2(pkg, os.path.join(config["basedir"]["repo"], repo, "os", config["arch"] + "/")) shutil.copy2(pkg, os.path.join(config["basedir"]["repo"], repo, "os", config["arch"] + "/"))
# repo # repo
with to_add_l: logging.debug("[%s/%s/%s] Adding packages to todo list: %s", process_name, repo, name,
to_add[repo].extend(glob.glob("*.pkg.tar.zst")) ", ".join(glob.glob("*.pkg.tar.zst")))
todo[repo].extend(glob.glob("*.pkg.tar.zst"))
logging.info("[%s/%s/%s] Build successful (%s)", process_name, repo, name, logging.info("[%s/%s/%s] Build successful (%s)", process_name, repo, name,
format_timespan(time.time() - start_time)) format_timespan(time.time() - start_time))
def run_worker() -> None: def run_worker(todo: dict) -> None:
os.nice(20) os.nice(20)
while True: while True:
try: try:
build(*q.get(block=True)) build(*q.get(block=True), todo=todo)
except Exception as e: except Exception as e:
logging.error("Error in worker: %s", e) logging.error("Error in worker: %s", e)
finally: finally:
@@ -106,25 +106,28 @@ def run_worker() -> None:
def do_repo_work(): def do_repo_work():
with to_add_l: for repo in d:
for repo in to_add: if d[repo]:
if to_add[repo]: logging.info("[REPO] Adding %s to %s", ", ".join(d[repo]), repo)
logging.info("Adding to %s: %s", repo, ", ".join(to_add[repo])) os.chdir(os.path.join(config["basedir"]["repo"], repo, "os", config["arch"]))
args = ["repo-add", "-s", "-v", args = ["repo-add", "-s", "-v",
os.path.join(config["basedir"]["repo"], repo, "os", config["arch"], repo + ".db.tar.xz")] os.path.join(config["basedir"]["repo"], repo, "os", config["arch"], repo + ".db.tar.xz")]
args.extend(to_add[repo]) args.extend(d[repo])
r_res = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) r_res = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
logging.debug("[REPO-ADD] %s", r_res.stdout.decode()) logging.debug("[REPO-ADD] %s", r_res.stdout.decode(errors="ignore"))
if r_res.returncode: if r_res.returncode:
logging.error("[%s] Repo action failed: %s", repo, r_res.stdout.decode()) logging.error("[REPO/%s] Repo action failed: %s", repo, r_res.stdout.decode(errors="ignore"))
p_res = subprocess.run( p_res = subprocess.run(
["paccache", "-rc", os.path.join(config["basedir"]["repo"], repo, "os", config["arch"]), "-k", "1"], ["paccache", "-rc", os.path.join(config["basedir"]["repo"], repo, "os", config["arch"]), "-k", "1"],
stdout=subprocess.PIPE, stderr=subprocess.STDOUT) stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
logging.debug("[PACCACHE] %s", p_res.stdout.decode()) logging.debug("[PACCACHE] %s", p_res.stdout.decode(errors="ignore"))
if p_res.returncode: if p_res.returncode:
logging.error("[%s] Repo cleanup failed: %s", repo, p_res.stdout.decode()) logging.error("[REPO/%s] Repo cleanup failed: %s", repo, p_res.stdout.decode(errors="ignore"))
to_add[repo].clear() d[repo][:] = []
os.chdir(sys.path[0])
else:
logging.debug("[REPO] Nothing to do for %s", repo)
def already_running() -> bool: def already_running() -> bool:
@@ -160,10 +163,13 @@ def get_failed_packages(repo) -> list:
def setup_chroot(): def setup_chroot():
if not os.path.exists(os.path.join(config["basedir"]["chroot"], "root")): if not os.path.exists(os.path.join(config["basedir"]["chroot"], "root")):
pathlib.Path(config["basedir"]["chroot"]).mkdir(parents=True, exist_ok=True) pathlib.Path(config["basedir"]["chroot"]).mkdir(parents=True, exist_ok=True)
logging.debug("[MKCHROOT] %s", s = subprocess.run(["mkarchroot", "-C", "/usr/share/devtools/pacman-extra.conf",
subprocess.run( os.path.join(config["basedir"]["chroot"], "root"), "base-devel"],
["mkarchroot", os.path.join(config["basedir"]["chroot"], "root"), "base-devel"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout.decode(errors='ignore')) logging.debug("[MKCHROOT] %s", s.stdout.decode(errors='ignore'))
if s.returncode:
logging.fatal("[MKCHROOT] Failed to create root chroot: %s", s.stdout.decode(errors="ignore"))
sys.exit(2)
else: else:
logging.debug("[NSPAWN] %s", subprocess.run( logging.debug("[NSPAWN] %s", subprocess.run(
["arch-nspawn", os.path.join(config["basedir"]["chroot"], "root"), "pacman", "-Syuu", "--noconfirm"])) ["arch-nspawn", os.path.join(config["basedir"]["chroot"], "root"), "pacman", "-Syuu", "--noconfirm"]))
@@ -227,15 +233,16 @@ def update_svn2git() -> None:
if not os.path.exists(git_path): if not os.path.exists(git_path):
logging.debug("[GIT] %s", logging.debug("[GIT] %s",
subprocess.run(["git", "clone", "--depth=1", git_url, git_path], check=True, subprocess.run(["git", "clone", "--depth=1", git_url, git_path], check=True,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout.decode()) stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout.decode(
errors="ignore"))
else: else:
os.chdir(git_path) os.chdir(git_path)
logging.debug("[GIT] %s", subprocess.run(["git", "clean", "-xdf"], check=False, stdout=subprocess.PIPE, logging.debug("[GIT] %s", subprocess.run(["git", "clean", "-xdf"], check=False, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT).stdout.decode()) stderr=subprocess.STDOUT).stdout.decode(errors="ignore"))
logging.debug("[GIT] %s", subprocess.run(["git", "reset", "--hard"], check=True, stdout=subprocess.PIPE, logging.debug("[GIT] %s", subprocess.run(["git", "reset", "--hard"], check=True, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT).stdout.decode()) stderr=subprocess.STDOUT).stdout.decode(errors="ignore"))
logging.debug("[GIT] %s", subprocess.run(["git", "pull"], check=True, stdout=subprocess.PIPE, logging.debug("[GIT] %s", subprocess.run(["git", "pull"], check=True, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT).stdout.decode()) stderr=subprocess.STDOUT).stdout.decode(errors="ignore"))
os.chdir(sys.path[0]) os.chdir(sys.path[0])
@@ -288,7 +295,7 @@ def sync_marchs_with_config() -> None:
repos_create = list(set(repo_quota) - set(repos)) repos_create = list(set(repo_quota) - set(repos))
repos_delete = list(set(repos) - set(repo_quota)) repos_delete = list(set(repos) - set(repo_quota))
for repo in repo_quota: for repo in repo_quota:
to_add[repo] = [] d[repo] = m.list()
for repo in repos_create: for repo in repos_create:
logging.debug("Create repo %s: %s", repo, os.path.join(config["basedir"]["repo"], repo, "os/x86_64")) logging.debug("Create repo %s: %s", repo, os.path.join(config["basedir"]["repo"], repo, "os/x86_64"))
@@ -361,31 +368,34 @@ if __name__ == '__main__':
os.nice(5) os.nice(5)
setup_chroot() with Manager() as m:
sync_marchs_with_config() d = m.dict()
update_svn2git() setup_chroot()
q = JoinableQueue() sync_marchs_with_config()
update_svn2git()
q = JoinableQueue()
with Pool(config["build"]["worker"], initializer=run_worker) as pool: with Pool(config["build"]["worker"], initializer=run_worker, initargs=(d,)) as pool:
fill_queue() fill_queue()
signal.signal(signal.SIGINT, signal.default_int_handler) signal.signal(signal.SIGINT, signal.default_int_handler)
while True: while True:
try: try:
if time.time() - update_last > 900 and q.qsize() == 0: if time.time() - update_last > 900 and q.empty():
q.join() q.join()
update_last = time.time() do_repo_work()
update_svn2git() update_last = time.time()
fill_queue() update_svn2git()
if q.qsize() > 0: fill_queue()
logging.info("New Queue size: %d", q.qsize()) if q.qsize() > 0:
else: logging.info("New Queue size: %d", q.qsize())
do_repo_work() else:
time.sleep(60) time.sleep(60)
except KeyboardInterrupt: do_repo_work()
with copy_l: except KeyboardInterrupt:
pool.close() with copy_l:
pool.terminate() pool.close()
q.close() pool.terminate()
do_repo_work() q.close()
sys.exit(0) do_repo_work()
sys.exit(0)