added multiple workers, added community
This commit is contained in:
252
master.py
252
master.py
@@ -7,10 +7,11 @@ import os
|
||||
import pathlib
|
||||
import re
|
||||
import shutil
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from queue import Queue, Empty
|
||||
from multiprocessing import Pool, Queue, current_process
|
||||
|
||||
import yaml
|
||||
from packaging import version
|
||||
@@ -20,10 +21,93 @@ regex_pkgrel = re.compile(r"^pkgrel\s*=\s*(.+)$", re.MULTILINE)
|
||||
regex_march = re.compile(r"(-march=)(.+?) ", re.MULTILINE)
|
||||
regex_validkeys = re.compile(r"^validpgpkeys\+?=\((.*?)\)", re.MULTILINE | re.DOTALL)
|
||||
fp = None
|
||||
q = Queue()
|
||||
update_last = time.time()
|
||||
|
||||
|
||||
def build(pkgbuild, repo):
|
||||
start_time = time.time()
|
||||
name = pathlib.Path(pkgbuild).parts[-4]
|
||||
process_name = current_process().name
|
||||
logging.info("[%s/%s] Build starting (Queue ~= %s)", repo, name, q.qsize())
|
||||
|
||||
# setup makepkg
|
||||
setup_makepkg(repo)
|
||||
|
||||
# import pgp keys
|
||||
import_keys(pkgbuild)
|
||||
|
||||
# build with devtools
|
||||
os.chdir(pathlib.Path(pkgbuild).parent)
|
||||
res = subprocess.run(
|
||||
["makechrootpkg", "-D", os.path.join(sys.path[0], config["basedir"]["makepkg"]), "-l", process_name, "-r",
|
||||
os.path.join(sys.path[0], config["basedir"]["chroot"]), "--", "--config",
|
||||
os.path.join(sys.path[0], config["basedir"]["makepkg"]) + "makepkg-" + '-'.join(
|
||||
repo.split("-")[1:]) + ".conf"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
||||
if res.returncode:
|
||||
logging.warning("[%s/%s] Build failed. Check repo/logs for more information.", repo, name)
|
||||
|
||||
# write packagename to failed list
|
||||
with open(os.path.join(config["basedir"]["repo"], repo + "_failed.txt"), "a") as f:
|
||||
f.write(name + "\n")
|
||||
|
||||
# write logs
|
||||
if not os.path.exists(os.path.join(config["basedir"]["repo"], "logs", repo)):
|
||||
pathlib.Path(os.path.join(sys.path[0], config["basedir"]["repo"], "logs", repo)).mkdir(parents=True,
|
||||
exist_ok=True)
|
||||
with open(os.path.join(config["basedir"]["repo"], "logs", repo, name + ".log"), "w") as log:
|
||||
log.write(res.stdout.decode())
|
||||
|
||||
build_cleanup()
|
||||
return
|
||||
|
||||
# signing
|
||||
pkgs = glob.glob("*.pkg.tar.zst")
|
||||
for pkg in pkgs:
|
||||
s_res = subprocess.run(["gpg", "--batch", "--detach-sign", pkg], stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT)
|
||||
if s_res.returncode:
|
||||
logging.error("[%s/%s] Signing failed: %s", repo, name, s_res.stdout.decode())
|
||||
build_cleanup()
|
||||
return
|
||||
|
||||
# copying
|
||||
pkgs.extend(glob.glob("*.pkg.tar.zst.sig"))
|
||||
for pkg in pkgs:
|
||||
logging.debug("[%s/%s] Copy %s to %s", repo, name, pkg,
|
||||
os.path.join(config["basedir"]["repo"], repo, "os", config["arch"] + "/"))
|
||||
shutil.copy2(pkg, os.path.join(config["basedir"]["repo"], repo, "os", config["arch"] + "/"))
|
||||
|
||||
# repo
|
||||
r_res = subprocess.run(["repo-add", "-s", "-v",
|
||||
os.path.join(config["basedir"]["repo"], repo, "os", config["arch"],
|
||||
repo + ".db.tar.xz"),
|
||||
pkgs[0]], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
||||
logging.debug("[REPO-ADD] %s", r_res.stdout.decode())
|
||||
if r_res.returncode:
|
||||
logging.error("[%s/%s] Repo action failed: %s", repo, name, r_res.stdout.decode())
|
||||
build_cleanup()
|
||||
return
|
||||
|
||||
p_res = subprocess.run(
|
||||
["paccache", "-rc", os.path.join(config["basedir"]["repo"], repo, "os", config["arch"]), "-k", "1"],
|
||||
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
||||
logging.debug("[PACCACHE] %s", p_res.stdout.decode())
|
||||
if p_res.returncode:
|
||||
logging.error("[%s/%s] Repo cleanup failed: %s", repo, name, p_res.stdout.decode())
|
||||
build_cleanup()
|
||||
return
|
||||
|
||||
# cleanup
|
||||
build_cleanup()
|
||||
logging.info("[%s/%s] Build successful (%s)", repo, name, int(time.time() - start_time))
|
||||
|
||||
|
||||
def run_worker() -> None:
|
||||
os.nice(20)
|
||||
while True:
|
||||
build(*q.get(block=True))
|
||||
|
||||
|
||||
def already_running():
|
||||
global fp
|
||||
fp = os.open(f"/tmp/alhp.lock", os.O_WRONLY | os.O_CREAT)
|
||||
@@ -63,85 +147,29 @@ def build_cleanup():
|
||||
os.chdir(sys.path[0])
|
||||
|
||||
|
||||
def build(pkgbuild, repo):
|
||||
start_time = time.time()
|
||||
name = pathlib.Path(pkgbuild).parts[-4]
|
||||
logging.info("[%s/%s] Build starting (Queue ~= %s)", repo, name, q.qsize())
|
||||
|
||||
# setup buildflags
|
||||
setup_makepkg(repo)
|
||||
|
||||
# import pgp keys
|
||||
import_keys(pkgbuild)
|
||||
|
||||
# build with devtools
|
||||
os.chdir(pathlib.Path(pkgbuild).parent)
|
||||
res = subprocess.run(["sudo", "extra-x86_64-build"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
||||
if res.returncode:
|
||||
logging.warning("[%s/%s] Build failed. Check repo/logs for more information.", repo, name)
|
||||
|
||||
# write packagename to failed list
|
||||
with open(os.path.join(config["basedir"]["repo"], repo + "_failed.txt"), "a") as f:
|
||||
f.write(name + "\n")
|
||||
|
||||
# write logs
|
||||
if not os.path.exists(os.path.join(config["basedir"]["repo"], "logs", repo)):
|
||||
pathlib.Path(os.path.join(config["basedir"]["repo"], "logs", repo)).mkdir(parents=True, exist_ok=True)
|
||||
with open(os.path.join(config["basedir"]["repo"], "logs", repo, name + ".log"), "w") as log:
|
||||
log.write(res.stdout.decode())
|
||||
|
||||
build_cleanup()
|
||||
return
|
||||
|
||||
# signing
|
||||
pkgs = glob.glob("*.pkg.tar.zst")
|
||||
for pkg in pkgs:
|
||||
s_res = subprocess.run(["gpg", "--batch", "--detach-sign", pkg], stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT)
|
||||
if s_res.returncode:
|
||||
logging.error("[%s/%s] Signing failed: %s", repo, name, s_res.stdout.decode())
|
||||
build_cleanup()
|
||||
return
|
||||
|
||||
# copying
|
||||
pkgs.extend(glob.glob("*.pkg.tar.zst.sig"))
|
||||
for pkg in pkgs:
|
||||
logging.debug("[%s/%s] Copy %s to %s", repo, name, pkg,
|
||||
os.path.join(config["basedir"]["repo"], repo, "os", config["arch"] + "/"))
|
||||
shutil.copy2(pkg, os.path.join(config["basedir"]["repo"], repo, "os", config["arch"] + "/"))
|
||||
|
||||
# repo
|
||||
r_res = subprocess.run(["repo-add", "-s", "-v",
|
||||
os.path.join(config["basedir"]["repo"], repo, "os", config["arch"], repo + ".db.tar.xz"),
|
||||
pkgs[0]], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
||||
logging.debug("[REPO-ADD] %s", r_res.stdout.decode())
|
||||
if r_res.returncode:
|
||||
logging.error("[%s/%s] Repo action failed: %s", repo, name, r_res.stdout.decode())
|
||||
build_cleanup()
|
||||
return
|
||||
|
||||
p_res = subprocess.run(
|
||||
["paccache", "-rc", os.path.join(config["basedir"]["repo"], repo, "os", config["arch"]), "-k", "1"],
|
||||
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
||||
logging.debug("[PACCACHE] %s", p_res.stdout.decode())
|
||||
if p_res.returncode:
|
||||
logging.error("[%s/%s] Repo cleanup failed: %s", repo, name, p_res.stdout.decode())
|
||||
build_cleanup()
|
||||
return
|
||||
|
||||
# cleanup
|
||||
build_cleanup()
|
||||
logging.info("[%s/%s] Build successful (%s)", repo, name, int(time.time() - start_time))
|
||||
def setup_chroot():
|
||||
if not os.path.exists(os.path.join(config["basedir"]["chroot"], "root")):
|
||||
pathlib.Path(config["basedir"]["chroot"]).mkdir(parents=True, exist_ok=True)
|
||||
logging.debug("[MKCHROOT] %s",
|
||||
subprocess.run(
|
||||
["mkarchroot", os.path.join(config["basedir"]["chroot"], "root"), "base-devel"],
|
||||
stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout.decode(errors='ignore'))
|
||||
|
||||
|
||||
def setup_makepkg(repo):
|
||||
with open(config["basedir"]["makepkg"]) as conf:
|
||||
c_all = conf.read()
|
||||
c_all = c_all.replace("-mtune=generic", "")
|
||||
c_all = c_all.replace("-O2", "-O3")
|
||||
c_all = regex_march.sub(r"\1" + repo.split("-")[1] + " ", c_all)
|
||||
with open(config["basedir"]["makepkg"], "w") as conf:
|
||||
conf.write(c_all)
|
||||
makepkg_repo = os.path.join(config["basedir"]["makepkg"], "makepkg-" + '-'.join(repo.split("-")[1:]) + ".conf")
|
||||
|
||||
if not os.path.exists(makepkg_repo):
|
||||
pathlib.Path(config["basedir"]["makepkg"]).mkdir(parents=True, exist_ok=True)
|
||||
shutil.copyfile("makepkg.tmpl", makepkg_repo)
|
||||
|
||||
with open(makepkg_repo) as conf:
|
||||
c_all = conf.read()
|
||||
c_all = c_all.replace("-mtune=generic", "")
|
||||
c_all = c_all.replace("-O2", "-O3")
|
||||
c_all = regex_march.sub(r"\1" + '-'.join(repo.split("-")[1:]) + " ", c_all)
|
||||
with open(makepkg_repo, "w") as conf:
|
||||
conf.write(c_all)
|
||||
|
||||
|
||||
def import_keys(pkgbuild):
|
||||
@@ -174,17 +202,18 @@ def package_exists(name, repo):
|
||||
|
||||
|
||||
def update_git2svn():
|
||||
if not os.path.exists(config["basedir"]["svn2git"]):
|
||||
logging.debug("[GIT] %s", subprocess.run(
|
||||
["git", "clone", "https://github.com/archlinux/svntogit-packages.git", config["basedir"]["svn2git"]],
|
||||
check=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout.decode())
|
||||
else:
|
||||
os.chdir(config["basedir"]["svn2git"])
|
||||
logging.debug("[GIT] %s", subprocess.run(["git", "clean", "-xdf"], check=True, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT).stdout.decode())
|
||||
logging.debug("[GIT] %s", subprocess.run(["git", "pull"], check=True, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT).stdout.decode())
|
||||
os.chdir("..")
|
||||
for git_dir, git_url in config["svn2git"].items():
|
||||
if not os.path.exists(git_dir):
|
||||
logging.debug("[GIT] %s",
|
||||
subprocess.run(["git", "clone", git_url, git_dir], check=True, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT).stdout.decode())
|
||||
else:
|
||||
os.chdir(git_dir)
|
||||
logging.debug("[GIT] %s", subprocess.run(["git", "clean", "-xdf"], check=True, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT).stdout.decode())
|
||||
logging.debug("[GIT] %s", subprocess.run(["git", "pull"], check=True, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT).stdout.decode())
|
||||
os.chdir("..")
|
||||
|
||||
|
||||
def parse_pkgbuild(pkgbuild_file):
|
||||
@@ -226,22 +255,28 @@ def sync_marchs_with_config():
|
||||
for repo in repos_create:
|
||||
logging.debug("Create repo %s: %s", repo, os.path.join(config["basedir"]["repo"], repo, "os/x86_64"))
|
||||
pathlib.Path(os.path.join(config["basedir"]["repo"], repo, "os/x86_64")).mkdir(parents=True, exist_ok=True)
|
||||
setup_makepkg(repo)
|
||||
|
||||
for repo in repos_delete:
|
||||
logging.debug("Delete repo %s: %s", repo, os.path.join(config["basedir"]["repo"], repo))
|
||||
shutil.rmtree(os.path.join(config["basedir"]["repo"], repo))
|
||||
os.remove(os.path.join(config["basedir"]["makepkg"], "makepkg-" + repo + ".conf"))
|
||||
|
||||
|
||||
def fill_queue():
|
||||
all_pkgbuild = glob.glob(os.path.join(config["basedir"]["svn2git"]) + "/**/PKGBUILD", recursive=True)
|
||||
all_pkgbuild = []
|
||||
|
||||
for git_dir, git_url in config["svn2git"].items():
|
||||
all_pkgbuild.extend(glob.glob(os.path.join(git_dir) + "/**/PKGBUILD", recursive=True))
|
||||
|
||||
to_delete = []
|
||||
|
||||
for pkgbuild in all_pkgbuild:
|
||||
path_split = pkgbuild.split("/")
|
||||
|
||||
# ignore pkgbuild if in trunk, -any package, not in repos, or on blacklist
|
||||
if path_split[-2] == "trunk" or path_split[-2].split("-")[0] not in config[
|
||||
"repos"] or "any" in path_split[-2] or path_split[-4] in config["blacklist"]:
|
||||
if path_split[-2] == "trunk" or path_split[-2].split("-")[0] not in config["repos"] or "any" in path_split[-2] \
|
||||
or path_split[-4] in config["blacklist"]:
|
||||
to_delete.append(pkgbuild)
|
||||
|
||||
final_pkgbuilds = list(set(all_pkgbuild) - set(to_delete))
|
||||
@@ -285,20 +320,27 @@ if __name__ == '__main__':
|
||||
if not os.path.exists(config["basedir"]["repo"]):
|
||||
pathlib.Path(config["basedir"]["repo"]).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
os.nice(5)
|
||||
|
||||
setup_chroot()
|
||||
sync_marchs_with_config()
|
||||
update_git2svn()
|
||||
fill_queue()
|
||||
q = Queue()
|
||||
|
||||
while True:
|
||||
if q.qsize() > 0:
|
||||
with Pool(config["build"]["worker"], initializer=run_worker) as pool:
|
||||
fill_queue()
|
||||
signal.signal(signal.SIGINT, signal.default_int_handler)
|
||||
|
||||
while True:
|
||||
try:
|
||||
build(*q.get_nowait())
|
||||
except Empty:
|
||||
pass
|
||||
else:
|
||||
time.sleep(60)
|
||||
|
||||
if time.time() - update_last > 900:
|
||||
update_git2svn()
|
||||
update_last = time.time()
|
||||
fill_queue()
|
||||
if time.time() - update_last > 900:
|
||||
update_last = time.time()
|
||||
update_git2svn()
|
||||
fill_queue()
|
||||
else:
|
||||
time.sleep(60)
|
||||
except KeyboardInterrupt:
|
||||
pool.close()
|
||||
pool.terminate()
|
||||
q.close()
|
||||
sys.exit(0)
|
||||
|
||||
Reference in New Issue
Block a user