changed way signing works
This commit is contained in:
107
master.py
107
master.py
@@ -29,7 +29,7 @@ update_last = time.time()
|
||||
copy_l = Lock()
|
||||
|
||||
|
||||
def build(pkgbuild: str, repo: str, todo: dict) -> None:
|
||||
def build(pkgbuild: str, repo: str) -> None:
|
||||
start_time = time.time()
|
||||
name = pathlib.Path(pkgbuild).parts[-4]
|
||||
process_name = current_process().name
|
||||
@@ -90,11 +90,6 @@ def build(pkgbuild: str, repo: str, todo: dict) -> None:
|
||||
os.path.join(config["basedir"]["repo"], repo, "os", config["arch"] + "/"))
|
||||
shutil.copy2(pkg, os.path.join(config["basedir"]["repo"], repo, "os", config["arch"] + "/"))
|
||||
|
||||
# repo
|
||||
logging.debug("[%s/%s/%s] Adding packages to todo list: %s", process_name, repo, name,
|
||||
", ".join(glob.glob("*.pkg.tar.zst")))
|
||||
todo[repo].extend(glob.glob("*.pkg.tar.zst"))
|
||||
|
||||
logging.debug("[GIT] %s", subprocess.run(["git", "clean", "-xdff"], check=False, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT).stdout.decode(errors="ignore"))
|
||||
|
||||
@@ -102,11 +97,11 @@ def build(pkgbuild: str, repo: str, todo: dict) -> None:
|
||||
format_timespan(time.time() - start_time))
|
||||
|
||||
|
||||
def run_worker(todo: dict) -> None:
|
||||
def run_worker() -> None:
|
||||
os.nice(20)
|
||||
while True:
|
||||
try:
|
||||
build(*q.get(block=True), todo=todo)
|
||||
build(*q.get(block=True))
|
||||
except Exception as e:
|
||||
logging.error("Error in worker: %s", e)
|
||||
finally:
|
||||
@@ -115,28 +110,21 @@ def run_worker(todo: dict) -> None:
|
||||
|
||||
|
||||
def do_repo_work() -> None:
|
||||
for repo in d:
|
||||
if d[repo]:
|
||||
logging.info("[REPO/%s] Adding %s", repo, ", ".join(d[repo]))
|
||||
os.chdir(os.path.join(config["basedir"]["repo"], repo, "os", config["arch"]))
|
||||
args = ["repo-add", "-s", "-v",
|
||||
os.path.join(config["basedir"]["repo"], repo, "os", config["arch"], repo + ".db.tar.xz")]
|
||||
args.extend(d[repo])
|
||||
r_res = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
||||
logging.debug("[REPO-ADD] %s", r_res.stdout.decode(errors="ignore"))
|
||||
if r_res.returncode:
|
||||
logging.error("[REPO/%s] Repo action failed: %s", repo, r_res.stdout.decode(errors="ignore"))
|
||||
for repo in config["repos"]:
|
||||
args = ["repo-add", "-s", "-v", "-p", "-n",
|
||||
os.path.join(config["basedir"]["repo"], repo, "os", config["arch"], repo + ".db.tar.xz"), "*.zst"]
|
||||
r_res = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
||||
logging.debug("[REPO-ADD] %s", r_res.stdout.decode(errors="ignore"))
|
||||
if r_res.returncode:
|
||||
logging.error("[REPO/%s] Repo action failed: %s", repo, r_res.stdout.decode(errors="ignore"))
|
||||
|
||||
p_res = subprocess.run(
|
||||
["paccache", "-rc", os.path.join(config["basedir"]["repo"], repo, "os", config["arch"]), "-k", "1"],
|
||||
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
||||
logging.debug("[PACCACHE] %s", p_res.stdout.decode(errors="ignore"))
|
||||
if p_res.returncode:
|
||||
logging.error("[REPO/%s] Repo cleanup failed: %s", repo, p_res.stdout.decode(errors="ignore"))
|
||||
d[repo][:] = []
|
||||
os.chdir(sys.path[0])
|
||||
else:
|
||||
logging.debug("[REPO/%s] Nothing to do", repo)
|
||||
p_res = subprocess.run(
|
||||
["paccache", "-rc", os.path.join(config["basedir"]["repo"], repo, "os", config["arch"]), "-k", "1"],
|
||||
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
||||
logging.debug("[PACCACHE] %s", p_res.stdout.decode(errors="ignore"))
|
||||
if p_res.returncode:
|
||||
logging.error("[REPO/%s] Repo cleanup failed: %s", repo, p_res.stdout.decode(errors="ignore"))
|
||||
os.chdir(sys.path[0])
|
||||
|
||||
|
||||
def already_running() -> bool:
|
||||
@@ -311,8 +299,6 @@ def sync_marchs_with_config() -> None:
|
||||
logging.info("Repos: %s", repo_quota)
|
||||
repos_create = list(set(repo_quota) - set(repos))
|
||||
repos_delete = list(set(repos) - set(repo_quota))
|
||||
for repo in repo_quota:
|
||||
d[repo] = m.list()
|
||||
|
||||
for repo in repos_create:
|
||||
logging.debug("Create repo %s: %s", repo, os.path.join(config["basedir"]["repo"], repo, "os/x86_64"))
|
||||
@@ -385,36 +371,33 @@ if __name__ == '__main__':
|
||||
|
||||
os.nice(5)
|
||||
|
||||
with Manager() as m:
|
||||
d = m.dict()
|
||||
setup_chroot()
|
||||
sync_marchs_with_config()
|
||||
update_svn2git()
|
||||
q = JoinableQueue()
|
||||
setup_chroot()
|
||||
sync_marchs_with_config()
|
||||
update_svn2git()
|
||||
q = JoinableQueue()
|
||||
|
||||
with Pool(config["build"]["worker"], initializer=run_worker, initargs=(d,)) as pool:
|
||||
fill_queue()
|
||||
signal.signal(signal.SIGINT, signal.default_int_handler)
|
||||
with Pool(config["build"]["worker"], initializer=run_worker) as pool:
|
||||
fill_queue()
|
||||
signal.signal(signal.SIGINT, signal.default_int_handler)
|
||||
|
||||
while True:
|
||||
try:
|
||||
if time.time() - update_last > 900 and q.empty():
|
||||
logging.info("[SVN2GIT] Waiting for queue to finish...")
|
||||
q.join()
|
||||
do_repo_work()
|
||||
update_last = time.time()
|
||||
update_svn2git()
|
||||
setup_chroot()
|
||||
fill_queue()
|
||||
if q.qsize() > 0:
|
||||
logging.info("[SVN2GIT] New Queue size: %d", q.qsize())
|
||||
else:
|
||||
time.sleep(60)
|
||||
do_repo_work()
|
||||
except KeyboardInterrupt:
|
||||
with copy_l:
|
||||
pool.close()
|
||||
pool.terminate()
|
||||
q.close()
|
||||
do_repo_work()
|
||||
sys.exit(0)
|
||||
while True:
|
||||
try:
|
||||
if time.time() - update_last > 900 and q.empty():
|
||||
logging.info("[SVN2GIT] Waiting for queue to finish...")
|
||||
q.join()
|
||||
update_last = time.time()
|
||||
update_svn2git()
|
||||
setup_chroot()
|
||||
fill_queue()
|
||||
if q.qsize() > 0:
|
||||
logging.info("[SVN2GIT] New Queue size: %d", q.qsize())
|
||||
else:
|
||||
time.sleep(300)
|
||||
do_repo_work()
|
||||
except KeyboardInterrupt:
|
||||
with copy_l:
|
||||
pool.close()
|
||||
pool.terminate()
|
||||
q.close()
|
||||
do_repo_work()
|
||||
sys.exit(0)
|
||||
|
||||
Reference in New Issue
Block a user