anonfunc/alhp
Archived
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

made upstream dirs configurable, clone only depth 1

Browse Source 
don't exit worker process after exception
This commit is contained in:
Giovanni Harting
2021-05-21 17:43:30 +02:00
parent af5e766b06
commit d0dd9661c6
2 changed files with 20 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
master.py
View File

@@ -28,7 +28,7 @@ def build(pkgbuild, repo):
start_time = time.time()
name = pathlib.Path(pkgbuild).parts[-4]
process_name = current_process().name
logging.info("[%s/%s] Build starting (Queue ~= %s)", repo, name, q.qsize())
logging.info("[%s/%s/%s] Build starting (Queue ~= %s)", process_name, repo, name, q.qsize())
# setup makepkg
setup_makepkg(repo)
@@ -108,7 +108,10 @@ def build(pkgbuild, repo):
def run_worker() -> None:
os.nice(20)
while True:
build(*q.get(block=True))
try:
build(*q.get(block=True))
except Exception as e:
logging.error("Error in worker: %s", e)
def already_running():
@@ -124,15 +127,12 @@ def already_running():
def find_all_files_for_pkg(name, repo):
searchpath = os.path.join(config["basedir"]["repo"], repo, "os", config["arch"]) + "/" + name + "-*.pkg.*"
# logging.debug("[REPO] Search for package %s (path=%s)", name, searchpath)
pkgs = glob.glob(searchpath)
for p in pkgs:
if p.endswith(".sig"):
pkgs.remove(p)
# logging.debug("[REPO] Found %s", pkgs)
return pkgs
@@ -202,21 +202,25 @@ def package_exists(name, repo):
return len(pkgs) > 0
def update_git2svn():
def update_svn2git():
if not os.path.exists(config["basedir"]["upstream"]):
pathlib.Path(config["basedir"]["upstream"]).mkdir(parents=True, exist_ok=True)
for git_dir, git_url in config["svn2git"].items():
if not os.path.exists(git_dir):
git_path = os.path.join(config["basedir"]["upstream"], git_dir)
if not os.path.exists(git_path):
logging.debug("[GIT] %s",
subprocess.run(["git", "clone", git_url, git_dir], check=True, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT).stdout.decode())
subprocess.run(["git", "clone", "--depth=1", git_url, git_path], check=True,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout.decode())
else:
os.chdir(git_dir)
os.chdir(git_path)
logging.debug("[GIT] %s", subprocess.run(["git", "clean", "-xdf"], check=False, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT).stdout.decode())
logging.debug("[GIT] %s", subprocess.run(["git", "reset", "--hard"], check=True, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT).stdout.decode())
logging.debug("[GIT] %s", subprocess.run(["git", "pull"], check=True, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT).stdout.decode())
os.chdir("..")
os.chdir(sys.path[0])
def parse_pkgbuild(pkgbuild_file):
@@ -280,7 +284,8 @@ def fill_queue():
all_pkgbuild = []
for git_dir, git_url in config["svn2git"].items():
all_pkgbuild.extend(glob.glob(os.path.join(git_dir) + "/**/PKGBUILD", recursive=True))
all_pkgbuild.extend(
glob.glob(os.path.join(config["basedir"]["upstream"], git_dir) + "/**/PKGBUILD", recursive=True))
to_delete = []
@@ -337,7 +342,7 @@ if __name__ == '__main__':
setup_chroot()
sync_marchs_with_config()
update_git2svn()
update_svn2git()
q = Queue()
with Pool(config["build"]["worker"], initializer=run_worker) as pool:
@@ -348,7 +353,7 @@ if __name__ == '__main__':
try:
if time.time() - update_last > 900 and q.qsize() == 0:
update_last = time.time()
update_git2svn()
update_svn2git()
fill_queue()
else:
time.sleep(60)