anonfunc/alhp
Archived
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

made upstream dirs configurable, clone only depth 1

Browse Source 
don't exit worker process after exception
This commit is contained in:
Giovanni Harting
2021-05-21 17:43:30 +02:00
parent af5e766b06
commit d0dd9661c6
2 changed files with 20 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
config.yaml
View File

@@ -12,6 +12,7 @@ basedir:
repo: /tmp/repo/ repo: /tmp/repo/
chroot: /tmp/chroot/ chroot: /tmp/chroot/
makepkg: /tmp/makepkg/ makepkg: /tmp/makepkg/
upstream: /tmp/upstream/
march: march:
- x86-64-v3 - x86-64-v3

33
master.py
View File

@@ -28,7 +28,7 @@ def build(pkgbuild, repo):
start_time = time.time() start_time = time.time()
name = pathlib.Path(pkgbuild).parts[-4] name = pathlib.Path(pkgbuild).parts[-4]
process_name = current_process().name process_name = current_process().name
logging.info("[%s/%s] Build starting (Queue ~= %s)", repo, name, q.qsize()) logging.info("[%s/%s/%s] Build starting (Queue ~= %s)", process_name, repo, name, q.qsize())
# setup makepkg # setup makepkg
setup_makepkg(repo) setup_makepkg(repo)
@@ -108,7 +108,10 @@ def build(pkgbuild, repo):
def run_worker() -> None: def run_worker() -> None:
os.nice(20) os.nice(20)
while True: while True:
build(*q.get(block=True)) try:
build(*q.get(block=True))
except Exception as e:
logging.error("Error in worker: %s", e)
def already_running(): def already_running():
@@ -124,15 +127,12 @@ def already_running():
def find_all_files_for_pkg(name, repo): def find_all_files_for_pkg(name, repo):
searchpath = os.path.join(config["basedir"]["repo"], repo, "os", config["arch"]) + "/" + name + "-*.pkg.*" searchpath = os.path.join(config["basedir"]["repo"], repo, "os", config["arch"]) + "/" + name + "-*.pkg.*"
# logging.debug("[REPO] Search for package %s (path=%s)", name, searchpath)
pkgs = glob.glob(searchpath) pkgs = glob.glob(searchpath)
for p in pkgs: for p in pkgs:
if p.endswith(".sig"): if p.endswith(".sig"):
pkgs.remove(p) pkgs.remove(p)
# logging.debug("[REPO] Found %s", pkgs)
return pkgs return pkgs
@@ -202,21 +202,25 @@ def package_exists(name, repo):
return len(pkgs) > 0 return len(pkgs) > 0
def update_git2svn(): def update_svn2git():
if not os.path.exists(config["basedir"]["upstream"]):
pathlib.Path(config["basedir"]["upstream"]).mkdir(parents=True, exist_ok=True)
for git_dir, git_url in config["svn2git"].items(): for git_dir, git_url in config["svn2git"].items():
if not os.path.exists(git_dir): git_path = os.path.join(config["basedir"]["upstream"], git_dir)
if not os.path.exists(git_path):
logging.debug("[GIT] %s", logging.debug("[GIT] %s",
subprocess.run(["git", "clone", git_url, git_dir], check=True, stdout=subprocess.PIPE, subprocess.run(["git", "clone", "--depth=1", git_url, git_path], check=True,
stderr=subprocess.STDOUT).stdout.decode()) stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout.decode())
else: else:
os.chdir(git_dir) os.chdir(git_path)
logging.debug("[GIT] %s", subprocess.run(["git", "clean", "-xdf"], check=False, stdout=subprocess.PIPE, logging.debug("[GIT] %s", subprocess.run(["git", "clean", "-xdf"], check=False, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT).stdout.decode()) stderr=subprocess.STDOUT).stdout.decode())
logging.debug("[GIT] %s", subprocess.run(["git", "reset", "--hard"], check=True, stdout=subprocess.PIPE, logging.debug("[GIT] %s", subprocess.run(["git", "reset", "--hard"], check=True, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT).stdout.decode()) stderr=subprocess.STDOUT).stdout.decode())
logging.debug("[GIT] %s", subprocess.run(["git", "pull"], check=True, stdout=subprocess.PIPE, logging.debug("[GIT] %s", subprocess.run(["git", "pull"], check=True, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT).stdout.decode()) stderr=subprocess.STDOUT).stdout.decode())
os.chdir("..") os.chdir(sys.path[0])
def parse_pkgbuild(pkgbuild_file): def parse_pkgbuild(pkgbuild_file):
@@ -280,7 +284,8 @@ def fill_queue():
all_pkgbuild = [] all_pkgbuild = []
for git_dir, git_url in config["svn2git"].items(): for git_dir, git_url in config["svn2git"].items():
all_pkgbuild.extend(glob.glob(os.path.join(git_dir) + "/**/PKGBUILD", recursive=True)) all_pkgbuild.extend(
glob.glob(os.path.join(config["basedir"]["upstream"], git_dir) + "/**/PKGBUILD", recursive=True))
to_delete = [] to_delete = []
@@ -337,7 +342,7 @@ if __name__ == '__main__':
setup_chroot() setup_chroot()
sync_marchs_with_config() sync_marchs_with_config()
update_git2svn() update_svn2git()
q = Queue() q = Queue()
with Pool(config["build"]["worker"], initializer=run_worker) as pool: with Pool(config["build"]["worker"], initializer=run_worker) as pool:
@@ -348,7 +353,7 @@ if __name__ == '__main__':
try: try:
if time.time() - update_last > 900 and q.qsize() == 0: if time.time() - update_last > 900 and q.qsize() == 0:
update_last = time.time() update_last = time.time()
update_git2svn() update_svn2git()
fill_queue() fill_queue()
else: else:
time.sleep(60) time.sleep(60)