From 8e4c056e7eeafaf92c550b4302a5cf64e7383204 Mon Sep 17 00:00:00 2001 From: Giovanni Harting <539@idlegandalf.com> Date: Mon, 29 Nov 2021 15:02:14 +0100 Subject: [PATCH] added build-dir to allow multiple building the same package for different marchs --- config_dist.yaml | 1 + git_clean.sh | 3 -- main.go | 106 ++++++++++++++++++++++++++++++----------------- utils.go | 52 ++++++++++++++++++----- 4 files changed, 109 insertions(+), 53 deletions(-) delete mode 100644 git_clean.sh diff --git a/config_dist.yaml b/config_dist.yaml index 5b919c1..695d608 100644 --- a/config_dist.yaml +++ b/config_dist.yaml @@ -30,6 +30,7 @@ basedir: chroot: /var/lib/alhp/chroot/ makepkg: /var/lib/alhp/makepkg/ upstream: /var/lib/alhp/upstream/ + build: /var/lib/alhp/build/ march: - x86-64-v3 diff --git a/git_clean.sh b/git_clean.sh deleted file mode 100644 index 84226a4..0000000 --- a/git_clean.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -cd "$1" && git clean -xdff diff --git a/main.go b/main.go index 3b6f569..56410ae 100644 --- a/main.go +++ b/main.go @@ -66,12 +66,23 @@ func (b *BuildManager) buildWorker(id int, march string) { log.Warningf("[%s/%s/%s] Failed to import pgp keys: %v", pkg.FullRepo, pkg.Pkgbase, pkg.Version, err) } + buildDir, err := pkg.setupBuildDir() + if err != nil { + log.Errorf("[%s/%s/%s] Error setting up builddir: %v", pkg.FullRepo, pkg.Pkgbase, pkg.Version, err) + b.buildWG.Done() + continue + } + buildNo := 1 versionSlice := strings.Split(pkg.DbPackage.LastVersionBuild, ".") if strings.Join(versionSlice[:len(versionSlice)-1], ".") == pkg.Version { buildNo, err = strconv.Atoi(versionSlice[len(versionSlice)-1]) if err != nil { log.Errorf("[%s/%s/%s] Failed to read build from pkgrel: %v", pkg.FullRepo, pkg.Pkgbase, pkg.Version, err) + err = cleanBuildDir(buildDir) + if err != nil { + log.Warningf("[%s/%s/%s] Error removing builddir: %v", pkg.FullRepo, pkg.Pkgbase, pkg.Version, err) + } b.buildWG.Done() continue } @@ -81,6 +92,10 @@ func (b *BuildManager) buildWorker(id int, march string) { err = pkg.increasePkgRel(buildNo) if err != nil { log.Errorf("[%s/%s/%s] Failed to increase pkgrel: %v", pkg.FullRepo, pkg.Pkgbase, pkg.Version, err) + err = cleanBuildDir(buildDir) + if err != nil { + log.Warningf("[%s/%s/%s] Error removing builddir: %v", pkg.FullRepo, pkg.Pkgbase, pkg.Version, err) + } b.buildWG.Done() continue } @@ -90,6 +105,10 @@ func (b *BuildManager) buildWorker(id int, march string) { if err != nil { log.Warningf("[%s/%s/%s] Failed to modify PKGBUILD for kernel patch: %v", pkg.FullRepo, pkg.Pkgbase, pkg.Version, err) pkg.DbPackage.Update().SetStatus(dbpackage.StatusFailed).SetSkipReason("failed to apply patch").SetHash(pkg.Hash).ExecX(context.Background()) + err = cleanBuildDir(buildDir) + if err != nil { + log.Warningf("[%s/%s/%s] Error removing builddir: %v", pkg.FullRepo, pkg.Pkgbase, pkg.Version, err) + } b.buildWG.Done() continue } @@ -129,6 +148,10 @@ func (b *BuildManager) buildWorker(id int, march string) { if err != nil { if b.exit { + err = cleanBuildDir(buildDir) + if err != nil { + log.Warningf("[%s/%s/%s] Error removing builddir: %v", pkg.FullRepo, pkg.Pkgbase, pkg.Version, err) + } b.buildWG.Done() continue } @@ -136,7 +159,10 @@ func (b *BuildManager) buildWorker(id int, march string) { if pkg.DbPackage.Lto != dbpackage.LtoAutoDisabled && pkg.DbPackage.Lto != dbpackage.LtoDisabled && reLdError.Match(out.Bytes()) { log.Infof("[%s/%s/%s] ld error detected, disabling LTO", pkg.FullRepo, pkg.Pkgbase, pkg.Version) pkg.DbPackage.Update().SetStatus(dbpackage.StatusQueued).SetSkipReason("non-LTO rebuild").SetLto(dbpackage.LtoAutoDisabled).ExecX(context.Background()) - gitClean(pkg) + err = cleanBuildDir(buildDir) + if err != nil { + log.Warningf("[%s/%s/%s] Error removing builddir: %v", pkg.FullRepo, pkg.Pkgbase, pkg.Version, err) + } b.buildWG.Done() continue } @@ -151,7 +177,10 @@ func (b *BuildManager) buildWorker(id int, march string) { // purge failed package from repo b.repoPurge[pkg.FullRepo] <- pkg - gitClean(pkg) + err = cleanBuildDir(buildDir) + if err != nil { + log.Warningf("[%s/%s/%s] Error removing builddir: %v", pkg.FullRepo, pkg.Pkgbase, pkg.Version, err) + } b.buildWG.Done() continue } @@ -162,7 +191,10 @@ func (b *BuildManager) buildWorker(id int, march string) { if len(pkgFiles) == 0 { log.Warningf("No packages found after building %s. Abort build.", pkg.Pkgbase) - gitClean(pkg) + err = cleanBuildDir(buildDir) + if err != nil { + log.Warningf("[%s/%s/%s] Error removing builddir: %v", pkg.FullRepo, pkg.Pkgbase, pkg.Version, err) + } b.buildWG.Done() continue } @@ -207,7 +239,10 @@ func (b *BuildManager) buildWorker(id int, march string) { log.Infof("[%s/%s/%s] Build successful (%s)", pkg.FullRepo, pkg.Pkgbase, pkg.Version, time.Since(start)) b.repoAdd[pkg.FullRepo] <- pkg - gitClean(pkg) + err = cleanBuildDir(buildDir) + if err != nil { + log.Warningf("[%s/%s/%s] Error removing builddir: %v", pkg.FullRepo, pkg.Pkgbase, pkg.Version, err) + } } } } @@ -568,16 +603,9 @@ func (b *BuildManager) syncWorker() { log.Debug(string(res)) check(err) } else if err == nil { - cmd := exec.Command("sudo", "git_clean.sh", gitPath) + cmd := exec.Command("sh", "-c", "cd "+gitPath+" && git reset --hard") res, err := cmd.CombinedOutput() log.Debug(string(res)) - if err != nil { - log.Warningf("Failed to execute %s: %v", cmd.String(), err) - } - - cmd = exec.Command("sh", "-c", "cd "+gitPath+" && git reset --hard") - res, err = cmd.CombinedOutput() - log.Debug(string(res)) check(err) cmd = exec.Command("sh", "-c", "cd "+gitPath+" && git pull") @@ -636,34 +664,34 @@ func (b *BuildManager) syncWorker() { continue } - // compare b3sum of PKGBUILD file to hash in database, only proceed if hash differs - // reduces the amount of PKGBUILDs that need to be parsed with makepkg, which is _really_ slow, significantly - dbPkg, dbErr := db.DbPackage.Query().Where(dbpackage.And( - dbpackage.Pkgbase(sPkgbuild[len(sPkgbuild)-4]), - dbpackage.RepositoryEQ(dbpackage.Repository(strings.Split(repo, "-")[0]))), - ).Only(context.Background()) - - if dbErr != nil { - switch dbErr.(type) { - case *ent.NotFoundError: - log.Debugf("[%s/%s] Package not found in database", strings.Split(repo, "-")[0], sPkgbuild[len(sPkgbuild)-4]) - break - default: - log.Errorf("[%s/%s] Problem querying db for package: %v", strings.Split(repo, "-")[0], sPkgbuild[len(sPkgbuild)-4], dbErr) - } - } - - b3s, err := b3sum(pkgbuild) - check(err) - - if dbPkg != nil && b3s == dbPkg.Hash { - log.Debugf("[%s/%s] Skipped: PKGBUILD hash matches db (%s)", strings.Split(repo, "-")[0], sPkgbuild[len(sPkgbuild)-4], b3s) - continue - } - - queued++ - // send to parse for _, march := range conf.March { + // compare b3sum of PKGBUILD file to hash in database, only proceed if hash differs + // reduces the amount of PKGBUILDs that need to be parsed with makepkg, which is _really_ slow, significantly + dbPkg, dbErr := db.DbPackage.Query().Where(dbpackage.And( + dbpackage.Pkgbase(sPkgbuild[len(sPkgbuild)-4]), + dbpackage.RepositoryEQ(dbpackage.Repository(strings.Split(repo, "-")[0])), dbpackage.March(march)), + ).Only(context.Background()) + + if dbErr != nil { + switch dbErr.(type) { + case *ent.NotFoundError: + log.Debugf("[%s/%s] Package not found in database", strings.Split(repo, "-")[0], sPkgbuild[len(sPkgbuild)-4]) + break + default: + log.Errorf("[%s/%s] Problem querying db for package: %v", strings.Split(repo, "-")[0], sPkgbuild[len(sPkgbuild)-4], dbErr) + } + } + + b3s, err := b3sum(pkgbuild) + check(err) + + if dbPkg != nil && b3s == dbPkg.Hash { + log.Debugf("[%s/%s] Skipped: PKGBUILD hash matches db (%s)", strings.Split(repo, "-")[0], sPkgbuild[len(sPkgbuild)-4], b3s) + continue + } + + queued++ + // send to parse b.parseWG.Add(1) b.parse <- &BuildPackage{ Pkgbuild: pkgbuild, diff --git a/utils.go b/utils.go index d7cc86a..90dd26b 100644 --- a/utils.go +++ b/utils.go @@ -74,7 +74,7 @@ type Conf struct { Repos, March []string Svn2git map[string]string Basedir struct { - Repo, Chroot, Makepkg, Upstream string + Repo, Chroot, Makepkg, Upstream, Build string } Db struct { Driver string @@ -170,6 +170,46 @@ func containsSubStr(str string, subList []string) bool { return false } +func cleanBuildDir(dir string) error { + if _, err := os.Stat(dir); err == nil { + err = os.RemoveAll(dir) + if err != nil { + return err + } + } + + return nil +} + +func (p *BuildPackage) setupBuildDir() (string, error) { + buildDir := filepath.Join(conf.Basedir.Build, p.March, p.Pkgbase+"-"+p.Version) + + err := cleanBuildDir(buildDir) + if err != nil { + return "", fmt.Errorf("removing old builddir failed: %v", err) + } + + err = os.MkdirAll(buildDir, 0755) + if err != nil { + return "", err + } + + files, err := filepath.Glob(filepath.Join(filepath.Dir(p.Pkgbuild), "*")) + if err != nil { + return "", err + } + + for _, file := range files { + _, err = copyFile(file, filepath.Join(buildDir, filepath.Base(file))) + if err != nil { + return "", err + } + } + + p.Pkgbuild = filepath.Join(buildDir, "PKGBUILD") + return buildDir, nil +} + func (p *BuildPackage) repoVersion() (string, error) { err := p.findPkgFiles() if err != nil { @@ -184,16 +224,6 @@ func (p *BuildPackage) repoVersion() (string, error) { return fNameSplit[len(fNameSplit)-3] + "-" + fNameSplit[len(fNameSplit)-2], nil } -func gitClean(pkg *BuildPackage) { - cmd := exec.Command("sudo", "git_clean.sh", filepath.Dir(pkg.Pkgbuild)) - res, err := cmd.CombinedOutput() - if err != nil { - log.Warningf("git clean failed with %v:\n%s", err, res) - } else { - log.Debug(string(res)) - } -} - func (p *BuildPackage) increasePkgRel(buildNo int) error { f, err := os.OpenFile(p.Pkgbuild, os.O_RDWR, 0644) if err != nil {