From 7baccce64baacc4dde24cee5aaaf54e02750e9fc Mon Sep 17 00:00:00 2001 From: Giovanni Harting <539@idlegandalf.com> Date: Sat, 20 Nov 2021 00:02:53 +0100 Subject: [PATCH] sped up housekeeping so no interval is needed --- config_dist.yaml | 3 -- main.go | 91 ++++++++++++++++++++++-------------------------- utils.go | 54 ++++++++++++++-------------- 3 files changed, 69 insertions(+), 79 deletions(-) diff --git a/config_dist.yaml b/config_dist.yaml index 06f1ce2..e50b40d 100644 --- a/config_dist.yaml +++ b/config_dist.yaml @@ -21,9 +21,6 @@ basedir: march: - x86-64-v3 -housekeeping: - interval: 12h - blacklist: packages: - tensorflow diff --git a/main.go b/main.go index 66480f1..0d2b9b8 100644 --- a/main.go +++ b/main.go @@ -40,7 +40,6 @@ var ( db *ent.Client journalLog = flag.Bool("journal", false, "Log to systemd journal instead of stdout") checkInterval = flag.Int("interval", 5, "How often svn2git should be checked in minutes (default: 5)") - lastHKRun time.Time ) func (b *BuildManager) buildWorker(id int) { @@ -63,8 +62,8 @@ func (b *BuildManager) buildWorker(id int) { log.Infof("[%s/%s] Build starting", pkg.FullRepo, pkg.Pkgbase) - dbPkg := pkg.toDbPackage(true) - dbPkg = dbPkg.Update().SetStatus(dbpackage.StatusBuilding).ClearSkipReason().SaveX(context.Background()) + pkg.toDbPackage(true) + pkg.DbPackage = pkg.DbPackage.Update().SetStatus(dbpackage.StatusBuilding).ClearSkipReason().SaveX(context.Background()) err := importKeys(pkg) if err != nil { @@ -86,7 +85,7 @@ func (b *BuildManager) buildWorker(id int) { // use non-lto makepkg.conf if LTO is blacklisted for this package makepkgFile = "makepkg-%s.conf" ltoDisabled = true - dbPkg.Update().SetLto(dbpackage.LtoDisabled).ExecX(context.Background()) + pkg.DbPackage.Update().SetLto(dbpackage.LtoDisabled).ExecX(context.Background()) } cmd := exec.Command("sh", "-c", "cd "+filepath.Dir(pkg.Pkgbuild)+"&&makechrootpkg -c -D "+conf.Basedir.Makepkg+" -l worker-"+strconv.Itoa(id)+" -r "+conf.Basedir.Chroot+" -- "+ @@ -123,7 +122,7 @@ func (b *BuildManager) buildWorker(id int) { check(os.MkdirAll(filepath.Join(conf.Basedir.Repo, "logs"), 0755)) check(os.WriteFile(filepath.Join(conf.Basedir.Repo, "logs", pkg.Pkgbase+".log"), out.Bytes(), 0644)) - dbPkg.Update().SetStatus(dbpackage.StatusFailed).ClearSkipReason().SetBuildTimeStart(start).SetBuildTimeEnd(time.Now().UTC()).SetHash(pkg.Hash).ExecX(context.Background()) + pkg.DbPackage.Update().SetStatus(dbpackage.StatusFailed).ClearSkipReason().SetBuildTimeStart(start).SetBuildTimeEnd(time.Now().UTC()).SetHash(pkg.Hash).ExecX(context.Background()) // purge failed package from repo b.repoPurge[pkg.FullRepo] <- pkg @@ -176,9 +175,9 @@ func (b *BuildManager) buildWorker(id int) { } if !ltoDisabled { - dbPkg.Update().SetStatus(dbpackage.StatusBuild).SetLto(dbpackage.LtoEnabled).SetBuildTimeStart(start).SetBuildTimeEnd(time.Now().UTC()).ExecX(context.Background()) + pkg.DbPackage.Update().SetStatus(dbpackage.StatusBuild).SetLto(dbpackage.LtoEnabled).SetBuildTimeStart(start).SetBuildTimeEnd(time.Now().UTC()).ExecX(context.Background()) } else { - dbPkg.Update().SetStatus(dbpackage.StatusBuild).SetLto(dbpackage.LtoDisabled).SetBuildTimeStart(start).SetBuildTimeEnd(time.Now().UTC()).ExecX(context.Background()) + pkg.DbPackage.Update().SetStatus(dbpackage.StatusBuild).SetLto(dbpackage.LtoDisabled).SetBuildTimeStart(start).SetBuildTimeEnd(time.Now().UTC()).ExecX(context.Background()) } log.Infof("[%s/%s] Build successful (%s)", pkg.FullRepo, pkg.Pkgbase, time.Now().Sub(start)) @@ -203,47 +202,49 @@ func (b *BuildManager) parseWorker() { } pkg.Version = constructVersion(pkg.Srcinfo.Pkgver, pkg.Srcinfo.Pkgrel, pkg.Srcinfo.Epoch) - dbPkg := pkg.toDbPackage(true) + pkg.toDbPackage(true) skipping := false if contains(pkg.Srcinfo.Arch, "any") { log.Debugf("Skipped %s: any-Package", pkg.Srcinfo.Pkgbase) - dbPkg.SkipReason = "arch = any" - dbPkg.Status = dbpackage.StatusSkipped + pkg.DbPackage.SkipReason = "arch = any" + pkg.DbPackage.Status = dbpackage.StatusSkipped skipping = true } else if contains(conf.Blacklist.Packages, pkg.Srcinfo.Pkgbase) { log.Debugf("Skipped %s: blacklisted package", pkg.Srcinfo.Pkgbase) - dbPkg.SkipReason = "blacklisted" - dbPkg.Status = dbpackage.StatusSkipped + pkg.DbPackage.SkipReason = "blacklisted" + pkg.DbPackage.Status = dbpackage.StatusSkipped skipping = true } else if contains(pkg.Srcinfo.MakeDepends, "ghc") || contains(pkg.Srcinfo.MakeDepends, "haskell-ghc") || contains(pkg.Srcinfo.Depends, "ghc") || contains(pkg.Srcinfo.Depends, "haskell-ghc") { // Skip Haskell packages for now, as we are facing linking problems with them, // most likely caused by not having a dependency check implemented yet and building at random. // https://git.harting.dev/anonfunc/ALHP.GO/issues/11 log.Debugf("Skipped %s: haskell package", pkg.Srcinfo.Pkgbase) - dbPkg.SkipReason = "blacklisted (haskell)" - dbPkg.Status = dbpackage.StatusSkipped + pkg.DbPackage.SkipReason = "blacklisted (haskell)" + pkg.DbPackage.Status = dbpackage.StatusSkipped skipping = true - } else if isPkgFailed(pkg, dbPkg) { + } else if isPkgFailed(pkg) { log.Debugf("Skipped %s: failed build", pkg.Srcinfo.Pkgbase) skipping = true } if skipping { - dbPkg.Update().SetUpdated(time.Now()).SetVersion(pkg.Version).SetPackages(packages2slice(pkg.Srcinfo.Packages)).SetStatus(dbPkg.Status).SetSkipReason(dbPkg.SkipReason).SetHash(pkg.Hash).ExecX(context.Background()) + pkg.DbPackage.Update().SetUpdated(time.Now()).SetVersion(pkg.Version). + SetPackages(packages2slice(pkg.Srcinfo.Packages)).SetStatus(pkg.DbPackage.Status). + SetSkipReason(pkg.DbPackage.SkipReason).SetHash(pkg.Hash).ExecX(context.Background()) b.repoPurge[pkg.FullRepo] <- pkg b.parseWG.Done() continue } else { - dbPkg = dbPkg.Update().SetUpdated(time.Now()).SetPackages(packages2slice(pkg.Srcinfo.Packages)).SetVersion(pkg.Version).SaveX(context.Background()) + pkg.DbPackage = pkg.DbPackage.Update().SetUpdated(time.Now()).SetPackages(packages2slice(pkg.Srcinfo.Packages)).SetVersion(pkg.Version).SaveX(context.Background()) } repoVer, err := pkg.repoVersion() if err != nil { - dbPkg = dbPkg.Update().ClearRepoVersion().SaveX(context.Background()) + pkg.DbPackage = pkg.DbPackage.Update().ClearRepoVersion().SaveX(context.Background()) } else if err == nil && alpm.VerCmp(repoVer, pkg.Version) > 0 { log.Debugf("Skipped %s: Version in repo higher than in PKGBUILD (%s < %s)", pkg.Srcinfo.Pkgbase, pkg.Version, repoVer) - dbPkg = dbPkg.Update().SetStatus(dbpackage.StatusLatest).ClearSkipReason().SetHash(pkg.Hash).SaveX(context.Background()) + pkg.DbPackage = pkg.DbPackage.Update().SetStatus(dbpackage.StatusLatest).ClearSkipReason().SetHash(pkg.Hash).SaveX(context.Background()) b.parseWG.Done() continue } @@ -255,28 +256,28 @@ func (b *BuildManager) parseWorker() { log.Warningf("[%s/%s] Problem solving dependencies: %v", pkg.FullRepo, pkg.Srcinfo.Pkgbase, err) case MultiplePKGBUILDError: log.Infof("Skipped %s: Multiple PKGBUILDs for dependency found: %v", pkg.Srcinfo.Pkgbase, err) - dbPkg = dbPkg.Update().SetStatus(dbpackage.StatusSkipped).SetSkipReason("multiple PKGBUILD for dep. found").SaveX(context.Background()) + pkg.DbPackage = pkg.DbPackage.Update().SetStatus(dbpackage.StatusSkipped).SetSkipReason("multiple PKGBUILD for dep. found").SaveX(context.Background()) b.repoPurge[pkg.FullRepo] <- pkg b.parseWG.Done() continue case UnableToSatisfyError: log.Infof("Skipped %s: unable to resolve dependencies: %v", pkg.Srcinfo.Pkgbase, err) - dbPkg = dbPkg.Update().SetStatus(dbpackage.StatusSkipped).SetSkipReason("unable to resolve dependencies").SaveX(context.Background()) + pkg.DbPackage = pkg.DbPackage.Update().SetStatus(dbpackage.StatusSkipped).SetSkipReason("unable to resolve dependencies").SaveX(context.Background()) b.repoPurge[pkg.FullRepo] <- pkg b.parseWG.Done() continue } } - dbPkg = dbPkg.Update().SetStatus(dbpackage.StatusQueued).SaveX(context.Background()) + pkg.DbPackage = pkg.DbPackage.Update().SetStatus(dbpackage.StatusQueued).SaveX(context.Background()) if !isLatest { if local != nil { log.Infof("Delayed %s: not all dependencies are up to date (local: %s==%s, sync: %s==%s)", pkg.Srcinfo.Pkgbase, local.Name(), local.Version(), local.Name(), syncVersion) - dbPkg.Update().SetSkipReason(fmt.Sprintf("waiting for %s==%s", local.Name(), syncVersion)).ExecX(context.Background()) + pkg.DbPackage.Update().SetSkipReason(fmt.Sprintf("waiting for %s==%s", local.Name(), syncVersion)).ExecX(context.Background()) } else { log.Infof("Delayed %s: not all dependencies are up to date or resolvable", pkg.Srcinfo.Pkgbase) - dbPkg.Update().SetSkipReason("waiting for mirror").ExecX(context.Background()) + pkg.DbPackage.Update().SetSkipReason("waiting for mirror").ExecX(context.Background()) } // Purge delayed packages in case delay is caused by inconsistencies in svn2git. @@ -413,8 +414,8 @@ func (b *BuildManager) repoWorker(repo string) { log.Panicf("%s while repo-add: %v", string(res), err) } - dbPkg := pkg.toDbPackage(true) - dbPkg = dbPkg.Update().SetStatus(dbpackage.StatusLatest).ClearSkipReason().SetRepoVersion(pkg.Version).SetHash(pkg.Hash).SaveX(context.Background()) + pkg.toDbPackage(true) + pkg.DbPackage = pkg.DbPackage.Update().SetStatus(dbpackage.StatusLatest).ClearSkipReason().SetRepoVersion(pkg.Version).SetHash(pkg.Hash).SaveX(context.Background()) cmd = exec.Command("paccache", "-rc", filepath.Join(conf.Basedir.Repo, pkg.FullRepo, "os", conf.Arch), @@ -456,9 +457,8 @@ func (b *BuildManager) repoWorker(repo string) { continue } - dbPkg := pkg.toDbPackage(false) - if dbPkg != nil { - dbPkg = dbPkg.Update().ClearRepoVersion().SaveX(context.Background()) + if pkg.DbPackage != nil { + pkg.DbPackage = pkg.DbPackage.Update().ClearRepoVersion().SaveX(context.Background()) } for _, file := range pkg.PkgFiles { @@ -514,32 +514,23 @@ func (b *BuildManager) syncWorker() { } // housekeeping - hkDur, err := time.ParseDuration(conf.Housekeeping.Interval) - if err != nil { - log.Warningf("Unable to parse housekeeping duration %s: %v", conf.Housekeeping.Interval, err) - hkDur, _ = time.ParseDuration("12h") - } - - if time.Since(lastHKRun) > hkDur { - lastHKRun = time.Now() - wg := new(sync.WaitGroup) - for _, repo := range repos { - wg.Add(1) - repo := repo - go func() { - err := housekeeping(repo, wg) - if err != nil { - log.Warningf("[%s] housekeeping failed: %v", repo, err) - } - }() - } - wg.Wait() + wg := new(sync.WaitGroup) + for _, repo := range repos { + wg.Add(1) + repo := repo + go func() { + err := housekeeping(repo, wg) + if err != nil { + log.Warningf("[%s] housekeeping failed: %v", repo, err) + } + }() } + wg.Wait() // fetch updates between sync runs b.alpmMutex.Lock() check(alpmHandle.Release()) - err = setupChroot() + err := setupChroot() for err != nil { log.Warningf("Unable to upgrade chroot, trying again later.") time.Sleep(time.Minute) diff --git a/utils.go b/utils.go index 716ebc2..5cc18ae 100644 --- a/utils.go +++ b/utils.go @@ -31,15 +31,16 @@ const ( ) type BuildPackage struct { - Pkgbase string - Pkgbuild string - Srcinfo *srcinfo.Srcinfo - PkgFiles []string - Repo dbpackage.Repository - March string - FullRepo string - Version string - Hash string + Pkgbase string + Pkgbuild string + Srcinfo *srcinfo.Srcinfo + PkgFiles []string + Repo dbpackage.Repository + March string + FullRepo string + Version string + Hash string + DbPackage *ent.DbPackage } type BuildManager struct { @@ -371,8 +372,8 @@ func (p *BuildPackage) SVN2GITVersion(h *alpm.Handle) (string, error) { return constructVersion(info.Pkgver, info.Pkgrel, info.Epoch), nil } -func isPkgFailed(pkg *BuildPackage, dbPkg *ent.DbPackage) bool { - if dbPkg.Version == "" { +func isPkgFailed(pkg *BuildPackage) bool { + if pkg.DbPackage.Version == "" { return false } @@ -384,10 +385,10 @@ func isPkgFailed(pkg *BuildPackage, dbPkg *ent.DbPackage) bool { pkg.Version = constructVersion(pkg.Srcinfo.Pkgver, pkg.Srcinfo.Pkgrel, pkg.Srcinfo.Epoch) } - if alpm.VerCmp(dbPkg.Version, pkg.Version) < 0 { + if alpm.VerCmp(pkg.DbPackage.Version, pkg.Version) < 0 { return false } else { - return dbPkg.Status == dbpackage.StatusFailed + return pkg.DbPackage.Status == dbpackage.StatusFailed } } @@ -495,19 +496,20 @@ func housekeeping(repo string, wg *sync.WaitGroup) error { } pkg := &BuildPackage{ - Pkgbase: dbPkg.Pkgbase, - Repo: dbPkg.Repository, - FullRepo: dbPkg.Repository.String() + "-" + dbPkg.March, + Pkgbase: dbPkg.Pkgbase, + Repo: dbPkg.Repository, + FullRepo: dbPkg.Repository.String() + "-" + dbPkg.March, + DbPackage: dbPkg, } var upstream string - switch dbPkg.Repository { + switch pkg.DbPackage.Repository { case dbpackage.RepositoryCore, dbpackage.RepositoryExtra: upstream = "upstream-core-extra" case dbpackage.RepositoryCommunity: upstream = "upstream-community" } - pkg.Pkgbuild = filepath.Join(conf.Basedir.Upstream, upstream, dbPkg.Pkgbase, "repos", dbPkg.Repository.String()+"-"+conf.Arch, "PKGBUILD") + pkg.Pkgbuild = filepath.Join(conf.Basedir.Upstream, upstream, dbPkg.Pkgbase, "repos", pkg.DbPackage.Repository.String()+"-"+conf.Arch, "PKGBUILD") // check if pkg signature is valid valid, err := pkgfile.isSignatureValid() @@ -525,13 +527,13 @@ func housekeeping(repo string, wg *sync.WaitGroup) error { if err != nil { log.Infof("[HK/%s/%s] package not present on disk", pkg.FullRepo, pkg.Pkgbase) // error means package was not found -> delete version & hash from db so rebuild can happen - err := dbPkg.Update().ClearHash().ClearRepoVersion().Exec(context.Background()) + err := pkg.DbPackage.Update().ClearHash().ClearRepoVersion().Exec(context.Background()) if err != nil { return err } } else if alpm.VerCmp(repoVer, dbPkg.RepoVersion) != 0 { log.Infof("[HK/%s/%s] update %s->%s in db", pkg.FullRepo, pkg.Pkgbase, dbPkg.RepoVersion, repoVer) - dbPkg, err = dbPkg.Update().SetRepoVersion(repoVer).Save(context.Background()) + pkg.DbPackage, err = pkg.DbPackage.Update().SetRepoVersion(repoVer).Save(context.Background()) if err != nil { return err } @@ -545,11 +547,11 @@ func housekeeping(repo string, wg *sync.WaitGroup) error { return err } pkgResolved, err := dbs.FindSatisfier(dbPkg.Packages[0]) - if err != nil || pkgResolved.DB().Name() != dbPkg.Repository.String() { + if err != nil || pkgResolved.DB().Name() != pkg.DbPackage.Repository.String() { // package not found on mirror/db -> not part of any repo anymore log.Infof("[HK/%s/%s] not part of repo", pkg.FullRepo, pkg.Pkgbase) buildManager.repoPurge[pkg.FullRepo] <- pkg - err = db.DbPackage.DeleteOne(dbPkg).Exec(context.Background()) + err = db.DbPackage.DeleteOne(pkg.DbPackage).Exec(context.Background()) if err != nil { return err } @@ -567,8 +569,8 @@ func (p *BuildPackage) findPkgFiles() error { } var realPkgs []string - for _, realPkg := range p.Srcinfo.Packages { - realPkgs = append(realPkgs, realPkg.Pkgname) + for _, realPkg := range p.DbPackage.Packages { + realPkgs = append(realPkgs, realPkg) } var fPkg []string @@ -586,13 +588,13 @@ func (p *BuildPackage) findPkgFiles() error { return nil } -func (p *BuildPackage) toDbPackage(create bool) *ent.DbPackage { +func (p *BuildPackage) toDbPackage(create bool) { dbPkg, err := db.DbPackage.Query().Where(dbpackage.Pkgbase(p.Pkgbase)).Only(context.Background()) if err != nil && create { dbPkg = db.DbPackage.Create().SetPkgbase(p.Pkgbase).SetMarch(p.March).SetPackages(packages2slice(p.Srcinfo.Packages)).SetRepository(p.Repo).SaveX(context.Background()) } - return dbPkg + p.DbPackage = dbPkg } func syncMarchs() {