diff --git a/main.go b/main.go index 06c2dd4..3014850 100644 --- a/main.go +++ b/main.go @@ -24,6 +24,7 @@ import ( "runtime" "strconv" "strings" + "sync" "syscall" "time" ) @@ -228,12 +229,12 @@ func (b *BuildManager) parseWorker() { } if skipping { - dbPkg.Update().SetUpdated(time.Now()).SetVersion(pkg.Version).SetStatus(dbPkg.Status).SetSkipReason(dbPkg.SkipReason).SetHash(pkg.Hash).ExecX(context.Background()) + dbPkg.Update().SetUpdated(time.Now()).SetVersion(pkg.Version).SetPackages(packages2slice(pkg.Srcinfo.Packages)).SetStatus(dbPkg.Status).SetSkipReason(dbPkg.SkipReason).SetHash(pkg.Hash).ExecX(context.Background()) b.repoPurge[pkg.FullRepo] <- pkg b.parseWG.Done() continue } else { - dbPkg = dbPkg.Update().SetUpdated(time.Now()).SetVersion(pkg.Version).SaveX(context.Background()) + dbPkg = dbPkg.Update().SetUpdated(time.Now()).SetPackages(packages2slice(pkg.Srcinfo.Packages)).SetVersion(pkg.Version).SaveX(context.Background()) } repoVer, err := pkg.repoVersion() @@ -510,15 +511,23 @@ func (b *BuildManager) syncWorker() { } // housekeeping - err := housekeeping() - if err != nil { - log.Warningf("Housekeeping failed: %v", err) + wg := new(sync.WaitGroup) + for _, repo := range repos { + wg.Add(1) + repo := repo + go func() { + err := housekeeping(repo, wg) + if err != nil { + log.Warningf("[%s] Housekeeping failed: %v", repo, err) + } + }() } + wg.Wait() // fetch updates between sync runs b.alpmMutex.Lock() check(alpmHandle.Release()) - err = setupChroot() + err := setupChroot() for err != nil { log.Warningf("Unable to upgrade chroot, trying again later.") time.Sleep(time.Minute) diff --git a/utils.go b/utils.go index 9c20030..f1a60a6 100644 --- a/utils.go +++ b/utils.go @@ -474,88 +474,87 @@ func (path PKGFile) isSignatureValid() (bool, error) { return false, nil } -func housekeeping() error { - log.Debugf("Start housekeeping") - for _, repo := range repos { - packages, err := Glob(filepath.Join(conf.Basedir.Repo, repo, "/**/*.pkg.tar.zst")) +func housekeeping(repo string, wg *sync.WaitGroup) error { + defer wg.Done() + log.Debugf("[%s] Start housekeeping", repo) + packages, err := Glob(filepath.Join(conf.Basedir.Repo, repo, "/**/*.pkg.tar.zst")) + if err != nil { + return err + } + + for _, path := range packages { + pkgfile := PKGFile(path) + dbPkg, err := pkgfile.DBPackage() if err != nil { + log.Warningf("[HK] Unable to find entry for %s in db: %v", filepath.Base(path), err) + // TODO: remove orphan file not tracked by db (WTF kmod-debug!) + continue + } + + pkg := &BuildPackage{ + Pkgbase: dbPkg.Pkgbase, + Repo: dbPkg.Repository, + FullRepo: dbPkg.Repository.String() + "-" + dbPkg.March, + } + + var upstream string + switch dbPkg.Repository { + case dbpackage.RepositoryCore, dbpackage.RepositoryExtra: + upstream = "upstream-core-extra" + case dbpackage.RepositoryCommunity: + upstream = "upstream-community" + } + + pkg.Pkgbuild = filepath.Join(conf.Basedir.Upstream, upstream, dbPkg.Pkgbase, "repos", dbPkg.Repository.String()+"-"+conf.Arch, "PKGBUILD") + if err = pkg.genSrcinfo(); err != nil { return err } - for _, path := range packages { - pkgfile := PKGFile(path) - dbPkg, err := pkgfile.DBPackage() - if err != nil { - log.Warningf("[HK] Unable to find entry for %s in db: %v", filepath.Base(path), err) - // TODO: remove orphan file not tracked by db (WTF kmod-debug!) - continue - } + // check if pkg signature is valid + valid, err := pkgfile.isSignatureValid() + if err != nil { + return err + } + if !valid { + log.Infof("[HK/%s/%s] invalid package signature", pkg.FullRepo, pkg.Pkgbase) + buildManager.repoPurge[pkg.FullRepo] <- pkg + continue + } - pkg := &BuildPackage{ - Pkgbase: dbPkg.Pkgbase, - Repo: dbPkg.Repository, - FullRepo: dbPkg.Repository.String() + "-" + dbPkg.March, - } - - var upstream string - switch dbPkg.Repository { - case dbpackage.RepositoryCore, dbpackage.RepositoryExtra: - upstream = "upstream-core-extra" - case dbpackage.RepositoryCommunity: - upstream = "upstream-community" - } - - pkg.Pkgbuild = filepath.Join(conf.Basedir.Upstream, upstream, dbPkg.Pkgbase, "repos", dbPkg.Repository.String()+"-"+conf.Arch, "PKGBUILD") - if err = pkg.genSrcinfo(); err != nil { - return err - } - - // check if pkg signature is valid - valid, err := pkgfile.isSignatureValid() + // compare db-version with repo version + repoVer, err := pkg.repoVersion() + if err != nil { + log.Infof("[HK/%s/%s] package not present on disk", pkg.FullRepo, pkg.Pkgbase) + // error means package was not found -> delete version & hash from db so rebuild can happen + err := dbPkg.Update().ClearHash().ClearRepoVersion().Exec(context.Background()) if err != nil { return err } - if !valid { - log.Infof("[HK/%s/%s] invalid package signature", pkg.FullRepo, pkg.Pkgbase) - buildManager.repoPurge[pkg.FullRepo] <- pkg - continue - } - - // compare db-version with repo version - repoVer, err := pkg.repoVersion() - if err != nil { - log.Infof("[HK/%s/%s] package not present on disk", pkg.FullRepo, pkg.Pkgbase) - // error means package was not found -> delete version & hash from db so rebuild can happen - err := dbPkg.Update().ClearHash().ClearRepoVersion().Exec(context.Background()) - if err != nil { - return err - } - } else if alpm.VerCmp(repoVer, dbPkg.RepoVersion) != 0 { - log.Infof("[HK/%s/%s] update %s->%s in db", pkg.FullRepo, pkg.Pkgbase, dbPkg.RepoVersion, repoVer) - dbPkg, err = dbPkg.Update().SetRepoVersion(repoVer).Save(context.Background()) - if err != nil { - return err - } - } - - // TODO: check split packages - - // check if package is still part of repo - dbs, err := alpmHandle.SyncDBs() + } else if alpm.VerCmp(repoVer, dbPkg.RepoVersion) != 0 { + log.Infof("[HK/%s/%s] update %s->%s in db", pkg.FullRepo, pkg.Pkgbase, dbPkg.RepoVersion, repoVer) + dbPkg, err = dbPkg.Update().SetRepoVersion(repoVer).Save(context.Background()) if err != nil { return err } - pkgResolved, err := dbs.FindSatisfier(pkg.Srcinfo.Packages[0].Pkgname) - if err != nil || pkgResolved.DB().Name() != dbPkg.Repository.String() { - // package not found on mirror/db -> not part of any repo anymore - log.Infof("[HK/%s/%s] not part of repo", pkg.FullRepo, pkg.Pkgbase) - buildManager.repoPurge[pkg.FullRepo] <- pkg - err = db.DbPackage.DeleteOne(dbPkg).Exec(context.Background()) - if err != nil { - return err - } - continue + } + + // TODO: check split packages + + // check if package is still part of repo + dbs, err := alpmHandle.SyncDBs() + if err != nil { + return err + } + pkgResolved, err := dbs.FindSatisfier(pkg.Srcinfo.Packages[0].Pkgname) + if err != nil || pkgResolved.DB().Name() != dbPkg.Repository.String() { + // package not found on mirror/db -> not part of any repo anymore + log.Infof("[HK/%s/%s] not part of repo", pkg.FullRepo, pkg.Pkgbase) + buildManager.repoPurge[pkg.FullRepo] <- pkg + err = db.DbPackage.DeleteOne(dbPkg).Exec(context.Background()) + if err != nil { + return err } + continue } }