forked from ALHP/ALHP.GO
run housekeeping concurrent per repo
update splitpackages on parse
This commit is contained in:
21
main.go
21
main.go
@@ -24,6 +24,7 @@ import (
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
@@ -228,12 +229,12 @@ func (b *BuildManager) parseWorker() {
|
||||
}
|
||||
|
||||
if skipping {
|
||||
dbPkg.Update().SetUpdated(time.Now()).SetVersion(pkg.Version).SetStatus(dbPkg.Status).SetSkipReason(dbPkg.SkipReason).SetHash(pkg.Hash).ExecX(context.Background())
|
||||
dbPkg.Update().SetUpdated(time.Now()).SetVersion(pkg.Version).SetPackages(packages2slice(pkg.Srcinfo.Packages)).SetStatus(dbPkg.Status).SetSkipReason(dbPkg.SkipReason).SetHash(pkg.Hash).ExecX(context.Background())
|
||||
b.repoPurge[pkg.FullRepo] <- pkg
|
||||
b.parseWG.Done()
|
||||
continue
|
||||
} else {
|
||||
dbPkg = dbPkg.Update().SetUpdated(time.Now()).SetVersion(pkg.Version).SaveX(context.Background())
|
||||
dbPkg = dbPkg.Update().SetUpdated(time.Now()).SetPackages(packages2slice(pkg.Srcinfo.Packages)).SetVersion(pkg.Version).SaveX(context.Background())
|
||||
}
|
||||
|
||||
repoVer, err := pkg.repoVersion()
|
||||
@@ -510,15 +511,23 @@ func (b *BuildManager) syncWorker() {
|
||||
}
|
||||
|
||||
// housekeeping
|
||||
err := housekeeping()
|
||||
if err != nil {
|
||||
log.Warningf("Housekeeping failed: %v", err)
|
||||
wg := new(sync.WaitGroup)
|
||||
for _, repo := range repos {
|
||||
wg.Add(1)
|
||||
repo := repo
|
||||
go func() {
|
||||
err := housekeeping(repo, wg)
|
||||
if err != nil {
|
||||
log.Warningf("[%s] Housekeeping failed: %v", repo, err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
// fetch updates between sync runs
|
||||
b.alpmMutex.Lock()
|
||||
check(alpmHandle.Release())
|
||||
err = setupChroot()
|
||||
err := setupChroot()
|
||||
for err != nil {
|
||||
log.Warningf("Unable to upgrade chroot, trying again later.")
|
||||
time.Sleep(time.Minute)
|
||||
|
||||
139
utils.go
139
utils.go
@@ -474,88 +474,87 @@ func (path PKGFile) isSignatureValid() (bool, error) {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func housekeeping() error {
|
||||
log.Debugf("Start housekeeping")
|
||||
for _, repo := range repos {
|
||||
packages, err := Glob(filepath.Join(conf.Basedir.Repo, repo, "/**/*.pkg.tar.zst"))
|
||||
func housekeeping(repo string, wg *sync.WaitGroup) error {
|
||||
defer wg.Done()
|
||||
log.Debugf("[%s] Start housekeeping", repo)
|
||||
packages, err := Glob(filepath.Join(conf.Basedir.Repo, repo, "/**/*.pkg.tar.zst"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, path := range packages {
|
||||
pkgfile := PKGFile(path)
|
||||
dbPkg, err := pkgfile.DBPackage()
|
||||
if err != nil {
|
||||
log.Warningf("[HK] Unable to find entry for %s in db: %v", filepath.Base(path), err)
|
||||
// TODO: remove orphan file not tracked by db (WTF kmod-debug!)
|
||||
continue
|
||||
}
|
||||
|
||||
pkg := &BuildPackage{
|
||||
Pkgbase: dbPkg.Pkgbase,
|
||||
Repo: dbPkg.Repository,
|
||||
FullRepo: dbPkg.Repository.String() + "-" + dbPkg.March,
|
||||
}
|
||||
|
||||
var upstream string
|
||||
switch dbPkg.Repository {
|
||||
case dbpackage.RepositoryCore, dbpackage.RepositoryExtra:
|
||||
upstream = "upstream-core-extra"
|
||||
case dbpackage.RepositoryCommunity:
|
||||
upstream = "upstream-community"
|
||||
}
|
||||
|
||||
pkg.Pkgbuild = filepath.Join(conf.Basedir.Upstream, upstream, dbPkg.Pkgbase, "repos", dbPkg.Repository.String()+"-"+conf.Arch, "PKGBUILD")
|
||||
if err = pkg.genSrcinfo(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, path := range packages {
|
||||
pkgfile := PKGFile(path)
|
||||
dbPkg, err := pkgfile.DBPackage()
|
||||
if err != nil {
|
||||
log.Warningf("[HK] Unable to find entry for %s in db: %v", filepath.Base(path), err)
|
||||
// TODO: remove orphan file not tracked by db (WTF kmod-debug!)
|
||||
continue
|
||||
}
|
||||
// check if pkg signature is valid
|
||||
valid, err := pkgfile.isSignatureValid()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !valid {
|
||||
log.Infof("[HK/%s/%s] invalid package signature", pkg.FullRepo, pkg.Pkgbase)
|
||||
buildManager.repoPurge[pkg.FullRepo] <- pkg
|
||||
continue
|
||||
}
|
||||
|
||||
pkg := &BuildPackage{
|
||||
Pkgbase: dbPkg.Pkgbase,
|
||||
Repo: dbPkg.Repository,
|
||||
FullRepo: dbPkg.Repository.String() + "-" + dbPkg.March,
|
||||
}
|
||||
|
||||
var upstream string
|
||||
switch dbPkg.Repository {
|
||||
case dbpackage.RepositoryCore, dbpackage.RepositoryExtra:
|
||||
upstream = "upstream-core-extra"
|
||||
case dbpackage.RepositoryCommunity:
|
||||
upstream = "upstream-community"
|
||||
}
|
||||
|
||||
pkg.Pkgbuild = filepath.Join(conf.Basedir.Upstream, upstream, dbPkg.Pkgbase, "repos", dbPkg.Repository.String()+"-"+conf.Arch, "PKGBUILD")
|
||||
if err = pkg.genSrcinfo(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// check if pkg signature is valid
|
||||
valid, err := pkgfile.isSignatureValid()
|
||||
// compare db-version with repo version
|
||||
repoVer, err := pkg.repoVersion()
|
||||
if err != nil {
|
||||
log.Infof("[HK/%s/%s] package not present on disk", pkg.FullRepo, pkg.Pkgbase)
|
||||
// error means package was not found -> delete version & hash from db so rebuild can happen
|
||||
err := dbPkg.Update().ClearHash().ClearRepoVersion().Exec(context.Background())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !valid {
|
||||
log.Infof("[HK/%s/%s] invalid package signature", pkg.FullRepo, pkg.Pkgbase)
|
||||
buildManager.repoPurge[pkg.FullRepo] <- pkg
|
||||
continue
|
||||
}
|
||||
|
||||
// compare db-version with repo version
|
||||
repoVer, err := pkg.repoVersion()
|
||||
if err != nil {
|
||||
log.Infof("[HK/%s/%s] package not present on disk", pkg.FullRepo, pkg.Pkgbase)
|
||||
// error means package was not found -> delete version & hash from db so rebuild can happen
|
||||
err := dbPkg.Update().ClearHash().ClearRepoVersion().Exec(context.Background())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else if alpm.VerCmp(repoVer, dbPkg.RepoVersion) != 0 {
|
||||
log.Infof("[HK/%s/%s] update %s->%s in db", pkg.FullRepo, pkg.Pkgbase, dbPkg.RepoVersion, repoVer)
|
||||
dbPkg, err = dbPkg.Update().SetRepoVersion(repoVer).Save(context.Background())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: check split packages
|
||||
|
||||
// check if package is still part of repo
|
||||
dbs, err := alpmHandle.SyncDBs()
|
||||
} else if alpm.VerCmp(repoVer, dbPkg.RepoVersion) != 0 {
|
||||
log.Infof("[HK/%s/%s] update %s->%s in db", pkg.FullRepo, pkg.Pkgbase, dbPkg.RepoVersion, repoVer)
|
||||
dbPkg, err = dbPkg.Update().SetRepoVersion(repoVer).Save(context.Background())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pkgResolved, err := dbs.FindSatisfier(pkg.Srcinfo.Packages[0].Pkgname)
|
||||
if err != nil || pkgResolved.DB().Name() != dbPkg.Repository.String() {
|
||||
// package not found on mirror/db -> not part of any repo anymore
|
||||
log.Infof("[HK/%s/%s] not part of repo", pkg.FullRepo, pkg.Pkgbase)
|
||||
buildManager.repoPurge[pkg.FullRepo] <- pkg
|
||||
err = db.DbPackage.DeleteOne(dbPkg).Exec(context.Background())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
// TODO: check split packages
|
||||
|
||||
// check if package is still part of repo
|
||||
dbs, err := alpmHandle.SyncDBs()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pkgResolved, err := dbs.FindSatisfier(pkg.Srcinfo.Packages[0].Pkgname)
|
||||
if err != nil || pkgResolved.DB().Name() != dbPkg.Repository.String() {
|
||||
// package not found on mirror/db -> not part of any repo anymore
|
||||
log.Infof("[HK/%s/%s] not part of repo", pkg.FullRepo, pkg.Pkgbase)
|
||||
buildManager.repoPurge[pkg.FullRepo] <- pkg
|
||||
err = db.DbPackage.DeleteOne(dbPkg).Exec(context.Background())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user