SunRed/ALHP.GO
1
0
Fork 0
forked from ALHP/ALHP.GO
Code Issues Pull Requests Packages Projects Releases Wiki Activity

run housekeeping concurrent per repo

Browse Source 
update splitpackages on parse
This commit is contained in:
Giovanni Harting
2021-11-19 23:01:19 +01:00
parent f9c5292f90
commit 5d6a498503
2 changed files with 84 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
main.go
View File

@@ -24,6 +24,7 @@ import (
"runtime" "runtime"
"strconv" "strconv"
"strings" "strings"
"sync"
"syscall" "syscall"
"time" "time"
) )
@@ -228,12 +229,12 @@ func (b *BuildManager) parseWorker() {
} }
if skipping { if skipping {
dbPkg.Update().SetUpdated(time.Now()).SetVersion(pkg.Version).SetStatus(dbPkg.Status).SetSkipReason(dbPkg.SkipReason).SetHash(pkg.Hash).ExecX(context.Background()) dbPkg.Update().SetUpdated(time.Now()).SetVersion(pkg.Version).SetPackages(packages2slice(pkg.Srcinfo.Packages)).SetStatus(dbPkg.Status).SetSkipReason(dbPkg.SkipReason).SetHash(pkg.Hash).ExecX(context.Background())
b.repoPurge[pkg.FullRepo] <- pkg b.repoPurge[pkg.FullRepo] <- pkg
b.parseWG.Done() b.parseWG.Done()
continue continue
} else { } else {
dbPkg = dbPkg.Update().SetUpdated(time.Now()).SetVersion(pkg.Version).SaveX(context.Background()) dbPkg = dbPkg.Update().SetUpdated(time.Now()).SetPackages(packages2slice(pkg.Srcinfo.Packages)).SetVersion(pkg.Version).SaveX(context.Background())
} }
repoVer, err := pkg.repoVersion() repoVer, err := pkg.repoVersion()
@@ -510,15 +511,23 @@ func (b *BuildManager) syncWorker() {
} }
// housekeeping // housekeeping
err := housekeeping() wg := new(sync.WaitGroup)
if err != nil { for _, repo := range repos {
log.Warningf("Housekeeping failed: %v", err) wg.Add(1)
repo := repo
go func() {
err := housekeeping(repo, wg)
if err != nil {
log.Warningf("[%s] Housekeeping failed: %v", repo, err)
}
}()
} }
wg.Wait()
// fetch updates between sync runs // fetch updates between sync runs
b.alpmMutex.Lock() b.alpmMutex.Lock()
check(alpmHandle.Release()) check(alpmHandle.Release())
err = setupChroot() err := setupChroot()
for err != nil { for err != nil {
log.Warningf("Unable to upgrade chroot, trying again later.") log.Warningf("Unable to upgrade chroot, trying again later.")
time.Sleep(time.Minute) time.Sleep(time.Minute)

139
utils.go
View File

@@ -474,88 +474,87 @@ func (path PKGFile) isSignatureValid() (bool, error) {
return false, nil return false, nil
} }
func housekeeping() error { func housekeeping(repo string, wg *sync.WaitGroup) error {
log.Debugf("Start housekeeping") defer wg.Done()
for _, repo := range repos { log.Debugf("[%s] Start housekeeping", repo)
packages, err := Glob(filepath.Join(conf.Basedir.Repo, repo, "/**/*.pkg.tar.zst")) packages, err := Glob(filepath.Join(conf.Basedir.Repo, repo, "/**/*.pkg.tar.zst"))
if err != nil {
return err
}
for _, path := range packages {
pkgfile := PKGFile(path)
dbPkg, err := pkgfile.DBPackage()
if err != nil { if err != nil {
log.Warningf("[HK] Unable to find entry for %s in db: %v", filepath.Base(path), err)
// TODO: remove orphan file not tracked by db (WTF kmod-debug!)
continue
}
pkg := &BuildPackage{
Pkgbase: dbPkg.Pkgbase,
Repo: dbPkg.Repository,
FullRepo: dbPkg.Repository.String() + "-" + dbPkg.March,
}
var upstream string
switch dbPkg.Repository {
case dbpackage.RepositoryCore, dbpackage.RepositoryExtra:
upstream = "upstream-core-extra"
case dbpackage.RepositoryCommunity:
upstream = "upstream-community"
}
pkg.Pkgbuild = filepath.Join(conf.Basedir.Upstream, upstream, dbPkg.Pkgbase, "repos", dbPkg.Repository.String()+"-"+conf.Arch, "PKGBUILD")
if err = pkg.genSrcinfo(); err != nil {
return err return err
} }
for _, path := range packages { // check if pkg signature is valid
pkgfile := PKGFile(path) valid, err := pkgfile.isSignatureValid()
dbPkg, err := pkgfile.DBPackage() if err != nil {
if err != nil { return err
log.Warningf("[HK] Unable to find entry for %s in db: %v", filepath.Base(path), err) }
// TODO: remove orphan file not tracked by db (WTF kmod-debug!) if !valid {
continue log.Infof("[HK/%s/%s] invalid package signature", pkg.FullRepo, pkg.Pkgbase)
} buildManager.repoPurge[pkg.FullRepo] <- pkg
continue
}
pkg := &BuildPackage{ // compare db-version with repo version
Pkgbase: dbPkg.Pkgbase, repoVer, err := pkg.repoVersion()
Repo: dbPkg.Repository, if err != nil {
FullRepo: dbPkg.Repository.String() + "-" + dbPkg.March, log.Infof("[HK/%s/%s] package not present on disk", pkg.FullRepo, pkg.Pkgbase)
} // error means package was not found -> delete version & hash from db so rebuild can happen
err := dbPkg.Update().ClearHash().ClearRepoVersion().Exec(context.Background())
var upstream string
switch dbPkg.Repository {
case dbpackage.RepositoryCore, dbpackage.RepositoryExtra:
upstream = "upstream-core-extra"
case dbpackage.RepositoryCommunity:
upstream = "upstream-community"
}
pkg.Pkgbuild = filepath.Join(conf.Basedir.Upstream, upstream, dbPkg.Pkgbase, "repos", dbPkg.Repository.String()+"-"+conf.Arch, "PKGBUILD")
if err = pkg.genSrcinfo(); err != nil {
return err
}
// check if pkg signature is valid
valid, err := pkgfile.isSignatureValid()
if err != nil { if err != nil {
return err return err
} }
if !valid { } else if alpm.VerCmp(repoVer, dbPkg.RepoVersion) != 0 {
log.Infof("[HK/%s/%s] invalid package signature", pkg.FullRepo, pkg.Pkgbase) log.Infof("[HK/%s/%s] update %s->%s in db", pkg.FullRepo, pkg.Pkgbase, dbPkg.RepoVersion, repoVer)
buildManager.repoPurge[pkg.FullRepo] <- pkg dbPkg, err = dbPkg.Update().SetRepoVersion(repoVer).Save(context.Background())
continue
}
// compare db-version with repo version
repoVer, err := pkg.repoVersion()
if err != nil {
log.Infof("[HK/%s/%s] package not present on disk", pkg.FullRepo, pkg.Pkgbase)
// error means package was not found -> delete version & hash from db so rebuild can happen
err := dbPkg.Update().ClearHash().ClearRepoVersion().Exec(context.Background())
if err != nil {
return err
}
} else if alpm.VerCmp(repoVer, dbPkg.RepoVersion) != 0 {
log.Infof("[HK/%s/%s] update %s->%s in db", pkg.FullRepo, pkg.Pkgbase, dbPkg.RepoVersion, repoVer)
dbPkg, err = dbPkg.Update().SetRepoVersion(repoVer).Save(context.Background())
if err != nil {
return err
}
}
// TODO: check split packages
// check if package is still part of repo
dbs, err := alpmHandle.SyncDBs()
if err != nil { if err != nil {
return err return err
} }
pkgResolved, err := dbs.FindSatisfier(pkg.Srcinfo.Packages[0].Pkgname) }
if err != nil || pkgResolved.DB().Name() != dbPkg.Repository.String() {
// package not found on mirror/db -> not part of any repo anymore // TODO: check split packages
log.Infof("[HK/%s/%s] not part of repo", pkg.FullRepo, pkg.Pkgbase)
buildManager.repoPurge[pkg.FullRepo] <- pkg // check if package is still part of repo
err = db.DbPackage.DeleteOne(dbPkg).Exec(context.Background()) dbs, err := alpmHandle.SyncDBs()
if err != nil { if err != nil {
return err return err
} }
continue pkgResolved, err := dbs.FindSatisfier(pkg.Srcinfo.Packages[0].Pkgname)
if err != nil || pkgResolved.DB().Name() != dbPkg.Repository.String() {
// package not found on mirror/db -> not part of any repo anymore
log.Infof("[HK/%s/%s] not part of repo", pkg.FullRepo, pkg.Pkgbase)
buildManager.repoPurge[pkg.FullRepo] <- pkg
err = db.DbPackage.DeleteOne(dbPkg).Exec(context.Background())
if err != nil {
return err
} }
continue
} }
} }