RubenKelevra/ALHP.GO
1
0
Fork 0
forked from ALHP/ALHP.GO
Code Issues Pull Requests Projects Releases Wiki Activity

use lists for repo-actions

Browse Source
This commit is contained in:
Giovanni Harting
2021-12-20 16:04:13 +01:00
parent 42e95d0b75
commit f009e9ff00
2 changed files with 68 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
main.go
View File

@@ -189,7 +189,7 @@ func (b *BuildManager) buildWorker(id int, march string) {
pkg.DbPackage.Update().SetStatus(dbpackage.StatusFailed).ClearSkipReason().SetBuildTimeStart(start).SetBuildTimeEnd(time.Now().UTC()).SetHash(pkg.Hash).ExecX(context.Background()) pkg.DbPackage.Update().SetStatus(dbpackage.StatusFailed).ClearSkipReason().SetBuildTimeStart(start).SetBuildTimeEnd(time.Now().UTC()).SetHash(pkg.Hash).ExecX(context.Background())
// purge failed package from repo // purge failed package from repo
b.repoPurge[pkg.FullRepo] <- pkg b.repoPurge[pkg.FullRepo] <- []*BuildPackage{pkg}
err = cleanBuildDir(buildDir) err = cleanBuildDir(buildDir)
if err != nil { if err != nil {
@@ -251,7 +251,7 @@ func (b *BuildManager) buildWorker(id int, march string) {
} }
log.Infof("[%s/%s/%s] Build successful (%s)", pkg.FullRepo, pkg.Pkgbase, pkg.Version, time.Since(start)) log.Infof("[%s/%s/%s] Build successful (%s)", pkg.FullRepo, pkg.Pkgbase, pkg.Version, time.Since(start))
b.repoAdd[pkg.FullRepo] <- pkg b.repoAdd[pkg.FullRepo] <- []*BuildPackage{pkg}
err = cleanBuildDir(buildDir) err = cleanBuildDir(buildDir)
if err != nil { if err != nil {
@@ -318,7 +318,7 @@ func (b *BuildManager) parseWorker() {
pkg.DbPackage = pkg.DbPackage.Update().SetUpdated(time.Now()).SetVersion(pkg.Version). pkg.DbPackage = pkg.DbPackage.Update().SetUpdated(time.Now()).SetVersion(pkg.Version).
SetPackages(packages2slice(pkg.Srcinfo.Packages)).SetStatus(pkg.DbPackage.Status). SetPackages(packages2slice(pkg.Srcinfo.Packages)).SetStatus(pkg.DbPackage.Status).
SetSkipReason(pkg.DbPackage.SkipReason).SetHash(pkg.Hash).SaveX(context.Background()) SetSkipReason(pkg.DbPackage.SkipReason).SetHash(pkg.Hash).SaveX(context.Background())
b.repoPurge[pkg.FullRepo] <- pkg b.repoPurge[pkg.FullRepo] <- []*BuildPackage{pkg}
b.parseWG.Done() b.parseWG.Done()
continue continue
} else { } else {
@@ -347,13 +347,13 @@ func (b *BuildManager) parseWorker() {
case MultiplePKGBUILDError: case MultiplePKGBUILDError:
log.Infof("Skipped %s: Multiple PKGBUILDs for dependency found: %v", pkg.Srcinfo.Pkgbase, err) log.Infof("Skipped %s: Multiple PKGBUILDs for dependency found: %v", pkg.Srcinfo.Pkgbase, err)
pkg.DbPackage = pkg.DbPackage.Update().SetStatus(dbpackage.StatusSkipped).SetSkipReason("multiple PKGBUILD for dep. found").SaveX(context.Background()) pkg.DbPackage = pkg.DbPackage.Update().SetStatus(dbpackage.StatusSkipped).SetSkipReason("multiple PKGBUILD for dep. found").SaveX(context.Background())
b.repoPurge[pkg.FullRepo] <- pkg b.repoPurge[pkg.FullRepo] <- []*BuildPackage{pkg}
b.parseWG.Done() b.parseWG.Done()
continue continue
case UnableToSatisfyError: case UnableToSatisfyError:
log.Infof("Skipped %s: unable to resolve dependencies: %v", pkg.Srcinfo.Pkgbase, err) log.Infof("Skipped %s: unable to resolve dependencies: %v", pkg.Srcinfo.Pkgbase, err)
pkg.DbPackage = pkg.DbPackage.Update().SetStatus(dbpackage.StatusSkipped).SetSkipReason("unable to resolve dependencies").SaveX(context.Background()) pkg.DbPackage = pkg.DbPackage.Update().SetStatus(dbpackage.StatusSkipped).SetSkipReason("unable to resolve dependencies").SaveX(context.Background())
b.repoPurge[pkg.FullRepo] <- pkg b.repoPurge[pkg.FullRepo] <- []*BuildPackage{pkg}
b.parseWG.Done() b.parseWG.Done()
continue continue
} }
@@ -374,7 +374,7 @@ func (b *BuildManager) parseWorker() {
// Worst case would be clients downloading a package update twice, once from their official mirror, // Worst case would be clients downloading a package update twice, once from their official mirror,
// and then after build from ALHP. Best case we prevent a not buildable package from staying in the repos // and then after build from ALHP. Best case we prevent a not buildable package from staying in the repos
// in an outdated version. // in an outdated version.
b.repoPurge[pkg.FullRepo] <- pkg b.repoPurge[pkg.FullRepo] <- []*BuildPackage{pkg}
b.parseWG.Done() b.parseWG.Done()
continue continue
} }
@@ -541,7 +541,8 @@ func (b *BuildManager) htmlWorker() {
func (b *BuildManager) repoWorker(repo string) { func (b *BuildManager) repoWorker(repo string) {
for { for {
select { select {
case pkg := <-b.repoAdd[repo]: case pkgL := <-b.repoAdd[repo]:
for _, pkg := range pkgL {
args := []string{"-s", "-v", "-p", "-n", filepath.Join(conf.Basedir.Repo, pkg.FullRepo, "os", conf.Arch, pkg.FullRepo) + ".db.tar.xz"} args := []string{"-s", "-v", "-p", "-n", filepath.Join(conf.Basedir.Repo, pkg.FullRepo, "os", conf.Arch, pkg.FullRepo) + ".db.tar.xz"}
args = append(args, pkg.PkgFiles...) args = append(args, pkg.PkgFiles...)
cmd := exec.Command("repo-add", args...) cmd := exec.Command("repo-add", args...)
@@ -562,7 +563,9 @@ func (b *BuildManager) repoWorker(repo string) {
check(err) check(err)
updateLastUpdated() updateLastUpdated()
b.buildWG.Done() b.buildWG.Done()
case pkg := <-b.repoPurge[repo]: }
case pkgL := <-b.repoPurge[repo]:
for _, pkg := range pkgL {
if _, err := os.Stat(filepath.Join(conf.Basedir.Repo, pkg.FullRepo, "os", conf.Arch, pkg.FullRepo) + ".db.tar.xz"); err != nil { if _, err := os.Stat(filepath.Join(conf.Basedir.Repo, pkg.FullRepo, "os", conf.Arch, pkg.FullRepo) + ".db.tar.xz"); err != nil {
continue continue
} }
@@ -603,6 +606,7 @@ func (b *BuildManager) repoWorker(repo string) {
} }
} }
} }
}
func (b *BuildManager) syncWorker() { func (b *BuildManager) syncWorker() {
check(os.MkdirAll(filepath.Join(conf.Basedir.Work, upstreamDir), 0755)) check(os.MkdirAll(filepath.Join(conf.Basedir.Work, upstreamDir), 0755))
@@ -784,8 +788,8 @@ func main() {
buildManager = &BuildManager{ buildManager = &BuildManager{
build: make(map[string]chan *BuildPackage), build: make(map[string]chan *BuildPackage),
parse: make(chan *BuildPackage, 10000), parse: make(chan *BuildPackage, 10000),
repoPurge: make(map[string]chan *BuildPackage), repoPurge: make(map[string]chan []*BuildPackage),
repoAdd: make(map[string]chan *BuildPackage), repoAdd: make(map[string]chan []*BuildPackage),
exit: false, exit: false,
} }

14
utils.go
View File

@@ -67,8 +67,8 @@ type BuildPackage struct {
type BuildManager struct { type BuildManager struct {
build map[string]chan *BuildPackage build map[string]chan *BuildPackage
parse chan *BuildPackage parse chan *BuildPackage
repoPurge map[string]chan *BuildPackage repoPurge map[string]chan []*BuildPackage
repoAdd map[string]chan *BuildPackage repoAdd map[string]chan []*BuildPackage
exit bool exit bool
buildWG sync.WaitGroup buildWG sync.WaitGroup
parseWG sync.WaitGroup parseWG sync.WaitGroup
@@ -710,7 +710,7 @@ func housekeeping(repo string, wg *sync.WaitGroup) error {
PkgFiles: []string{path}, PkgFiles: []string{path},
March: mPackage.MArch(), March: mPackage.MArch(),
} }
buildManager.repoPurge[pkg.FullRepo] <- pkg buildManager.repoPurge[pkg.FullRepo] <- []*BuildPackage{pkg}
continue continue
} }
@@ -742,7 +742,7 @@ func housekeeping(repo string, wg *sync.WaitGroup) error {
if err != nil || pkgResolved.DB().Name() != pkg.DbPackage.Repository.String() || pkgResolved.DB().Name() != pkg.Repo.String() { if err != nil || pkgResolved.DB().Name() != pkg.DbPackage.Repository.String() || pkgResolved.DB().Name() != pkg.Repo.String() {
// package not found on mirror/db -> not part of any repo anymore // package not found on mirror/db -> not part of any repo anymore
log.Infof("[HK/%s/%s] not included in repo", pkg.FullRepo, pkg.Pkgbase) log.Infof("[HK/%s/%s] not included in repo", pkg.FullRepo, pkg.Pkgbase)
buildManager.repoPurge[pkg.FullRepo] <- pkg buildManager.repoPurge[pkg.FullRepo] <- []*BuildPackage{pkg}
err = db.DbPackage.DeleteOne(pkg.DbPackage).Exec(context.Background()) err = db.DbPackage.DeleteOne(pkg.DbPackage).Exec(context.Background())
if err != nil { if err != nil {
return err return err
@@ -757,7 +757,7 @@ func housekeeping(repo string, wg *sync.WaitGroup) error {
} }
if !valid { if !valid {
log.Infof("[HK/%s/%s] invalid package signature", pkg.FullRepo, pkg.Pkgbase) log.Infof("[HK/%s/%s] invalid package signature", pkg.FullRepo, pkg.Pkgbase)
buildManager.repoPurge[pkg.FullRepo] <- pkg buildManager.repoPurge[pkg.FullRepo] <- []*BuildPackage{pkg}
continue continue
} }
@@ -861,8 +861,8 @@ func syncMarchs() {
for _, repo := range conf.Repos { for _, repo := range conf.Repos {
fRepo := fmt.Sprintf("%s-%s", repo, march) fRepo := fmt.Sprintf("%s-%s", repo, march)
repos = append(repos, fRepo) repos = append(repos, fRepo)
buildManager.repoAdd[fRepo] = make(chan *BuildPackage, conf.Build.Worker) buildManager.repoAdd[fRepo] = make(chan []*BuildPackage, conf.Build.Worker)
buildManager.repoPurge[fRepo] = make(chan *BuildPackage, 10000) buildManager.repoPurge[fRepo] = make(chan []*BuildPackage, 10000)
go buildManager.repoWorker(fRepo) go buildManager.repoWorker(fRepo)
if _, err := os.Stat(filepath.Join(filepath.Join(conf.Basedir.Repo, fRepo, "os", conf.Arch))); os.IsNotExist(err) { if _, err := os.Stat(filepath.Join(filepath.Join(conf.Basedir.Repo, fRepo, "os", conf.Arch))); os.IsNotExist(err) {