BertalanD/ALHP.GO
1
0
Fork 0
forked from ALHP/ALHP.GO
Code Pull Requests Activity

use lists for repo-actions

Browse Source
This commit is contained in:
Giovanni Harting
2021-12-20 16:04:13 +01:00
parent 42e95d0b75
commit f009e9ff00
2 changed files with 68 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

118
main.go
View File

@@ -189,7 +189,7 @@ func (b *BuildManager) buildWorker(id int, march string) {
pkg.DbPackage.Update().SetStatus(dbpackage.StatusFailed).ClearSkipReason().SetBuildTimeStart(start).SetBuildTimeEnd(time.Now().UTC()).SetHash(pkg.Hash).ExecX(context.Background()) pkg.DbPackage.Update().SetStatus(dbpackage.StatusFailed).ClearSkipReason().SetBuildTimeStart(start).SetBuildTimeEnd(time.Now().UTC()).SetHash(pkg.Hash).ExecX(context.Background())
// purge failed package from repo // purge failed package from repo
b.repoPurge[pkg.FullRepo] <- pkg b.repoPurge[pkg.FullRepo] <- []*BuildPackage{pkg}
err = cleanBuildDir(buildDir) err = cleanBuildDir(buildDir)
if err != nil { if err != nil {
@@ -251,7 +251,7 @@ func (b *BuildManager) buildWorker(id int, march string) {
} }
log.Infof("[%s/%s/%s] Build successful (%s)", pkg.FullRepo, pkg.Pkgbase, pkg.Version, time.Since(start)) log.Infof("[%s/%s/%s] Build successful (%s)", pkg.FullRepo, pkg.Pkgbase, pkg.Version, time.Since(start))
b.repoAdd[pkg.FullRepo] <- pkg b.repoAdd[pkg.FullRepo] <- []*BuildPackage{pkg}
err = cleanBuildDir(buildDir) err = cleanBuildDir(buildDir)
if err != nil { if err != nil {
@@ -318,7 +318,7 @@ func (b *BuildManager) parseWorker() {
pkg.DbPackage = pkg.DbPackage.Update().SetUpdated(time.Now()).SetVersion(pkg.Version). pkg.DbPackage = pkg.DbPackage.Update().SetUpdated(time.Now()).SetVersion(pkg.Version).
SetPackages(packages2slice(pkg.Srcinfo.Packages)).SetStatus(pkg.DbPackage.Status). SetPackages(packages2slice(pkg.Srcinfo.Packages)).SetStatus(pkg.DbPackage.Status).
SetSkipReason(pkg.DbPackage.SkipReason).SetHash(pkg.Hash).SaveX(context.Background()) SetSkipReason(pkg.DbPackage.SkipReason).SetHash(pkg.Hash).SaveX(context.Background())
b.repoPurge[pkg.FullRepo] <- pkg b.repoPurge[pkg.FullRepo] <- []*BuildPackage{pkg}
b.parseWG.Done() b.parseWG.Done()
continue continue
} else { } else {
@@ -347,13 +347,13 @@ func (b *BuildManager) parseWorker() {
case MultiplePKGBUILDError: case MultiplePKGBUILDError:
log.Infof("Skipped %s: Multiple PKGBUILDs for dependency found: %v", pkg.Srcinfo.Pkgbase, err) log.Infof("Skipped %s: Multiple PKGBUILDs for dependency found: %v", pkg.Srcinfo.Pkgbase, err)
pkg.DbPackage = pkg.DbPackage.Update().SetStatus(dbpackage.StatusSkipped).SetSkipReason("multiple PKGBUILD for dep. found").SaveX(context.Background()) pkg.DbPackage = pkg.DbPackage.Update().SetStatus(dbpackage.StatusSkipped).SetSkipReason("multiple PKGBUILD for dep. found").SaveX(context.Background())
b.repoPurge[pkg.FullRepo] <- pkg b.repoPurge[pkg.FullRepo] <- []*BuildPackage{pkg}
b.parseWG.Done() b.parseWG.Done()
continue continue
case UnableToSatisfyError: case UnableToSatisfyError:
log.Infof("Skipped %s: unable to resolve dependencies: %v", pkg.Srcinfo.Pkgbase, err) log.Infof("Skipped %s: unable to resolve dependencies: %v", pkg.Srcinfo.Pkgbase, err)
pkg.DbPackage = pkg.DbPackage.Update().SetStatus(dbpackage.StatusSkipped).SetSkipReason("unable to resolve dependencies").SaveX(context.Background()) pkg.DbPackage = pkg.DbPackage.Update().SetStatus(dbpackage.StatusSkipped).SetSkipReason("unable to resolve dependencies").SaveX(context.Background())
b.repoPurge[pkg.FullRepo] <- pkg b.repoPurge[pkg.FullRepo] <- []*BuildPackage{pkg}
b.parseWG.Done() b.parseWG.Done()
continue continue
} }
@@ -374,7 +374,7 @@ func (b *BuildManager) parseWorker() {
// Worst case would be clients downloading a package update twice, once from their official mirror, // Worst case would be clients downloading a package update twice, once from their official mirror,
// and then after build from ALHP. Best case we prevent a not buildable package from staying in the repos // and then after build from ALHP. Best case we prevent a not buildable package from staying in the repos
// in an outdated version. // in an outdated version.
b.repoPurge[pkg.FullRepo] <- pkg b.repoPurge[pkg.FullRepo] <- []*BuildPackage{pkg}
b.parseWG.Done() b.parseWG.Done()
continue continue
} }
@@ -541,65 +541,69 @@ func (b *BuildManager) htmlWorker() {
func (b *BuildManager) repoWorker(repo string) { func (b *BuildManager) repoWorker(repo string) {
for { for {
select { select {
case pkg := <-b.repoAdd[repo]: case pkgL := <-b.repoAdd[repo]:
args := []string{"-s", "-v", "-p", "-n", filepath.Join(conf.Basedir.Repo, pkg.FullRepo, "os", conf.Arch, pkg.FullRepo) + ".db.tar.xz"} for _, pkg := range pkgL {
args = append(args, pkg.PkgFiles...) args := []string{"-s", "-v", "-p", "-n", filepath.Join(conf.Basedir.Repo, pkg.FullRepo, "os", conf.Arch, pkg.FullRepo) + ".db.tar.xz"}
cmd := exec.Command("repo-add", args...) args = append(args, pkg.PkgFiles...)
res, err := cmd.CombinedOutput() cmd := exec.Command("repo-add", args...)
log.Debug(string(res)) res, err := cmd.CombinedOutput()
if err != nil && cmd.ProcessState.ExitCode() != 1 { log.Debug(string(res))
log.Panicf("%s while repo-add: %v", string(res), err) if err != nil && cmd.ProcessState.ExitCode() != 1 {
} log.Panicf("%s while repo-add: %v", string(res), err)
}
pkg.toDbPackage(true) pkg.toDbPackage(true)
pkg.DbPackage = pkg.DbPackage.Update().SetStatus(dbpackage.StatusLatest).ClearSkipReason().SetRepoVersion(pkg.Version).SetHash(pkg.Hash).SaveX(context.Background()) pkg.DbPackage = pkg.DbPackage.Update().SetStatus(dbpackage.StatusLatest).ClearSkipReason().SetRepoVersion(pkg.Version).SetHash(pkg.Hash).SaveX(context.Background())
cmd = exec.Command("paccache", cmd = exec.Command("paccache",
"-rc", filepath.Join(conf.Basedir.Repo, pkg.FullRepo, "os", conf.Arch), "-rc", filepath.Join(conf.Basedir.Repo, pkg.FullRepo, "os", conf.Arch),
"-k", "1") "-k", "1")
res, err = cmd.CombinedOutput() res, err = cmd.CombinedOutput()
log.Debug(string(res)) log.Debug(string(res))
check(err) check(err)
updateLastUpdated() updateLastUpdated()
b.buildWG.Done() b.buildWG.Done()
case pkg := <-b.repoPurge[repo]:
if _, err := os.Stat(filepath.Join(conf.Basedir.Repo, pkg.FullRepo, "os", conf.Arch, pkg.FullRepo) + ".db.tar.xz"); err != nil {
continue
} }
if len(pkg.PkgFiles) == 0 { case pkgL := <-b.repoPurge[repo]:
if err := pkg.findPkgFiles(); err != nil { for _, pkg := range pkgL {
log.Warningf("[%s/%s] Unable to find files: %v", pkg.FullRepo, pkg.Pkgbase, err) if _, err := os.Stat(filepath.Join(conf.Basedir.Repo, pkg.FullRepo, "os", conf.Arch, pkg.FullRepo) + ".db.tar.xz"); err != nil {
continue
} else if len(pkg.PkgFiles) == 0 {
continue continue
} }
} if len(pkg.PkgFiles) == 0 {
if err := pkg.findPkgFiles(); err != nil {
log.Warningf("[%s/%s] Unable to find files: %v", pkg.FullRepo, pkg.Pkgbase, err)
continue
} else if len(pkg.PkgFiles) == 0 {
continue
}
}
var realPkgs []string var realPkgs []string
for _, filePath := range pkg.PkgFiles { for _, filePath := range pkg.PkgFiles {
realPkgs = append(realPkgs, Package(filePath).Name()) realPkgs = append(realPkgs, Package(filePath).Name())
} }
b.repoWG.Add(1) b.repoWG.Add(1)
args := []string{"-s", "-v", filepath.Join(conf.Basedir.Repo, pkg.FullRepo, "os", conf.Arch, pkg.FullRepo) + ".db.tar.xz"} args := []string{"-s", "-v", filepath.Join(conf.Basedir.Repo, pkg.FullRepo, "os", conf.Arch, pkg.FullRepo) + ".db.tar.xz"}
args = append(args, realPkgs...) args = append(args, realPkgs...)
cmd := exec.Command("repo-remove", args...) cmd := exec.Command("repo-remove", args...)
res, err := cmd.CombinedOutput() res, err := cmd.CombinedOutput()
log.Debug(string(res)) log.Debug(string(res))
if err != nil && cmd.ProcessState.ExitCode() == 1 { if err != nil && cmd.ProcessState.ExitCode() == 1 {
log.Warningf("Error while deleting package %s: %s", pkg.Pkgbase, string(res)) log.Warningf("Error while deleting package %s: %s", pkg.Pkgbase, string(res))
} }
if pkg.DbPackage != nil { if pkg.DbPackage != nil {
_ = pkg.DbPackage.Update().ClearRepoVersion().Exec(context.Background()) _ = pkg.DbPackage.Update().ClearRepoVersion().Exec(context.Background())
} }
for _, file := range pkg.PkgFiles { for _, file := range pkg.PkgFiles {
_ = os.Remove(file) _ = os.Remove(file)
_ = os.Remove(file + ".sig") _ = os.Remove(file + ".sig")
}
updateLastUpdated()
b.repoWG.Done()
} }
updateLastUpdated()
b.repoWG.Done()
} }
} }
} }
@@ -784,8 +788,8 @@ func main() {
buildManager = &BuildManager{ buildManager = &BuildManager{
build: make(map[string]chan *BuildPackage), build: make(map[string]chan *BuildPackage),
parse: make(chan *BuildPackage, 10000), parse: make(chan *BuildPackage, 10000),
repoPurge: make(map[string]chan *BuildPackage), repoPurge: make(map[string]chan []*BuildPackage),
repoAdd: make(map[string]chan *BuildPackage), repoAdd: make(map[string]chan []*BuildPackage),
exit: false, exit: false,
} }

14
utils.go
View File

@@ -67,8 +67,8 @@ type BuildPackage struct {
type BuildManager struct { type BuildManager struct {
build map[string]chan *BuildPackage build map[string]chan *BuildPackage
parse chan *BuildPackage parse chan *BuildPackage
repoPurge map[string]chan *BuildPackage repoPurge map[string]chan []*BuildPackage
repoAdd map[string]chan *BuildPackage repoAdd map[string]chan []*BuildPackage
exit bool exit bool
buildWG sync.WaitGroup buildWG sync.WaitGroup
parseWG sync.WaitGroup parseWG sync.WaitGroup
@@ -710,7 +710,7 @@ func housekeeping(repo string, wg *sync.WaitGroup) error {
PkgFiles: []string{path}, PkgFiles: []string{path},
March: mPackage.MArch(), March: mPackage.MArch(),
} }
buildManager.repoPurge[pkg.FullRepo] <- pkg buildManager.repoPurge[pkg.FullRepo] <- []*BuildPackage{pkg}
continue continue
} }
@@ -742,7 +742,7 @@ func housekeeping(repo string, wg *sync.WaitGroup) error {
if err != nil || pkgResolved.DB().Name() != pkg.DbPackage.Repository.String() || pkgResolved.DB().Name() != pkg.Repo.String() { if err != nil || pkgResolved.DB().Name() != pkg.DbPackage.Repository.String() || pkgResolved.DB().Name() != pkg.Repo.String() {
// package not found on mirror/db -> not part of any repo anymore // package not found on mirror/db -> not part of any repo anymore
log.Infof("[HK/%s/%s] not included in repo", pkg.FullRepo, pkg.Pkgbase) log.Infof("[HK/%s/%s] not included in repo", pkg.FullRepo, pkg.Pkgbase)
buildManager.repoPurge[pkg.FullRepo] <- pkg buildManager.repoPurge[pkg.FullRepo] <- []*BuildPackage{pkg}
err = db.DbPackage.DeleteOne(pkg.DbPackage).Exec(context.Background()) err = db.DbPackage.DeleteOne(pkg.DbPackage).Exec(context.Background())
if err != nil { if err != nil {
return err return err
@@ -757,7 +757,7 @@ func housekeeping(repo string, wg *sync.WaitGroup) error {
} }
if !valid { if !valid {
log.Infof("[HK/%s/%s] invalid package signature", pkg.FullRepo, pkg.Pkgbase) log.Infof("[HK/%s/%s] invalid package signature", pkg.FullRepo, pkg.Pkgbase)
buildManager.repoPurge[pkg.FullRepo] <- pkg buildManager.repoPurge[pkg.FullRepo] <- []*BuildPackage{pkg}
continue continue
} }
@@ -861,8 +861,8 @@ func syncMarchs() {
for _, repo := range conf.Repos { for _, repo := range conf.Repos {
fRepo := fmt.Sprintf("%s-%s", repo, march) fRepo := fmt.Sprintf("%s-%s", repo, march)
repos = append(repos, fRepo) repos = append(repos, fRepo)
buildManager.repoAdd[fRepo] = make(chan *BuildPackage, conf.Build.Worker) buildManager.repoAdd[fRepo] = make(chan []*BuildPackage, conf.Build.Worker)
buildManager.repoPurge[fRepo] = make(chan *BuildPackage, 10000) buildManager.repoPurge[fRepo] = make(chan []*BuildPackage, 10000)
go buildManager.repoWorker(fRepo) go buildManager.repoWorker(fRepo)
if _, err := os.Stat(filepath.Join(filepath.Join(conf.Basedir.Repo, fRepo, "os", conf.Arch))); os.IsNotExist(err) { if _, err := os.Stat(filepath.Join(filepath.Join(conf.Basedir.Repo, fRepo, "os", conf.Arch))); os.IsNotExist(err) {