forked from ALHP/ALHP.GO
faster SRCINFO parsing + memory-based building
Switched to parsing srcinfo with channels on all available cpus, speeding up srcinfo-parsing and queue generation by a lot. New memory-limit based building will max out the available memory while not building the same packages at the same time for different marchs, fixing some long-standing bugs like firefox not building at the same time because the same ports are used for profile-based optimization. This also drops the artificial delay on build-start, speeding up things even more. This also means there is no hard-coded limit on how many packages can be build at once anymore. As long as there is RAM available, builds will be started.
This commit is contained in:
631
buildmanager.go
Normal file
631
buildmanager.go
Normal file
@@ -0,0 +1,631 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/c2h5oh/datasize"
|
||||
"github.com/sethvargo/go-retry"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"html/template"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"somegit.dev/ALHP/ALHP.GO/ent"
|
||||
"somegit.dev/ALHP/ALHP.GO/ent/dbpackage"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type BuildManager struct {
|
||||
repoPurge map[string]chan []*ProtoPackage
|
||||
repoAdd map[string]chan []*ProtoPackage
|
||||
repoWG *sync.WaitGroup
|
||||
alpmMutex *sync.RWMutex
|
||||
building []*ProtoPackage
|
||||
buildingLock *sync.RWMutex
|
||||
queueSignal chan struct{}
|
||||
}
|
||||
|
||||
func (b *BuildManager) refreshSRCINFOs(ctx context.Context, path string) error {
|
||||
pkgBuilds, err := Glob(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error scanning for PKGBUILDs: %w", err)
|
||||
}
|
||||
|
||||
wg := new(sync.WaitGroup)
|
||||
workerChan := make(chan string, runtime.NumCPU())
|
||||
|
||||
for o := 0; o < runtime.NumCPU(); o++ {
|
||||
wg.Add(1)
|
||||
go b.SRCINFOWorker(ctx, workerChan, wg)
|
||||
}
|
||||
|
||||
go func() {
|
||||
for _, pkgbuild := range pkgBuilds {
|
||||
workerChan <- pkgbuild
|
||||
}
|
||||
close(workerChan)
|
||||
}()
|
||||
|
||||
wg.Wait()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *BuildManager) SRCINFOWorker(ctx context.Context, workIn chan string, wg *sync.WaitGroup) {
|
||||
defer wg.Done()
|
||||
for pkgbuild := range workIn {
|
||||
mPkgbuild := PKGBUILD(pkgbuild)
|
||||
if mPkgbuild.FullRepo() == "trunk" || !Contains(conf.Repos, mPkgbuild.Repo()) ||
|
||||
containsSubStr(mPkgbuild.FullRepo(), conf.Blacklist.Repo) {
|
||||
continue
|
||||
}
|
||||
|
||||
for _, march := range conf.March {
|
||||
dbPkg, dbErr := db.DbPackage.Query().Where(
|
||||
dbpackage.And(
|
||||
dbpackage.Pkgbase(mPkgbuild.PkgBase()),
|
||||
dbpackage.RepositoryEQ(dbpackage.Repository(mPkgbuild.Repo())),
|
||||
dbpackage.March(march),
|
||||
),
|
||||
).Only(context.Background())
|
||||
|
||||
if ent.IsNotFound(dbErr) {
|
||||
log.Debugf("[%s/%s] Package not found in database", mPkgbuild.Repo(), mPkgbuild.PkgBase())
|
||||
} else if dbErr != nil {
|
||||
log.Errorf("[%s/%s] Problem querying db for package: %v", mPkgbuild.Repo(), mPkgbuild.PkgBase(), dbErr)
|
||||
}
|
||||
|
||||
// compare b3sum of PKGBUILD file to hash in database, only proceed if hash differs
|
||||
// reduces the amount of PKGBUILDs that need to be parsed with makepkg, which is _really_ slow, significantly
|
||||
b3s, err := b3sum(pkgbuild)
|
||||
if err != nil {
|
||||
log.Errorf("Error hashing PKGBUILD: %v", err)
|
||||
}
|
||||
|
||||
if dbPkg != nil && b3s == dbPkg.Hash {
|
||||
log.Debugf("[%s/%s] Skipped: PKGBUILD hash matches db (%s)", mPkgbuild.Repo(), mPkgbuild.PkgBase(), b3s)
|
||||
continue
|
||||
} else if dbPkg != nil && b3s != dbPkg.Hash && dbPkg.SrcinfoHash != b3s {
|
||||
log.Debugf("[%s/%s] srcinfo cleared", mPkgbuild.Repo(), mPkgbuild.PkgBase())
|
||||
dbPkg = dbPkg.Update().ClearSrcinfo().SaveX(context.Background())
|
||||
}
|
||||
|
||||
proto := &ProtoPackage{
|
||||
Pkgbuild: pkgbuild,
|
||||
Pkgbase: mPkgbuild.PkgBase(),
|
||||
Repo: dbpackage.Repository(mPkgbuild.Repo()),
|
||||
March: march,
|
||||
FullRepo: mPkgbuild.Repo() + "-" + march,
|
||||
Hash: b3s,
|
||||
DBPackage: dbPkg,
|
||||
}
|
||||
|
||||
_, err = proto.isEligible(ctx)
|
||||
if err != nil {
|
||||
log.Infof("Unable to determine status for package %s: %v", proto.Pkgbase, err)
|
||||
b.repoPurge[proto.FullRepo] <- []*ProtoPackage{proto}
|
||||
} else if proto.DBPackage != nil {
|
||||
proto.DBPackage.Update().SetPkgbuild(proto.Pkgbuild).ExecX(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (b *BuildManager) buildQueue(queue []*ProtoPackage, ctx context.Context) error {
|
||||
var doneQ []*ProtoPackage
|
||||
doneQLock := new(sync.RWMutex)
|
||||
var unknownBuilds bool
|
||||
|
||||
for len(doneQ) != len(queue) {
|
||||
up := 0
|
||||
b.buildingLock.RLock()
|
||||
if (pkgList2MaxMem(b.building) < conf.Build.MemoryLimit && !unknownBuilds) || (unknownBuilds && len(b.building) < 1) {
|
||||
b.buildingLock.RUnlock()
|
||||
for _, pkg := range queue {
|
||||
// check if package is already build
|
||||
doneQLock.RLock()
|
||||
if ContainsPkg(doneQ, pkg, true) {
|
||||
doneQLock.RUnlock()
|
||||
continue
|
||||
}
|
||||
doneQLock.RUnlock()
|
||||
|
||||
// check if package is already building (we do not build packages from different marchs simultaneously)
|
||||
b.buildingLock.RLock()
|
||||
if ContainsPkg(b.building, pkg, false) {
|
||||
log.Debugf("[Q] skipped already building package %s->%s", pkg.FullRepo, pkg.Pkgbase)
|
||||
b.buildingLock.RUnlock()
|
||||
continue
|
||||
}
|
||||
b.buildingLock.RUnlock()
|
||||
|
||||
// only check for memory on known-memory-builds
|
||||
// otherwise build them one-at-a-time
|
||||
// TODO: add initial compile mode for new repos
|
||||
if !unknownBuilds {
|
||||
// check if package has unknown memory usage
|
||||
if pkg.DBPackage.MaxRss == nil {
|
||||
log.Debugf("[Q] skipped unknown package %s->%s", pkg.FullRepo, pkg.Pkgbase)
|
||||
up++
|
||||
continue
|
||||
}
|
||||
|
||||
// check if package can be build with current memory limit
|
||||
if datasize.ByteSize(*pkg.DBPackage.MaxRss)*datasize.KB > conf.Build.MemoryLimit {
|
||||
log.Warningf("[Q] %s->%s exeeds memory limit: %s->%s", pkg.FullRepo, pkg.Pkgbase,
|
||||
datasize.ByteSize(*pkg.DBPackage.MaxRss)*datasize.KB, conf.Build.MemoryLimit)
|
||||
doneQLock.Lock()
|
||||
doneQ = append(doneQ, pkg)
|
||||
doneQLock.Unlock()
|
||||
continue
|
||||
}
|
||||
|
||||
b.buildingLock.RLock()
|
||||
currentMemLoad := pkgList2MaxMem(b.building)
|
||||
b.buildingLock.RUnlock()
|
||||
|
||||
// check if package can be build right now
|
||||
if !unknownBuilds && currentMemLoad+(datasize.ByteSize(*pkg.DBPackage.MaxRss)*datasize.KB) > conf.Build.MemoryLimit {
|
||||
log.Debugf("[Q] skipped package with max_rss %s while load %s: %s->%s",
|
||||
datasize.ByteSize(*pkg.DBPackage.MaxRss)*datasize.KB, currentMemLoad, pkg.Pkgbase, pkg.March)
|
||||
continue
|
||||
}
|
||||
} else {
|
||||
b.buildingLock.RLock()
|
||||
if len(b.building) >= 1 {
|
||||
b.buildingLock.RUnlock()
|
||||
continue
|
||||
}
|
||||
b.buildingLock.RUnlock()
|
||||
}
|
||||
|
||||
b.buildingLock.Lock()
|
||||
b.building = append(b.building, pkg)
|
||||
b.buildingLock.Unlock()
|
||||
|
||||
go func(pkg *ProtoPackage) {
|
||||
dur, err := pkg.build(ctx)
|
||||
if err != nil {
|
||||
log.Warningf("[Q] error building package %s->%s in %s: %s", pkg.FullRepo, pkg.Pkgbase, dur, err)
|
||||
b.repoPurge[pkg.FullRepo] <- []*ProtoPackage{pkg}
|
||||
} else {
|
||||
log.Infof("[Q] build successful: %s->%s (%s)", pkg.FullRepo, pkg.Pkgbase, dur)
|
||||
}
|
||||
doneQLock.Lock()
|
||||
b.buildingLock.Lock()
|
||||
doneQ = append(doneQ, pkg)
|
||||
|
||||
for i := 0; i < len(b.building); i++ {
|
||||
if b.building[i].PkgbaseEquals(pkg, true) {
|
||||
b.building = append(b.building[:i], b.building[i+1:]...)
|
||||
break
|
||||
}
|
||||
}
|
||||
doneQLock.Unlock()
|
||||
b.buildingLock.Unlock()
|
||||
b.queueSignal <- struct{}{}
|
||||
}(pkg)
|
||||
}
|
||||
} else {
|
||||
log.Debugf("[Q] memory/build limit reached, waiting for package to finish...")
|
||||
b.buildingLock.RUnlock()
|
||||
<-b.queueSignal
|
||||
}
|
||||
|
||||
// if only unknown packages are left, enable unknown buildmode
|
||||
b.buildingLock.RLock()
|
||||
if up == len(queue)-(len(doneQ)+len(b.building)) {
|
||||
unknownBuilds = true
|
||||
}
|
||||
b.buildingLock.RUnlock()
|
||||
up = 0
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *BuildManager) htmlWorker(ctx context.Context) {
|
||||
type Pkg struct {
|
||||
Pkgbase string
|
||||
Status string
|
||||
Class string
|
||||
Skip string
|
||||
Version string
|
||||
Svn2GitVersion string
|
||||
BuildDate string
|
||||
BuildDuration time.Duration
|
||||
Checked string
|
||||
Log string
|
||||
LTO bool
|
||||
LTOUnknown bool
|
||||
LTODisabled bool
|
||||
LTOAutoDisabled bool
|
||||
DebugSym bool
|
||||
DebugSymNotAvailable bool
|
||||
DebugSymUnknown bool
|
||||
}
|
||||
|
||||
type Repo struct {
|
||||
Name string
|
||||
Packages []Pkg
|
||||
}
|
||||
|
||||
type March struct {
|
||||
Name string
|
||||
Repos []Repo
|
||||
}
|
||||
|
||||
type tpl struct {
|
||||
March []March
|
||||
Generated string
|
||||
Latest int
|
||||
Failed int
|
||||
Skipped int
|
||||
Queued int
|
||||
LTOEnabled int
|
||||
LTOUnknown int
|
||||
LTODisabled int
|
||||
}
|
||||
|
||||
for {
|
||||
gen := &tpl{}
|
||||
|
||||
for _, march := range conf.March {
|
||||
addMarch := March{
|
||||
Name: march,
|
||||
}
|
||||
|
||||
for _, repo := range conf.Repos {
|
||||
addRepo := Repo{
|
||||
Name: repo,
|
||||
}
|
||||
|
||||
pkgs := db.DbPackage.Query().Order(ent.Asc(dbpackage.FieldPkgbase)).
|
||||
Where(dbpackage.MarchEQ(march), dbpackage.RepositoryEQ(dbpackage.Repository(repo))).AllX(ctx)
|
||||
|
||||
for _, pkg := range pkgs {
|
||||
addPkg := Pkg{
|
||||
Pkgbase: pkg.Pkgbase,
|
||||
Status: strings.ToUpper(pkg.Status.String()),
|
||||
Class: statusID2string(pkg.Status),
|
||||
Skip: pkg.SkipReason,
|
||||
Version: pkg.RepoVersion,
|
||||
Svn2GitVersion: pkg.Version,
|
||||
}
|
||||
|
||||
if pkg.STime != nil && pkg.UTime != nil {
|
||||
addPkg.BuildDuration = time.Duration(*pkg.STime+*pkg.UTime) * time.Second
|
||||
}
|
||||
|
||||
if !pkg.BuildTimeStart.IsZero() {
|
||||
addPkg.BuildDate = pkg.BuildTimeStart.UTC().Format(time.RFC1123)
|
||||
}
|
||||
|
||||
if !pkg.Updated.IsZero() {
|
||||
addPkg.Checked = pkg.Updated.UTC().Format(time.RFC1123)
|
||||
}
|
||||
|
||||
if pkg.Status == dbpackage.StatusFailed {
|
||||
addPkg.Log = fmt.Sprintf("%s/%s/%s.log", logDir, pkg.March, pkg.Pkgbase)
|
||||
}
|
||||
|
||||
switch pkg.Lto {
|
||||
case dbpackage.LtoUnknown:
|
||||
if pkg.Status != dbpackage.StatusSkipped && pkg.Status != dbpackage.StatusFailed {
|
||||
addPkg.LTOUnknown = true
|
||||
}
|
||||
case dbpackage.LtoEnabled:
|
||||
addPkg.LTO = true
|
||||
case dbpackage.LtoDisabled:
|
||||
addPkg.LTODisabled = true
|
||||
case dbpackage.LtoAutoDisabled:
|
||||
addPkg.LTOAutoDisabled = true
|
||||
}
|
||||
|
||||
switch pkg.DebugSymbols {
|
||||
case dbpackage.DebugSymbolsUnknown:
|
||||
if pkg.Status != dbpackage.StatusSkipped && pkg.Status != dbpackage.StatusFailed {
|
||||
addPkg.DebugSymUnknown = true
|
||||
}
|
||||
case dbpackage.DebugSymbolsAvailable:
|
||||
addPkg.DebugSym = true
|
||||
case dbpackage.DebugSymbolsNotAvailable:
|
||||
addPkg.DebugSymNotAvailable = true
|
||||
}
|
||||
|
||||
addRepo.Packages = append(addRepo.Packages, addPkg)
|
||||
}
|
||||
addMarch.Repos = append(addMarch.Repos, addRepo)
|
||||
}
|
||||
gen.March = append(gen.March, addMarch)
|
||||
}
|
||||
|
||||
gen.Generated = time.Now().UTC().Format(time.RFC1123)
|
||||
|
||||
var v []struct {
|
||||
Status dbpackage.Status `json:"status"`
|
||||
Count int `json:"count"`
|
||||
}
|
||||
|
||||
db.DbPackage.Query().GroupBy(dbpackage.FieldStatus).Aggregate(ent.Count()).ScanX(ctx, &v)
|
||||
|
||||
for _, c := range v {
|
||||
switch c.Status {
|
||||
case dbpackage.StatusFailed:
|
||||
gen.Failed = c.Count
|
||||
case dbpackage.StatusSkipped:
|
||||
gen.Skipped = c.Count
|
||||
case dbpackage.StatusLatest:
|
||||
gen.Latest = c.Count
|
||||
case dbpackage.StatusQueued:
|
||||
gen.Queued = c.Count
|
||||
}
|
||||
}
|
||||
|
||||
var v2 []struct {
|
||||
Status dbpackage.Lto `json:"lto"`
|
||||
Count int `json:"count"`
|
||||
}
|
||||
|
||||
db.DbPackage.Query().Where(dbpackage.StatusNEQ(dbpackage.StatusSkipped)).
|
||||
GroupBy(dbpackage.FieldLto).Aggregate(ent.Count()).ScanX(ctx, &v2)
|
||||
|
||||
for _, c := range v2 {
|
||||
switch c.Status {
|
||||
case dbpackage.LtoUnknown:
|
||||
gen.LTOUnknown = c.Count
|
||||
case dbpackage.LtoDisabled, dbpackage.LtoAutoDisabled:
|
||||
gen.LTODisabled += c.Count
|
||||
case dbpackage.LtoEnabled:
|
||||
gen.LTOEnabled = c.Count
|
||||
}
|
||||
}
|
||||
|
||||
statusTpl, err := template.ParseFiles("tpl/packages.html")
|
||||
if err != nil {
|
||||
log.Warningf("[HTML] Error parsing template file: %v", err)
|
||||
continue
|
||||
}
|
||||
|
||||
f, err := os.OpenFile(filepath.Join(conf.Basedir.Repo, "packages.html"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644)
|
||||
if err != nil {
|
||||
log.Warningf("[HTML] Erro ropening output file: %v", err)
|
||||
continue
|
||||
}
|
||||
err = statusTpl.Execute(f, gen)
|
||||
if err != nil {
|
||||
log.Warningf("[HTML] Error filling template: %v", err)
|
||||
}
|
||||
_ = f.Close()
|
||||
|
||||
time.Sleep(time.Minute * 5)
|
||||
}
|
||||
}
|
||||
|
||||
func (b *BuildManager) repoWorker(repo string) {
|
||||
for {
|
||||
select {
|
||||
case pkgL := <-b.repoAdd[repo]:
|
||||
b.repoWG.Add(1)
|
||||
toAdd := make([]string, 0)
|
||||
for _, pkg := range pkgL {
|
||||
toAdd = append(toAdd, pkg.PkgFiles...)
|
||||
}
|
||||
|
||||
args := []string{"-s", "-v", "-p", "-n", filepath.Join(conf.Basedir.Repo, repo, "os", conf.Arch, repo) + ".db.tar.xz"}
|
||||
args = append(args, toAdd...)
|
||||
cmd := exec.Command("repo-add", args...)
|
||||
res, err := cmd.CombinedOutput()
|
||||
log.Debug(string(res))
|
||||
if err != nil && cmd.ProcessState.ExitCode() != 1 {
|
||||
log.Panicf("%s while repo-add: %v", string(res), err)
|
||||
}
|
||||
|
||||
for _, pkg := range pkgL {
|
||||
pkg.toDBPackage(true)
|
||||
if _, err := os.Stat(filepath.Join(conf.Basedir.Debug, pkg.March,
|
||||
pkg.DBPackage.Packages[0]+"-debug-"+pkg.Version+"-"+conf.Arch+".pkg.tar.zst")); err == nil {
|
||||
pkg.DBPackage = pkg.DBPackage.Update().
|
||||
SetStatus(dbpackage.StatusLatest).
|
||||
ClearSkipReason().
|
||||
SetDebugSymbols(dbpackage.DebugSymbolsAvailable).
|
||||
SetRepoVersion(pkg.Version).
|
||||
SetHash(pkg.Hash).
|
||||
SaveX(context.Background())
|
||||
} else {
|
||||
pkg.DBPackage = pkg.DBPackage.Update().
|
||||
SetStatus(dbpackage.StatusLatest).
|
||||
ClearSkipReason().
|
||||
SetDebugSymbols(dbpackage.DebugSymbolsNotAvailable).
|
||||
SetRepoVersion(pkg.Version).
|
||||
SetHash(pkg.Hash).
|
||||
SaveX(context.Background())
|
||||
}
|
||||
}
|
||||
|
||||
cmd = exec.Command("paccache", "-rc", filepath.Join(conf.Basedir.Repo, repo, "os", conf.Arch), "-k", "1") //nolint:gosec
|
||||
res, err = cmd.CombinedOutput()
|
||||
log.Debug(string(res))
|
||||
if err != nil {
|
||||
log.Warningf("Error running paccache: %v", err)
|
||||
}
|
||||
err = updateLastUpdated()
|
||||
if err != nil {
|
||||
log.Warningf("Error updating lastupdate: %v", err)
|
||||
}
|
||||
b.repoWG.Done()
|
||||
case pkgL := <-b.repoPurge[repo]:
|
||||
for _, pkg := range pkgL {
|
||||
if _, err := os.Stat(filepath.Join(conf.Basedir.Repo, pkg.FullRepo, "os", conf.Arch, pkg.FullRepo) + ".db.tar.xz"); err != nil {
|
||||
continue
|
||||
}
|
||||
if len(pkg.PkgFiles) == 0 {
|
||||
if err := pkg.findPkgFiles(); err != nil {
|
||||
log.Warningf("[%s/%s] Unable to find files: %v", pkg.FullRepo, pkg.Pkgbase, err)
|
||||
continue
|
||||
} else if len(pkg.PkgFiles) == 0 {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
var realPkgs []string
|
||||
for _, filePath := range pkg.PkgFiles {
|
||||
if _, err := os.Stat(filePath); err == nil {
|
||||
realPkgs = append(realPkgs, Package(filePath).Name())
|
||||
}
|
||||
}
|
||||
|
||||
if len(realPkgs) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
b.repoWG.Add(1)
|
||||
args := []string{"-s", "-v", filepath.Join(conf.Basedir.Repo, pkg.FullRepo, "os", conf.Arch, pkg.FullRepo) + ".db.tar.xz"}
|
||||
args = append(args, realPkgs...)
|
||||
cmd := exec.Command("repo-remove", args...)
|
||||
res, err := cmd.CombinedOutput()
|
||||
log.Debug(string(res))
|
||||
if err != nil && cmd.ProcessState.ExitCode() == 1 {
|
||||
log.Warningf("Error while deleting package %s: %s", pkg.Pkgbase, string(res))
|
||||
}
|
||||
|
||||
if pkg.DBPackage != nil {
|
||||
_ = pkg.DBPackage.Update().ClearRepoVersion().ClearHash().Exec(context.Background())
|
||||
}
|
||||
|
||||
for _, file := range pkg.PkgFiles {
|
||||
_ = os.Remove(file)
|
||||
_ = os.Remove(file + ".sig")
|
||||
}
|
||||
err = updateLastUpdated()
|
||||
if err != nil {
|
||||
log.Warningf("Error updating lastupdate: %v", err)
|
||||
}
|
||||
b.repoWG.Done()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (b *BuildManager) syncWorker(ctx context.Context) error {
|
||||
err := os.MkdirAll(filepath.Join(conf.Basedir.Work, upstreamDir), 0o755)
|
||||
if err != nil {
|
||||
log.Fatalf("Error creating upstream dir: %v", err)
|
||||
}
|
||||
|
||||
for {
|
||||
for gitDir, gitURL := range conf.Svn2git {
|
||||
gitPath := filepath.Join(conf.Basedir.Work, upstreamDir, gitDir)
|
||||
|
||||
if _, err := os.Stat(gitPath); os.IsNotExist(err) {
|
||||
cmd := exec.Command("git", "clone", "--depth=1", gitURL, gitPath)
|
||||
res, err := cmd.CombinedOutput()
|
||||
log.Debug(string(res))
|
||||
if err != nil {
|
||||
log.Fatalf("Error running git clone: %v", err)
|
||||
}
|
||||
} else if err == nil {
|
||||
cmd := exec.Command("git", "reset", "--hard")
|
||||
cmd.Dir = gitPath
|
||||
res, err := cmd.CombinedOutput()
|
||||
log.Debug(string(res))
|
||||
if err != nil {
|
||||
log.Fatalf("Error running git reset: %v", err)
|
||||
}
|
||||
|
||||
cmd = exec.Command("git", "pull")
|
||||
cmd.Dir = gitPath
|
||||
res, err = cmd.CombinedOutput()
|
||||
log.Debug(string(res))
|
||||
if err != nil {
|
||||
log.Warningf("Failed to update git repo %s: %v", gitDir, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// housekeeping
|
||||
wg := new(sync.WaitGroup)
|
||||
for _, repo := range repos {
|
||||
wg.Add(1)
|
||||
splitRepo := strings.Split(repo, "-")
|
||||
repo := repo
|
||||
go func() {
|
||||
err := housekeeping(splitRepo[0], strings.Join(splitRepo[1:], "-"), wg)
|
||||
if err != nil {
|
||||
log.Warningf("[%s] housekeeping failed: %v", repo, err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
err := logHK()
|
||||
if err != nil {
|
||||
log.Warningf("log-housekeeping failed: %v", err)
|
||||
}
|
||||
|
||||
// fetch updates between sync runs
|
||||
b.alpmMutex.Lock()
|
||||
err = alpmHandle.Release()
|
||||
if err != nil {
|
||||
log.Fatalf("Error releasing ALPM handle: %v", err)
|
||||
}
|
||||
|
||||
if err := retry.Fibonacci(ctx, 1*time.Second, func(ctx context.Context) error {
|
||||
if err := setupChroot(); err != nil {
|
||||
log.Warningf("Unable to upgrade chroot, trying again later.")
|
||||
return retry.RetryableError(err)
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
alpmHandle, err = initALPM(filepath.Join(conf.Basedir.Work, chrootDir, pristineChroot),
|
||||
filepath.Join(conf.Basedir.Work, chrootDir, pristineChroot, "/var/lib/pacman"))
|
||||
if err != nil {
|
||||
log.Warningf("Error while ALPM-init: %v", err)
|
||||
}
|
||||
b.alpmMutex.Unlock()
|
||||
|
||||
// do refreshSRCINFOs twice here
|
||||
// since MirrorLatest depends on the DB being correct, there can be packages queued which should not be queued,
|
||||
// so we check them twice to eliminate those.
|
||||
log.Debugf("generating build-queue for PKGBUILDs found in %s", filepath.Join(conf.Basedir.Work, upstreamDir, "/**/PKGBUILD"))
|
||||
err = b.refreshSRCINFOs(ctx, filepath.Join(conf.Basedir.Work, upstreamDir, "/**/PKGBUILD"))
|
||||
if err != nil {
|
||||
log.Fatalf("error refreshing PKGBUILDs: %v", err)
|
||||
}
|
||||
log.Debugf("regenerating build-queue for PKGBUILDs found in %s", filepath.Join(conf.Basedir.Work, upstreamDir, "/**/PKGBUILD"))
|
||||
err = b.refreshSRCINFOs(ctx, filepath.Join(conf.Basedir.Work, upstreamDir, "/**/PKGBUILD"))
|
||||
if err != nil {
|
||||
log.Fatalf("error refreshing PKGBUILDs: %v", err)
|
||||
}
|
||||
|
||||
queue, err := genQueue()
|
||||
if err != nil {
|
||||
log.Warningf("Error building buildQueue: %v", err)
|
||||
} else {
|
||||
log.Debugf("buildQueue with %d items", len(queue))
|
||||
|
||||
err = b.buildQueue(queue, ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if ctx.Err() == nil {
|
||||
for _, repo := range repos {
|
||||
err = movePackagesLive(repo)
|
||||
if err != nil {
|
||||
log.Errorf("[%s] Error moving packages live: %v", repo, err)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
log.Debugf("build-cycle finished")
|
||||
time.Sleep(time.Duration(*checkInterval) * time.Minute)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user