From 8467b17ffc6f89db2b7c79bee68f307cd3e605df Mon Sep 17 00:00:00 2001 From: Giovanni Harting <539@idlegandalf.com> Date: Thu, 18 Nov 2021 13:22:10 +0100 Subject: [PATCH] switched from textfiles to db for failed package status --- main.go | 36 ++++++++++++------------------- utils.go | 65 +++++++++++++------------------------------------------- 2 files changed, 28 insertions(+), 73 deletions(-) diff --git a/main.go b/main.go index 83de23b..eb60120 100644 --- a/main.go +++ b/main.go @@ -62,7 +62,7 @@ func (b *BuildManager) buildWorker(id int) { log.Infof("[%s/%s] Build starting", pkg.FullRepo, pkg.Pkgbase) dbPkg := getDbPackage(pkg) - dbPkg = dbPkg.Update().SetStatus(dbpackage.StatusBuilding).SetBuildTimeStart(time.Now().UTC()).SetSkipReason("").SaveX(context.Background()) + dbPkg = dbPkg.Update().SetStatus(dbpackage.StatusBuilding).SetBuildTimeStart(time.Now().UTC()).ClearSkipReason().SaveX(context.Background()) err := importKeys(pkg) if err != nil { @@ -112,26 +112,16 @@ func (b *BuildManager) buildWorker(id int) { if err != nil { if b.exit { - gitClean(pkg) b.buildWG.Done() continue } log.Warningf("[%s/%s] Build failed, exit code %d", pkg.FullRepo, pkg.Pkgbase, cmd.ProcessState.ExitCode()) - b.failedMutex.Lock() - f, err := os.OpenFile(filepath.Join(conf.Basedir.Repo, pkg.FullRepo+"_failed.txt"), os.O_WRONLY|os.O_APPEND|os.O_CREATE|os.O_SYNC, 0644) - check(err) - - _, err = f.WriteString(fmt.Sprintf("%s==%s\n", pkg.Pkgbase, constructVersion(pkg.Srcinfo.Pkgver, pkg.Srcinfo.Pkgrel, pkg.Srcinfo.Epoch))) - check(err) - check(f.Close()) - b.failedMutex.Unlock() - check(os.MkdirAll(filepath.Join(conf.Basedir.Repo, "logs"), 0755)) check(os.WriteFile(filepath.Join(conf.Basedir.Repo, "logs", pkg.Pkgbase+".log"), out.Bytes(), 0644)) - dbPkg.Update().SetStatus(dbpackage.StatusFailed).SetBuildTimeEnd(time.Now()).SetHash(pkg.Hash).ExecX(context.Background()) + dbPkg.Update().SetStatus(dbpackage.StatusFailed).ClearSkipReason().SetBuildTimeEnd(time.Now()).SetHash(pkg.Hash).ExecX(context.Background()) // purge failed package from repo b.repoPurge[pkg.FullRepo] <- pkg @@ -215,7 +205,6 @@ func (b *BuildManager) parseWorker() { pkg.Version = constructVersion(pkg.Srcinfo.Pkgver, pkg.Srcinfo.Pkgrel, pkg.Srcinfo.Epoch) dbPkg := getDbPackage(pkg) - dbPkg = dbPkg.Update().SetUpdated(time.Now()).SetVersion(pkg.Version).SaveX(context.Background()) skipping := false if contains(info.Arch, "any") { @@ -236,25 +225,26 @@ func (b *BuildManager) parseWorker() { dbPkg.SkipReason = "blacklisted (haskell)" dbPkg.Status = dbpackage.StatusSkipped skipping = true - } else if isPkgFailed(pkg) { + } else if isPkgFailed(pkg, dbPkg) { log.Debugf("Skipped %s: failed build", info.Pkgbase) - dbPkg.SkipReason = "" - dbPkg.Status = dbpackage.StatusFailed skipping = true } if skipping { - dbPkg = dbPkg.Update().SetStatus(dbPkg.Status).SetSkipReason(dbPkg.SkipReason).SetHash(pkg.Hash).SaveX(context.Background()) + dbPkg.Update().SetUpdated(time.Now()).SetVersion(pkg.Version).SetStatus(dbPkg.Status).SetSkipReason(dbPkg.SkipReason).SetHash(pkg.Hash).ExecX(context.Background()) b.repoPurge[pkg.FullRepo] <- pkg b.parseWG.Done() continue + } else { + dbPkg = dbPkg.Update().SetUpdated(time.Now()).SetVersion(pkg.Version).SaveX(context.Background()) } - repoVer := getVersionFromRepo(pkg) - dbPkg = dbPkg.Update().SetRepoVersion(repoVer).SaveX(context.Background()) - if repoVer != "" && alpm.VerCmp(repoVer, pkg.Version) > 0 { + repoVer, err := getVersionFromRepo(pkg) + if err != nil { + dbPkg = dbPkg.Update().ClearRepoVersion().SaveX(context.Background()) + } else if err == nil && alpm.VerCmp(repoVer, pkg.Version) > 0 { log.Debugf("Skipped %s: Version in repo higher than in PKGBUILD (%s < %s)", info.Pkgbase, pkg.Version, repoVer) - dbPkg = dbPkg.Update().SetStatus(dbpackage.StatusLatest).SetSkipReason("").SetHash(pkg.Hash).SaveX(context.Background()) + dbPkg = dbPkg.Update().SetStatus(dbpackage.StatusLatest).ClearSkipReason().SetHash(pkg.Hash).SaveX(context.Background()) b.parseWG.Done() continue } @@ -423,7 +413,7 @@ func (b *BuildManager) repoWorker(repo string) { } dbPkg := getDbPackage(pkg) - dbPkg = dbPkg.Update().SetStatus(dbpackage.StatusLatest).SetSkipReason("").SetRepoVersion(pkg.Version).SetHash(pkg.Hash).SaveX(context.Background()) + dbPkg = dbPkg.Update().SetStatus(dbpackage.StatusLatest).ClearSkipReason().SetRepoVersion(pkg.Version).SetHash(pkg.Hash).SaveX(context.Background()) cmd = exec.Command("paccache", "-rc", filepath.Join(conf.Basedir.Repo, pkg.FullRepo, "os", conf.Arch), @@ -462,7 +452,7 @@ func (b *BuildManager) repoWorker(repo string) { } dbPkg := getDbPackage(pkg) - dbPkg = dbPkg.Update().SetRepoVersion("").SaveX(context.Background()) + dbPkg = dbPkg.Update().ClearRepoVersion().SaveX(context.Background()) for _, file := range pkg.PkgFiles { check(os.Remove(file)) diff --git a/utils.go b/utils.go index c99afb4..ef208c3 100644 --- a/utils.go +++ b/utils.go @@ -3,7 +3,6 @@ package main import ( "ALHP.go/ent" "ALHP.go/ent/dbpackage" - "bufio" "context" "encoding/hex" "fmt" @@ -17,7 +16,6 @@ import ( "os" "os/exec" "path/filepath" - "sort" "strconv" "strings" "sync" @@ -53,7 +51,6 @@ type BuildManager struct { buildWG sync.WaitGroup parseWG sync.WaitGroup repoWG sync.WaitGroup - failedMutex sync.RWMutex buildProcesses []*os.Process buildProcMutex sync.RWMutex alpmMutex sync.RWMutex @@ -152,15 +149,15 @@ func statusId2string(s dbpackage.Status) (string, string) { } } -func getVersionFromRepo(pkg *BuildPackage) string { +func getVersionFromRepo(pkg *BuildPackage) (string, error) { findPkgFiles(pkg) if len(pkg.PkgFiles) == 0 { - return "" + return "", fmt.Errorf("not found") } fNameSplit := strings.Split(pkg.PkgFiles[0], "-") - return fNameSplit[len(fNameSplit)-3] + "-" + fNameSplit[len(fNameSplit)-2] + return fNameSplit[len(fNameSplit)-3] + "-" + fNameSplit[len(fNameSplit)-2], nil } func gitClean(pkg *BuildPackage) { @@ -367,52 +364,20 @@ func getSVN2GITVersion(pkg *BuildPackage, h *alpm.Handle) (string, error) { return constructVersion(info.Pkgver, info.Pkgrel, info.Epoch), nil } -func isPkgFailed(pkg *BuildPackage) bool { - buildManager.failedMutex.Lock() - defer buildManager.failedMutex.Unlock() - - file, err := os.OpenFile(filepath.Join(conf.Basedir.Repo, pkg.FullRepo+"_failed.txt"), os.O_RDWR|os.O_CREATE|os.O_SYNC, 0664) - check(err) - defer func(file *os.File) { - check(file.Close()) - }(file) - - failed := false - var newContent []string - scanner := bufio.NewScanner(file) - found := false - for scanner.Scan() { - line := scanner.Text() - splitPkg := strings.Split(line, "==") - - if splitPkg[0] == pkg.Pkgbase { - found = true - pkgVer := constructVersion(pkg.Srcinfo.Pkgver, pkg.Srcinfo.Pkgrel, pkg.Srcinfo.Epoch) - - // try to build new versions of previously failed packages - if alpm.VerCmp(splitPkg[1], pkgVer) < 0 { - failed = false - } else { - failed = true - newContent = append(newContent, line+"\n") - } - } else { - newContent = append(newContent, line+"\n") - } - } - check(scanner.Err()) - - if found { - sort.Strings(newContent) - - _, err = file.Seek(0, 0) - check(err) - check(file.Truncate(0)) - _, err = file.WriteString(strings.Join(newContent, "")) - check(err) +func isPkgFailed(pkg *BuildPackage, dbPkg *ent.DbPackage) bool { + if dbPkg.Version == "" { + return false } - return failed + if pkg.Version == "" { + pkg.Version = constructVersion(pkg.Srcinfo.Pkgver, pkg.Srcinfo.Pkgrel, pkg.Srcinfo.Epoch) + } + + if alpm.VerCmp(dbPkg.Version, pkg.Version) < 0 { + return false + } else { + return dbPkg.Status == dbpackage.StatusFailed + } } func genSRCINFO(pkgbuild string) (*srcinfo.Srcinfo, error) {