switched from textfiles to db for failed package status

main.go

@@ -62,7 +62,7 @@ func (b *BuildManager) buildWorker(id int) {
 			log.Infof("[%s/%s] Build starting", pkg.FullRepo, pkg.Pkgbase)
 
 			dbPkg := getDbPackage(pkg)
-			dbPkg = dbPkg.Update().SetStatus(dbpackage.StatusBuilding).SetBuildTimeStart(time.Now().UTC()).SetSkipReason("").SaveX(context.Background())
+			dbPkg = dbPkg.Update().SetStatus(dbpackage.StatusBuilding).SetBuildTimeStart(time.Now().UTC()).ClearSkipReason().SaveX(context.Background())
 
 			err := importKeys(pkg)
 			if err != nil {
@@ -112,26 +112,16 @@ func (b *BuildManager) buildWorker(id int) {
 
 			if err != nil {
 				if b.exit {
-					gitClean(pkg)
 					b.buildWG.Done()
 					continue
 				}
 
 				log.Warningf("[%s/%s] Build failed, exit code %d", pkg.FullRepo, pkg.Pkgbase, cmd.ProcessState.ExitCode())
 
-				b.failedMutex.Lock()
-				f, err := os.OpenFile(filepath.Join(conf.Basedir.Repo, pkg.FullRepo+"_failed.txt"), os.O_WRONLY|os.O_APPEND|os.O_CREATE|os.O_SYNC, 0644)
-				check(err)
-
-				_, err = f.WriteString(fmt.Sprintf("%s==%s\n", pkg.Pkgbase, constructVersion(pkg.Srcinfo.Pkgver, pkg.Srcinfo.Pkgrel, pkg.Srcinfo.Epoch)))
-				check(err)
-				check(f.Close())
-				b.failedMutex.Unlock()
-
 				check(os.MkdirAll(filepath.Join(conf.Basedir.Repo, "logs"), 0755))
 				check(os.WriteFile(filepath.Join(conf.Basedir.Repo, "logs", pkg.Pkgbase+".log"), out.Bytes(), 0644))
 
-				dbPkg.Update().SetStatus(dbpackage.StatusFailed).SetBuildTimeEnd(time.Now()).SetHash(pkg.Hash).ExecX(context.Background())
+				dbPkg.Update().SetStatus(dbpackage.StatusFailed).ClearSkipReason().SetBuildTimeEnd(time.Now()).SetHash(pkg.Hash).ExecX(context.Background())
 
 				// purge failed package from repo
 				b.repoPurge[pkg.FullRepo] <- pkg
@@ -215,7 +205,6 @@ func (b *BuildManager) parseWorker() {
 			pkg.Version = constructVersion(pkg.Srcinfo.Pkgver, pkg.Srcinfo.Pkgrel, pkg.Srcinfo.Epoch)
 
 			dbPkg := getDbPackage(pkg)
-			dbPkg = dbPkg.Update().SetUpdated(time.Now()).SetVersion(pkg.Version).SaveX(context.Background())
 
 			skipping := false
 			if contains(info.Arch, "any") {
@@ -236,25 +225,26 @@ func (b *BuildManager) parseWorker() {
 				dbPkg.SkipReason = "blacklisted (haskell)"
 				dbPkg.Status = dbpackage.StatusSkipped
 				skipping = true
-			} else if isPkgFailed(pkg) {
+			} else if isPkgFailed(pkg, dbPkg) {
 				log.Debugf("Skipped %s: failed build", info.Pkgbase)
-				dbPkg.SkipReason = ""
-				dbPkg.Status = dbpackage.StatusFailed
 				skipping = true
 			}
 
 			if skipping {
-				dbPkg = dbPkg.Update().SetStatus(dbPkg.Status).SetSkipReason(dbPkg.SkipReason).SetHash(pkg.Hash).SaveX(context.Background())
+				dbPkg.Update().SetUpdated(time.Now()).SetVersion(pkg.Version).SetStatus(dbPkg.Status).SetSkipReason(dbPkg.SkipReason).SetHash(pkg.Hash).ExecX(context.Background())
 				b.repoPurge[pkg.FullRepo] <- pkg
 				b.parseWG.Done()
 				continue
+			} else {
+				dbPkg = dbPkg.Update().SetUpdated(time.Now()).SetVersion(pkg.Version).SaveX(context.Background())
 			}
 
-			repoVer := getVersionFromRepo(pkg)
+			repoVer, err := getVersionFromRepo(pkg)
-			dbPkg = dbPkg.Update().SetRepoVersion(repoVer).SaveX(context.Background())
+			if err != nil {
-			if repoVer != "" && alpm.VerCmp(repoVer, pkg.Version) > 0 {
+				dbPkg = dbPkg.Update().ClearRepoVersion().SaveX(context.Background())
+			} else if err == nil && alpm.VerCmp(repoVer, pkg.Version) > 0 {
 				log.Debugf("Skipped %s: Version in repo higher than in PKGBUILD (%s < %s)", info.Pkgbase, pkg.Version, repoVer)
-				dbPkg = dbPkg.Update().SetStatus(dbpackage.StatusLatest).SetSkipReason("").SetHash(pkg.Hash).SaveX(context.Background())
+				dbPkg = dbPkg.Update().SetStatus(dbpackage.StatusLatest).ClearSkipReason().SetHash(pkg.Hash).SaveX(context.Background())
 				b.parseWG.Done()
 				continue
 			}
@@ -423,7 +413,7 @@ func (b *BuildManager) repoWorker(repo string) {
 			}
 
 			dbPkg := getDbPackage(pkg)
-			dbPkg = dbPkg.Update().SetStatus(dbpackage.StatusLatest).SetSkipReason("").SetRepoVersion(pkg.Version).SetHash(pkg.Hash).SaveX(context.Background())
+			dbPkg = dbPkg.Update().SetStatus(dbpackage.StatusLatest).ClearSkipReason().SetRepoVersion(pkg.Version).SetHash(pkg.Hash).SaveX(context.Background())
 
 			cmd = exec.Command("paccache",
 				"-rc", filepath.Join(conf.Basedir.Repo, pkg.FullRepo, "os", conf.Arch),
@@ -462,7 +452,7 @@ func (b *BuildManager) repoWorker(repo string) {
 			}
 
 			dbPkg := getDbPackage(pkg)
-			dbPkg = dbPkg.Update().SetRepoVersion("").SaveX(context.Background())
+			dbPkg = dbPkg.Update().ClearRepoVersion().SaveX(context.Background())
 
 			for _, file := range pkg.PkgFiles {
 				check(os.Remove(file))

utils.go

@@ -3,7 +3,6 @@ package main
 import (
 	"ALHP.go/ent"
 	"ALHP.go/ent/dbpackage"
-	"bufio"
 	"context"
 	"encoding/hex"
 	"fmt"
@@ -17,7 +16,6 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
-	"sort"
 	"strconv"
 	"strings"
 	"sync"
@@ -53,7 +51,6 @@ type BuildManager struct {
 	buildWG        sync.WaitGroup
 	parseWG        sync.WaitGroup
 	repoWG         sync.WaitGroup
-	failedMutex    sync.RWMutex
 	buildProcesses []*os.Process
 	buildProcMutex sync.RWMutex
 	alpmMutex      sync.RWMutex
@@ -152,15 +149,15 @@ func statusId2string(s dbpackage.Status) (string, string) {
 	}
 }
 
-func getVersionFromRepo(pkg *BuildPackage) string {
+func getVersionFromRepo(pkg *BuildPackage) (string, error) {
 	findPkgFiles(pkg)
 
 	if len(pkg.PkgFiles) == 0 {
-		return ""
+		return "", fmt.Errorf("not found")
 	}
 
 	fNameSplit := strings.Split(pkg.PkgFiles[0], "-")
-	return fNameSplit[len(fNameSplit)-3] + "-" + fNameSplit[len(fNameSplit)-2]
+	return fNameSplit[len(fNameSplit)-3] + "-" + fNameSplit[len(fNameSplit)-2], nil
 }
 
 func gitClean(pkg *BuildPackage) {
@@ -367,52 +364,20 @@ func getSVN2GITVersion(pkg *BuildPackage, h *alpm.Handle) (string, error) {
 	return constructVersion(info.Pkgver, info.Pkgrel, info.Epoch), nil
 }
 
-func isPkgFailed(pkg *BuildPackage) bool {
+func isPkgFailed(pkg *BuildPackage, dbPkg *ent.DbPackage) bool {
-	buildManager.failedMutex.Lock()
+	if dbPkg.Version == "" {
-	defer buildManager.failedMutex.Unlock()
+		return false
-
-	file, err := os.OpenFile(filepath.Join(conf.Basedir.Repo, pkg.FullRepo+"_failed.txt"), os.O_RDWR|os.O_CREATE|os.O_SYNC, 0664)
-	check(err)
-	defer func(file *os.File) {
-		check(file.Close())
-	}(file)
-
-	failed := false
-	var newContent []string
-	scanner := bufio.NewScanner(file)
-	found := false
-	for scanner.Scan() {
-		line := scanner.Text()
-		splitPkg := strings.Split(line, "==")
-
-		if splitPkg[0] == pkg.Pkgbase {
-			found = true
-			pkgVer := constructVersion(pkg.Srcinfo.Pkgver, pkg.Srcinfo.Pkgrel, pkg.Srcinfo.Epoch)
-
-			// try to build new versions of previously failed packages
-			if alpm.VerCmp(splitPkg[1], pkgVer) < 0 {
-				failed = false
-			} else {
-				failed = true
-				newContent = append(newContent, line+"\n")
-			}
-		} else {
-			newContent = append(newContent, line+"\n")
-		}
-	}
-	check(scanner.Err())
-
-	if found {
-		sort.Strings(newContent)
-
-		_, err = file.Seek(0, 0)
-		check(err)
-		check(file.Truncate(0))
-		_, err = file.WriteString(strings.Join(newContent, ""))
-		check(err)
 	}
 
-	return failed
+	if pkg.Version == "" {
+		pkg.Version = constructVersion(pkg.Srcinfo.Pkgver, pkg.Srcinfo.Pkgrel, pkg.Srcinfo.Epoch)
+	}
+
+	if alpm.VerCmp(dbPkg.Version, pkg.Version) < 0 {
+		return false
+	} else {
+		return dbPkg.Status == dbpackage.StatusFailed
+	}
 }
 
 func genSRCINFO(pkgbuild string) (*srcinfo.Srcinfo, error) {