reworked queue building to use the db; multi threading for SRCINFO gen

2022-08-13 21:54:07 +02:00
parent 0f46a95995
commit 87cf1e5631
3 changed files with 96 additions and 52 deletions
--- a/main.go
+++ b/main.go
@@ -21,6 +21,7 @@ import (
 	"os/exec"
 	"os/signal"
 	"path/filepath"
+	"runtime"
 	"strings"
 	"sync"
 	"syscall"
@@ -318,6 +319,80 @@ func (b *BuildManager) repoWorker(repo string) {
 	}
 }

+func (b *BuildManager) refreshSRCINFOs(ctx context.Context, path string) error {
+	pkgBuilds, err := Glob(path)
+	if err != nil {
+		return fmt.Errorf("error scanning for PKGBUILDs: %w", err)
+	}
+
+	step := int(float32(len(pkgBuilds)) / float32(runtime.NumCPU()))
+	cur := 0
+	for i := 0; i < runtime.NumCPU(); i++ {
+		if cur+step > len(pkgBuilds) {
+			step -= cur + step - len(pkgBuilds)
+		}
+
+		go func(pkgBuilds []string) {
+			for _, pkgbuild := range pkgBuilds {
+				mPkgbuild := PKGBUILD(pkgbuild)
+				if mPkgbuild.FullRepo() == "trunk" || !Contains(conf.Repos, mPkgbuild.Repo()) || containsSubStr(mPkgbuild.FullRepo(), conf.Blacklist.Repo) {
+					continue
+				}
+
+				for _, march := range conf.March {
+					dbPkg, dbErr := db.DbPackage.Query().Where(
+						dbpackage.And(
+							dbpackage.Pkgbase(mPkgbuild.PkgBase()),
+							dbpackage.RepositoryEQ(dbpackage.Repository(mPkgbuild.Repo())),
+							dbpackage.March(march),
+						),
+					).Only(context.Background())
+
+					if ent.IsNotFound(dbErr) {
+						log.Debugf("[%s/%s] Package not found in database", mPkgbuild.Repo(), mPkgbuild.PkgBase())
+					} else if err != nil {
+						log.Errorf("[%s/%s] Problem querying db for package: %v", mPkgbuild.Repo(), mPkgbuild.PkgBase(), dbErr)
+					}
+
+					// compare b3sum of PKGBUILD file to hash in database, only proceed if hash differs
+					// reduces the amount of PKGBUILDs that need to be parsed with makepkg, which is _really_ slow, significantly
+					b3s, err := b3sum(pkgbuild)
+					if err != nil {
+						log.Fatalf("Error hashing PKGBUILD: %v", err)
+					}
+
+					if dbPkg != nil && b3s == dbPkg.Hash {
+						log.Debugf("[%s/%s] Skipped: PKGBUILD hash matches db (%s)", mPkgbuild.Repo(), mPkgbuild.PkgBase(), b3s)
+						continue
+					} else if dbPkg != nil && b3s != dbPkg.Hash {
+						log.Debugf("[%s/%s] srcinfo cleared", mPkgbuild.Repo(), mPkgbuild.PkgBase())
+						dbPkg = dbPkg.Update().ClearSrcinfo().SaveX(context.Background())
+					}
+
+					proto := &ProtoPackage{
+						Pkgbuild:  pkgbuild,
+						Pkgbase:   mPkgbuild.PkgBase(),
+						Repo:      dbpackage.Repository(mPkgbuild.Repo()),
+						March:     march,
+						FullRepo:  mPkgbuild.Repo() + "-" + march,
+						Hash:      b3s,
+						DbPackage: dbPkg,
+					}
+
+					_, err = proto.isEligible(ctx)
+					if err != nil {
+						log.Debugf("unknown status of %s: %v", proto.Pkgbase, err)
+					}
+				}
+			}
+		}(pkgBuilds[cur : cur+step])
+
+		cur += step
+	}
+
+	return nil
+}
+
 func (b *BuildManager) syncWorker(ctx context.Context) error {
 	err := os.MkdirAll(filepath.Join(conf.Basedir.Work, upstreamDir), 0755)
 	if err != nil {
@@ -394,7 +469,11 @@ func (b *BuildManager) syncWorker(ctx context.Context) error {
 		b.alpmMutex.Unlock()

 		log.Debugf("generating build-queue for PKGBUILDs found in %s", filepath.Join(conf.Basedir.Work, upstreamDir, "/**/PKGBUILD"))
-		queue, err := b.queue(filepath.Join(conf.Basedir.Work, upstreamDir, "/**/PKGBUILD"))
+		err = b.refreshSRCINFOs(ctx, filepath.Join(conf.Basedir.Work, upstreamDir, "/**/PKGBUILD"))
+		if err != nil {
+			log.Fatalf("error refreshing PKGBUILDs: %v", err)
+		}
+		queue, err := b.queue()
 		if err != nil {
 			log.Warningf("Error building buildQueue: %v", err)
 		} else {
--- a/proto_package.go
+++ b/proto_package.go
@@ -640,6 +640,7 @@ func (p *ProtoPackage) SVN2GITVersion(h *alpm.Handle) (string, error) {
 			return "", MultiplePKGBUILDError{fmt.Errorf("%s: multiple PKGBUILD found: %s", p.Pkgbase, fPkgbuilds)}
 		}
 		log.Infof("%s: resolving successful: MirrorRepo=%s; PKGBUILD chosen: %s", p.Pkgbase, iPackage.DB().Name(), fPkgbuilds[0])
+		p.Repo = dbpackage.Repository(iPackage.DB().Name())
 	} else if len(fPkgbuilds) == 0 {
 		return "", fmt.Errorf("%s: no matching PKGBUILD found (searched: %s, canidates: %s)", p.Pkgbase, filepath.Join(conf.Basedir.Work, upstreamDir, "**/"+p.Pkgbase+"/repos/*/PKGBUILD"), pkgBuilds)
 	}
@@ -795,6 +796,7 @@ func (p *ProtoPackage) isMirrorLatest(h *alpm.Handle) (bool, alpm.IPackage, stri

 		svn2gitVer, err := (&ProtoPackage{
 			Pkgbase: pkg.Base(),
+			March:   p.March,
 		}).SVN2GITVersion(h)
 		if err != nil {
 			return false, nil, "", err
--- a/utils.go
+++ b/utils.go
@@ -185,8 +185,8 @@ func cleanBuildDir(dir string, chrootDir string) error {
 	return nil
 }

-func (b *BuildManager) queue(path string) ([]*ProtoPackage, error) {
-	unsortedQueue, err := genQueue(path)
+func (b *BuildManager) queue() ([]*ProtoPackage, error) {
+	unsortedQueue, err := genQueue()
 	if err != nil {
 		return nil, fmt.Errorf("error building queue: %w", err)
 	}
@@ -218,59 +218,22 @@ func (b *BuildManager) buildQueue(queue []*ProtoPackage, ctx context.Context) er
 	return nil
 }

-func genQueue(path string) ([]*ProtoPackage, error) {
-	pkgBuilds, err := Glob(path)
+func genQueue() ([]*ProtoPackage, error) {
+	pkgs, err := db.DbPackage.Query().Where(dbpackage.StatusEQ(dbpackage.StatusQueued)).All(context.Background())
 	if err != nil {
-		return nil, fmt.Errorf("error scanning for PKGBUILDs: %w", err)
+		return nil, err
 	}

 	var pkgbuilds []*ProtoPackage
-	for _, pkgbuild := range pkgBuilds {
-		mPkgbuild := PKGBUILD(pkgbuild)
-		if mPkgbuild.FullRepo() == "trunk" || !Contains(conf.Repos, mPkgbuild.Repo()) || containsSubStr(mPkgbuild.FullRepo(), conf.Blacklist.Repo) {
-			continue
-		}
-
-		for _, march := range conf.March {
-			dbPkg, dbErr := db.DbPackage.Query().Where(
-				dbpackage.And(
-					dbpackage.Pkgbase(mPkgbuild.PkgBase()),
-					dbpackage.RepositoryEQ(dbpackage.Repository(mPkgbuild.Repo())),
-					dbpackage.March(march),
-				),
-			).Only(context.Background())
-
-			if ent.IsNotFound(dbErr) {
-				log.Debugf("[%s/%s] Package not found in database", mPkgbuild.Repo(), mPkgbuild.PkgBase())
-			} else if err != nil {
-				log.Errorf("[%s/%s] Problem querying db for package: %v", mPkgbuild.Repo(), mPkgbuild.PkgBase(), dbErr)
-			}
-
-			// compare b3sum of PKGBUILD file to hash in database, only proceed if hash differs
-			// reduces the amount of PKGBUILDs that need to be parsed with makepkg, which is _really_ slow, significantly
-			b3s, err := b3sum(pkgbuild)
-			if err != nil {
-				log.Fatalf("Error hashing PKGBUILD: %v", err)
-			}
-
-			if dbPkg != nil && b3s == dbPkg.Hash {
-				log.Debugf("[%s/%s] Skipped: PKGBUILD hash matches db (%s)", mPkgbuild.Repo(), mPkgbuild.PkgBase(), b3s)
-				continue
-			} else if dbPkg != nil && b3s != dbPkg.Hash {
-				log.Debugf("[%s/%s] srcinfo cleared", mPkgbuild.Repo(), mPkgbuild.PkgBase())
-				dbPkg = dbPkg.Update().ClearSrcinfo().SaveX(context.Background())
-			}
-
-			pkgbuilds = append(pkgbuilds, &ProtoPackage{
-				Pkgbuild:  pkgbuild,
-				Pkgbase:   mPkgbuild.PkgBase(),
-				Repo:      dbpackage.Repository(mPkgbuild.Repo()),
-				March:     march,
-				FullRepo:  mPkgbuild.Repo() + "-" + march,
-				Hash:      b3s,
-				DbPackage: dbPkg,
-			})
-		}
+	for _, pkg := range pkgs {
+		pkgbuilds = append(pkgbuilds, &ProtoPackage{
+			Pkgbase:   pkg.Pkgbase,
+			Repo:      pkg.Repository,
+			March:     pkg.March,
+			FullRepo:  pkg.Repository.String() + "-" + pkg.March,
+			Hash:      pkg.Hash,
+			DbPackage: pkg,
+		})
 	}
 	return pkgbuilds, nil
 }