some refactoring; remove orphaned packages from db

main.go

@@ -574,9 +574,7 @@ func (b *BuildManager) repoWorker(repo string) {
 
 			var realPkgs []string
 			for _, filePath := range pkg.PkgFiles {
-				fNameSplit := strings.Split(filepath.Base(filePath), "-")
+				realPkgs = append(realPkgs, Package(filePath).Name())
-				pkgname := strings.Join(fNameSplit[:len(fNameSplit)-3], "-")
-				realPkgs = append(realPkgs, pkgname)
 			}
 
 			b.repoWG.Add(1)
@@ -677,36 +675,37 @@ func (b *BuildManager) syncWorker() {
 				return
 			}
 
-			sPkgbuild := strings.Split(pkgbuild, "/")
+			mPkgbuild := PKGBUILD(pkgbuild)
-			repo := sPkgbuild[len(sPkgbuild)-2]
+			if mPkgbuild.FullRepo() == "trunk" || !contains(conf.Repos, mPkgbuild.Repo()) || containsSubStr(mPkgbuild.FullRepo(), conf.Blacklist.Repo) {
-
-			if repo == "trunk" || !contains(conf.Repos, strings.Split(repo, "-")[0]) || containsSubStr(repo, conf.Blacklist.Repo) {
 				continue
 			}
 
 			for _, march := range conf.March {
-				// compare b3sum of PKGBUILD file to hash in database, only proceed if hash differs
+				dbPkg, dbErr := db.DbPackage.Query().Where(
-				// reduces the amount of PKGBUILDs that need to be parsed with makepkg, which is _really_ slow, significantly
+					dbpackage.And(
-				dbPkg, dbErr := db.DbPackage.Query().Where(dbpackage.And(
+						dbpackage.Pkgbase(mPkgbuild.PkgBase()),
-					dbpackage.Pkgbase(sPkgbuild[len(sPkgbuild)-4]),
+						dbpackage.RepositoryEQ(dbpackage.Repository(mPkgbuild.Repo())),
-					dbpackage.RepositoryEQ(dbpackage.Repository(strings.Split(repo, "-")[0])), dbpackage.March(march)),
+						dbpackage.March(march),
+					),
 				).Only(context.Background())
 
 				if dbErr != nil {
 					switch dbErr.(type) {
 					case *ent.NotFoundError:
-						log.Debugf("[%s/%s] Package not found in database", strings.Split(repo, "-")[0], sPkgbuild[len(sPkgbuild)-4])
+						log.Debugf("[%s/%s] Package not found in database", mPkgbuild.Repo(), mPkgbuild.PkgBase())
 						break
 					default:
-						log.Errorf("[%s/%s] Problem querying db for package: %v", strings.Split(repo, "-")[0], sPkgbuild[len(sPkgbuild)-4], dbErr)
+						log.Errorf("[%s/%s] Problem querying db for package: %v", mPkgbuild.Repo(), mPkgbuild.PkgBase(), dbErr)
 					}
 				}
 
+				// compare b3sum of PKGBUILD file to hash in database, only proceed if hash differs
+				// reduces the amount of PKGBUILDs that need to be parsed with makepkg, which is _really_ slow, significantly
 				b3s, err := b3sum(pkgbuild)
 				check(err)
 
 				if dbPkg != nil && b3s == dbPkg.Hash {
-					log.Debugf("[%s/%s] Skipped: PKGBUILD hash matches db (%s)", strings.Split(repo, "-")[0], sPkgbuild[len(sPkgbuild)-4], b3s)
+					log.Debugf("[%s/%s] Skipped: PKGBUILD hash matches db (%s)", mPkgbuild.Repo(), mPkgbuild.PkgBase(), b3s)
 					continue
 				}
 
@@ -714,10 +713,10 @@ func (b *BuildManager) syncWorker() {
 				b.parseWG.Add(1)
 				b.parse <- &BuildPackage{
 					Pkgbuild: pkgbuild,
-					Pkgbase:  sPkgbuild[len(sPkgbuild)-4],
+					Pkgbase:  mPkgbuild.PkgBase(),
-					Repo:     dbpackage.Repository(strings.Split(repo, "-")[0]),
+					Repo:     dbpackage.Repository(mPkgbuild.Repo()),
 					March:    march,
-					FullRepo: strings.Split(repo, "-")[0] + "-" + march,
+					FullRepo: mPkgbuild.Repo() + "-" + march,
 					Hash:     b3s,
 				}
 			}

utils.go

@@ -111,7 +111,8 @@ type Conf struct {
 }
 
 type Globs []string
-type PKGFile string
+type Package string
+type PKGBUILD string
 
 type MultiplePKGBUILDError struct {
 	error
@@ -126,10 +127,44 @@ func check(e error) {
 	}
 }
 
+func (p PKGBUILD) FullRepo() string {
+	sPkgbuild := strings.Split(string(p), string(filepath.Separator))
+	return sPkgbuild[len(sPkgbuild)-2]
+}
+
+func (p PKGBUILD) Repo() string {
+	return strings.Split(p.FullRepo(), "-")[0]
+}
+
+func (p PKGBUILD) PkgBase() string {
+	sPkgbuild := strings.Split(string(p), string(filepath.Separator))
+	return sPkgbuild[len(sPkgbuild)-4]
+}
+
 func updateLastUpdated() {
 	check(os.WriteFile(filepath.Join(conf.Basedir.Repo, lastUpdate), []byte(strconv.FormatInt(time.Now().Unix(), 10)), 0644))
 }
 
+func (path Package) Name() string {
+	fNameSplit := strings.Split(filepath.Base(string(path)), "-")
+	return strings.Join(fNameSplit[:len(fNameSplit)-3], "-")
+}
+
+func (path Package) MArch() string {
+	splitPath := strings.Split(string(path), string(filepath.Separator))
+	return strings.Join(strings.Split(splitPath[len(splitPath)-4], "-")[1:], "-")
+}
+
+func (path Package) Repo() dbpackage.Repository {
+	splitPath := strings.Split(string(path), string(filepath.Separator))
+	return dbpackage.Repository(strings.Split(splitPath[len(splitPath)-4], "-")[0])
+}
+
+func (path Package) FullRepo() string {
+	splitPath := strings.Split(string(path), string(filepath.Separator))
+	return splitPath[len(splitPath)-4]
+}
+
 func statusId2string(s dbpackage.Status) string {
 	switch s {
 	case dbpackage.StatusSkipped:
@@ -480,10 +515,8 @@ func (p *BuildPackage) SVN2GITVersion(h *alpm.Handle) (string, error) {
 
 	var fPkgbuilds []string
 	for _, pkgbuild := range pkgBuilds {
-		sPkgbuild := strings.Split(pkgbuild, string(filepath.Separator))
+		mPkgbuild := PKGBUILD(pkgbuild)
-		repo := sPkgbuild[len(sPkgbuild)-2]
+		if mPkgbuild.FullRepo() == "trunk" || containsSubStr(mPkgbuild.FullRepo(), conf.Blacklist.Repo) {
-
-		if repo == "trunk" || containsSubStr(repo, conf.Blacklist.Repo) {
 			continue
 		}
 
@@ -613,23 +646,20 @@ func setupChroot() error {
 	return nil
 }
 
-func (path PKGFile) DBPackage(march string, repo dbpackage.Repository) (*ent.DbPackage, error) {
+func (path *Package) DBPackage() (*ent.DbPackage, error) {
-	fNameSplit := strings.Split(filepath.Base(string(path)), "-")
-	pkgname := strings.Join(fNameSplit[:len(fNameSplit)-3], "-")
-
 	dbPkg, err := db.DbPackage.Query().Where(func(s *sql.Selector) {
 		s.Where(
 			sql.And(
-				sqljson.ValueContains(dbpackage.FieldPackages, pkgname),
+				sqljson.ValueContains(dbpackage.FieldPackages, path.Name()),
-				sql.EQ(dbpackage.FieldMarch, march),
+				sql.EQ(dbpackage.FieldMarch, path.MArch()),
-				sql.EQ(dbpackage.FieldRepository, repo)),
+				sql.EQ(dbpackage.FieldRepository, path.Repo())),
 		)
 	}).Only(context.Background())
 	if err != nil {
 		switch err.(type) {
 		case *ent.NotFoundError:
-			log.Debugf("Not found in database: %s", pkgname)
+			log.Debugf("Not found in database: %s", path.Name())
-			return nil, fmt.Errorf("package not found in DB: %s", pkgname)
+			return nil, fmt.Errorf("package not found in DB: %s", path.Name())
 		default:
 			return nil, err
 		}
@@ -637,7 +667,7 @@ func (path PKGFile) DBPackage(march string, repo dbpackage.Repository) (*ent.DbP
 	return dbPkg, nil
 }
 
-func (path PKGFile) isSignatureValid() (bool, error) {
+func (path Package) hasValidSignature() (bool, error) {
 	cmd := exec.Command("gpg", "--verify", string(path)+".sig")
 	res, err := cmd.CombinedOutput()
 	log.Debug(string(res))
@@ -661,18 +691,15 @@ func housekeeping(repo string, wg *sync.WaitGroup) error {
 	}
 
 	for _, path := range packages {
-		pkgfile := PKGFile(path)
+		mPackage := Package(path)
-		splitPath := strings.Split(path, string(filepath.Separator))
-		march := strings.Join(strings.Split(splitPath[len(splitPath)-4], "-")[1:], "-")
-		mRepo := dbpackage.Repository(strings.Split(splitPath[len(splitPath)-4], "-")[0])
 
-		dbPkg, err := pkgfile.DBPackage(march, mRepo)
+		dbPkg, err := mPackage.DBPackage()
 		if err != nil {
 			log.Infof("[HK/%s] removing orphan %s", repo, filepath.Base(path))
 			pkg := &BuildPackage{
-				FullRepo: splitPath[len(splitPath)-4],
+				FullRepo: mPackage.FullRepo(),
 				PkgFiles: []string{path},
-				March:    march,
+				March:    mPackage.MArch(),
 			}
 			buildManager.repoPurge[pkg.FullRepo] <- pkg
 			continue
@@ -680,10 +707,10 @@ func housekeeping(repo string, wg *sync.WaitGroup) error {
 
 		pkg := &BuildPackage{
 			Pkgbase:   dbPkg.Pkgbase,
-			Repo:      mRepo,
+			Repo:      mPackage.Repo(),
-			FullRepo:  splitPath[len(splitPath)-4],
+			FullRepo:  mPackage.FullRepo(),
 			DbPackage: dbPkg,
-			March:     march,
+			March:     mPackage.MArch(),
 		}
 
 		var upstream string
@@ -715,7 +742,7 @@ func housekeeping(repo string, wg *sync.WaitGroup) error {
 		}
 
 		// check if pkg signature is valid
-		valid, err := pkgfile.isSignatureValid()
+		valid, err := mPackage.hasValidSignature()
 		if err != nil {
 			return err
 		}
@@ -738,6 +765,30 @@ func housekeeping(repo string, wg *sync.WaitGroup) error {
 		// TODO: check split packages
 	}
 
+	// check all dbpackages for existence
+	dbpackages, err := db.DbPackage.Query().All(context.Background())
+	if err != nil {
+		return err
+	}
+
+	for _, dbpkg := range dbpackages {
+		pkg := &BuildPackage{
+			Pkgbase:   dbpkg.Pkgbase,
+			Repo:      dbpkg.Repository,
+			March:     dbpkg.March,
+			FullRepo:  dbpkg.Repository.String() + "-" + dbpkg.March,
+			DbPackage: dbpkg,
+		}
+
+		if !pkg.isAvailable(alpmHandle) {
+			log.Infof("[HK/%s/%s] not found on mirror, removing", pkg.FullRepo, pkg.Pkgbase)
+			err = db.DbPackage.DeleteOne(dbpkg).Exec(context.Background())
+			if err != nil {
+				return err
+			}
+		}
+	}
+
 	return nil
 }