From 77a5ec6321cfbf7a5daf2a7eebfa61cdd3e41f21 Mon Sep 17 00:00:00 2001 From: Giovanni Harting <539@idlegandalf.com> Date: Fri, 10 Dec 2021 09:26:51 +0100 Subject: [PATCH] some refactoring; remove orphaned packages from db --- main.go | 35 +++++++++---------- utils.go | 103 +++++++++++++++++++++++++++++++++++++++++-------------- 2 files changed, 94 insertions(+), 44 deletions(-) diff --git a/main.go b/main.go index 8c3d216..5ad34e5 100644 --- a/main.go +++ b/main.go @@ -574,9 +574,7 @@ func (b *BuildManager) repoWorker(repo string) { var realPkgs []string for _, filePath := range pkg.PkgFiles { - fNameSplit := strings.Split(filepath.Base(filePath), "-") - pkgname := strings.Join(fNameSplit[:len(fNameSplit)-3], "-") - realPkgs = append(realPkgs, pkgname) + realPkgs = append(realPkgs, Package(filePath).Name()) } b.repoWG.Add(1) @@ -677,36 +675,37 @@ func (b *BuildManager) syncWorker() { return } - sPkgbuild := strings.Split(pkgbuild, "/") - repo := sPkgbuild[len(sPkgbuild)-2] - - if repo == "trunk" || !contains(conf.Repos, strings.Split(repo, "-")[0]) || containsSubStr(repo, conf.Blacklist.Repo) { + mPkgbuild := PKGBUILD(pkgbuild) + if mPkgbuild.FullRepo() == "trunk" || !contains(conf.Repos, mPkgbuild.Repo()) || containsSubStr(mPkgbuild.FullRepo(), conf.Blacklist.Repo) { continue } for _, march := range conf.March { - // compare b3sum of PKGBUILD file to hash in database, only proceed if hash differs - // reduces the amount of PKGBUILDs that need to be parsed with makepkg, which is _really_ slow, significantly - dbPkg, dbErr := db.DbPackage.Query().Where(dbpackage.And( - dbpackage.Pkgbase(sPkgbuild[len(sPkgbuild)-4]), - dbpackage.RepositoryEQ(dbpackage.Repository(strings.Split(repo, "-")[0])), dbpackage.March(march)), + dbPkg, dbErr := db.DbPackage.Query().Where( + dbpackage.And( + dbpackage.Pkgbase(mPkgbuild.PkgBase()), + dbpackage.RepositoryEQ(dbpackage.Repository(mPkgbuild.Repo())), + dbpackage.March(march), + ), ).Only(context.Background()) if dbErr != nil { switch dbErr.(type) { case *ent.NotFoundError: - log.Debugf("[%s/%s] Package not found in database", strings.Split(repo, "-")[0], sPkgbuild[len(sPkgbuild)-4]) + log.Debugf("[%s/%s] Package not found in database", mPkgbuild.Repo(), mPkgbuild.PkgBase()) break default: - log.Errorf("[%s/%s] Problem querying db for package: %v", strings.Split(repo, "-")[0], sPkgbuild[len(sPkgbuild)-4], dbErr) + log.Errorf("[%s/%s] Problem querying db for package: %v", mPkgbuild.Repo(), mPkgbuild.PkgBase(), dbErr) } } + // compare b3sum of PKGBUILD file to hash in database, only proceed if hash differs + // reduces the amount of PKGBUILDs that need to be parsed with makepkg, which is _really_ slow, significantly b3s, err := b3sum(pkgbuild) check(err) if dbPkg != nil && b3s == dbPkg.Hash { - log.Debugf("[%s/%s] Skipped: PKGBUILD hash matches db (%s)", strings.Split(repo, "-")[0], sPkgbuild[len(sPkgbuild)-4], b3s) + log.Debugf("[%s/%s] Skipped: PKGBUILD hash matches db (%s)", mPkgbuild.Repo(), mPkgbuild.PkgBase(), b3s) continue } @@ -714,10 +713,10 @@ func (b *BuildManager) syncWorker() { b.parseWG.Add(1) b.parse <- &BuildPackage{ Pkgbuild: pkgbuild, - Pkgbase: sPkgbuild[len(sPkgbuild)-4], - Repo: dbpackage.Repository(strings.Split(repo, "-")[0]), + Pkgbase: mPkgbuild.PkgBase(), + Repo: dbpackage.Repository(mPkgbuild.Repo()), March: march, - FullRepo: strings.Split(repo, "-")[0] + "-" + march, + FullRepo: mPkgbuild.Repo() + "-" + march, Hash: b3s, } } diff --git a/utils.go b/utils.go index c4b14e4..fa15882 100644 --- a/utils.go +++ b/utils.go @@ -111,7 +111,8 @@ type Conf struct { } type Globs []string -type PKGFile string +type Package string +type PKGBUILD string type MultiplePKGBUILDError struct { error @@ -126,10 +127,44 @@ func check(e error) { } } +func (p PKGBUILD) FullRepo() string { + sPkgbuild := strings.Split(string(p), string(filepath.Separator)) + return sPkgbuild[len(sPkgbuild)-2] +} + +func (p PKGBUILD) Repo() string { + return strings.Split(p.FullRepo(), "-")[0] +} + +func (p PKGBUILD) PkgBase() string { + sPkgbuild := strings.Split(string(p), string(filepath.Separator)) + return sPkgbuild[len(sPkgbuild)-4] +} + func updateLastUpdated() { check(os.WriteFile(filepath.Join(conf.Basedir.Repo, lastUpdate), []byte(strconv.FormatInt(time.Now().Unix(), 10)), 0644)) } +func (path Package) Name() string { + fNameSplit := strings.Split(filepath.Base(string(path)), "-") + return strings.Join(fNameSplit[:len(fNameSplit)-3], "-") +} + +func (path Package) MArch() string { + splitPath := strings.Split(string(path), string(filepath.Separator)) + return strings.Join(strings.Split(splitPath[len(splitPath)-4], "-")[1:], "-") +} + +func (path Package) Repo() dbpackage.Repository { + splitPath := strings.Split(string(path), string(filepath.Separator)) + return dbpackage.Repository(strings.Split(splitPath[len(splitPath)-4], "-")[0]) +} + +func (path Package) FullRepo() string { + splitPath := strings.Split(string(path), string(filepath.Separator)) + return splitPath[len(splitPath)-4] +} + func statusId2string(s dbpackage.Status) string { switch s { case dbpackage.StatusSkipped: @@ -480,10 +515,8 @@ func (p *BuildPackage) SVN2GITVersion(h *alpm.Handle) (string, error) { var fPkgbuilds []string for _, pkgbuild := range pkgBuilds { - sPkgbuild := strings.Split(pkgbuild, string(filepath.Separator)) - repo := sPkgbuild[len(sPkgbuild)-2] - - if repo == "trunk" || containsSubStr(repo, conf.Blacklist.Repo) { + mPkgbuild := PKGBUILD(pkgbuild) + if mPkgbuild.FullRepo() == "trunk" || containsSubStr(mPkgbuild.FullRepo(), conf.Blacklist.Repo) { continue } @@ -613,23 +646,20 @@ func setupChroot() error { return nil } -func (path PKGFile) DBPackage(march string, repo dbpackage.Repository) (*ent.DbPackage, error) { - fNameSplit := strings.Split(filepath.Base(string(path)), "-") - pkgname := strings.Join(fNameSplit[:len(fNameSplit)-3], "-") - +func (path *Package) DBPackage() (*ent.DbPackage, error) { dbPkg, err := db.DbPackage.Query().Where(func(s *sql.Selector) { s.Where( sql.And( - sqljson.ValueContains(dbpackage.FieldPackages, pkgname), - sql.EQ(dbpackage.FieldMarch, march), - sql.EQ(dbpackage.FieldRepository, repo)), + sqljson.ValueContains(dbpackage.FieldPackages, path.Name()), + sql.EQ(dbpackage.FieldMarch, path.MArch()), + sql.EQ(dbpackage.FieldRepository, path.Repo())), ) }).Only(context.Background()) if err != nil { switch err.(type) { case *ent.NotFoundError: - log.Debugf("Not found in database: %s", pkgname) - return nil, fmt.Errorf("package not found in DB: %s", pkgname) + log.Debugf("Not found in database: %s", path.Name()) + return nil, fmt.Errorf("package not found in DB: %s", path.Name()) default: return nil, err } @@ -637,7 +667,7 @@ func (path PKGFile) DBPackage(march string, repo dbpackage.Repository) (*ent.DbP return dbPkg, nil } -func (path PKGFile) isSignatureValid() (bool, error) { +func (path Package) hasValidSignature() (bool, error) { cmd := exec.Command("gpg", "--verify", string(path)+".sig") res, err := cmd.CombinedOutput() log.Debug(string(res)) @@ -661,18 +691,15 @@ func housekeeping(repo string, wg *sync.WaitGroup) error { } for _, path := range packages { - pkgfile := PKGFile(path) - splitPath := strings.Split(path, string(filepath.Separator)) - march := strings.Join(strings.Split(splitPath[len(splitPath)-4], "-")[1:], "-") - mRepo := dbpackage.Repository(strings.Split(splitPath[len(splitPath)-4], "-")[0]) + mPackage := Package(path) - dbPkg, err := pkgfile.DBPackage(march, mRepo) + dbPkg, err := mPackage.DBPackage() if err != nil { log.Infof("[HK/%s] removing orphan %s", repo, filepath.Base(path)) pkg := &BuildPackage{ - FullRepo: splitPath[len(splitPath)-4], + FullRepo: mPackage.FullRepo(), PkgFiles: []string{path}, - March: march, + March: mPackage.MArch(), } buildManager.repoPurge[pkg.FullRepo] <- pkg continue @@ -680,10 +707,10 @@ func housekeeping(repo string, wg *sync.WaitGroup) error { pkg := &BuildPackage{ Pkgbase: dbPkg.Pkgbase, - Repo: mRepo, - FullRepo: splitPath[len(splitPath)-4], + Repo: mPackage.Repo(), + FullRepo: mPackage.FullRepo(), DbPackage: dbPkg, - March: march, + March: mPackage.MArch(), } var upstream string @@ -715,7 +742,7 @@ func housekeeping(repo string, wg *sync.WaitGroup) error { } // check if pkg signature is valid - valid, err := pkgfile.isSignatureValid() + valid, err := mPackage.hasValidSignature() if err != nil { return err } @@ -738,6 +765,30 @@ func housekeeping(repo string, wg *sync.WaitGroup) error { // TODO: check split packages } + // check all dbpackages for existence + dbpackages, err := db.DbPackage.Query().All(context.Background()) + if err != nil { + return err + } + + for _, dbpkg := range dbpackages { + pkg := &BuildPackage{ + Pkgbase: dbpkg.Pkgbase, + Repo: dbpkg.Repository, + March: dbpkg.March, + FullRepo: dbpkg.Repository.String() + "-" + dbpkg.March, + DbPackage: dbpkg, + } + + if !pkg.isAvailable(alpmHandle) { + log.Infof("[HK/%s/%s] not found on mirror, removing", pkg.FullRepo, pkg.Pkgbase) + err = db.DbPackage.DeleteOne(dbpkg).Exec(context.Background()) + if err != nil { + return err + } + } + } + return nil }