moved repo-add to end of build/sync-cycle

2021-12-20 17:20:06 +01:00
parent f009e9ff00
commit e0e76af757
2 changed files with 92 additions and 24 deletions
--- a/main.go
+++ b/main.go
@@ -228,7 +228,8 @@ func (b *BuildManager) buildWorker(id int, march string) {
 			check(err)

 			for _, file := range copyFiles {
-				_, err = copyFile(file, filepath.Join(conf.Basedir.Repo, pkg.FullRepo, "os", conf.Arch, filepath.Base(file)))
+				check(os.MkdirAll(filepath.Join(conf.Basedir.Work, waitingDir, pkg.FullRepo), 755))
+				_, err = copyFile(file, filepath.Join(conf.Basedir.Work, waitingDir, pkg.FullRepo, filepath.Base(file)))
 				if err != nil {
 					check(err)
 					b.buildWG.Done()
@@ -236,7 +237,7 @@ func (b *BuildManager) buildWorker(id int, march string) {
 				}

 				if filepath.Ext(file) != ".sig" {
-					pkg.PkgFiles = append(pkg.PkgFiles, filepath.Join(conf.Basedir.Repo, pkg.FullRepo, "os", conf.Arch, filepath.Base(file)))
+					pkg.PkgFiles = append(pkg.PkgFiles, filepath.Join(conf.Basedir.Work, waitingDir, pkg.FullRepo, filepath.Base(file)))
 				}
 			}

@@ -245,18 +246,30 @@ func (b *BuildManager) buildWorker(id int, march string) {
 			}

 			if pkg.DbPackage.Lto != dbpackage.LtoDisabled && pkg.DbPackage.Lto != dbpackage.LtoAutoDisabled {
-				pkg.DbPackage.Update().SetStatus(dbpackage.StatusBuild).SetLto(dbpackage.LtoEnabled).SetBuildTimeStart(start).SetLastVersionBuild(pkg.Version).SetBuildTimeEnd(time.Now().UTC()).ExecX(context.Background())
+				pkg.DbPackage.Update().
+					SetStatus(dbpackage.StatusBuild).
+					SetLto(dbpackage.LtoEnabled).
+					SetBuildTimeStart(start).
+					SetLastVersionBuild(pkg.Version).
+					SetBuildTimeEnd(time.Now().UTC()).
+					SetHash(pkg.Hash).
+					ExecX(context.Background())
 			} else {
-				pkg.DbPackage.Update().SetStatus(dbpackage.StatusBuild).SetBuildTimeStart(start).SetBuildTimeEnd(time.Now().UTC()).SetLastVersionBuild(pkg.Version).ExecX(context.Background())
+				pkg.DbPackage.Update().
+					SetStatus(dbpackage.StatusBuild).
+					SetBuildTimeStart(start).
+					SetBuildTimeEnd(time.Now().UTC()).
+					SetLastVersionBuild(pkg.Version).
+					SetHash(pkg.Hash).ExecX(context.Background())
 			}

 			log.Infof("[%s/%s/%s] Build successful (%s)", pkg.FullRepo, pkg.Pkgbase, pkg.Version, time.Since(start))
-			b.repoAdd[pkg.FullRepo] <- []*BuildPackage{pkg}

 			err = cleanBuildDir(buildDir)
 			if err != nil {
 				log.Warningf("[%s/%s/%s] Error removing builddir: %v", pkg.FullRepo, pkg.Pkgbase, pkg.Version, err)
 			}
+			b.buildWG.Done()
 		}
 	}
 }
@@ -542,28 +555,30 @@ func (b *BuildManager) repoWorker(repo string) {
 	for {
 		select {
 		case pkgL := <-b.repoAdd[repo]:
+			toAdd := make([]string, 0)
 			for _, pkg := range pkgL {
-				args := []string{"-s", "-v", "-p", "-n", filepath.Join(conf.Basedir.Repo, pkg.FullRepo, "os", conf.Arch, pkg.FullRepo) + ".db.tar.xz"}
-				args = append(args, pkg.PkgFiles...)
-				cmd := exec.Command("repo-add", args...)
-				res, err := cmd.CombinedOutput()
-				log.Debug(string(res))
-				if err != nil && cmd.ProcessState.ExitCode() != 1 {
-					log.Panicf("%s while repo-add: %v", string(res), err)
-				}
+				toAdd = append(toAdd, pkg.PkgFiles...)
+			}

+			args := []string{"-s", "-v", "-p", "-n", filepath.Join(conf.Basedir.Repo, repo, "os", conf.Arch, repo) + ".db.tar.xz"}
+			args = append(args, toAdd...)
+			cmd := exec.Command("repo-add", args...)
+			res, err := cmd.CombinedOutput()
+			log.Debug(string(res))
+			if err != nil && cmd.ProcessState.ExitCode() != 1 {
+				log.Panicf("%s while repo-add: %v", string(res), err)
+			}
+
+			for _, pkg := range pkgL {
 				pkg.toDbPackage(true)
 				pkg.DbPackage = pkg.DbPackage.Update().SetStatus(dbpackage.StatusLatest).ClearSkipReason().SetRepoVersion(pkg.Version).SetHash(pkg.Hash).SaveX(context.Background())
-
-				cmd = exec.Command("paccache",
-					"-rc", filepath.Join(conf.Basedir.Repo, pkg.FullRepo, "os", conf.Arch),
-					"-k", "1")
-				res, err = cmd.CombinedOutput()
-				log.Debug(string(res))
-				check(err)
-				updateLastUpdated()
-				b.buildWG.Done()
 			}
+
+			cmd = exec.Command("paccache", "-rc", filepath.Join(conf.Basedir.Repo, repo, "os", conf.Arch), "-k", "1")
+			res, err = cmd.CombinedOutput()
+			log.Debug(string(res))
+			check(err)
+			updateLastUpdated()
 		case pkgL := <-b.repoPurge[repo]:
 			for _, pkg := range pkgL {
 				if _, err := os.Stat(filepath.Join(conf.Basedir.Repo, pkg.FullRepo, "os", conf.Arch, pkg.FullRepo) + ".db.tar.xz"); err != nil {
@@ -616,8 +631,6 @@ func (b *BuildManager) syncWorker() {
 	}

 	for {
-		b.buildWG.Wait()
-
 		for gitDir, gitURL := range conf.Svn2git {
 			gitPath := filepath.Join(conf.Basedir.Work, upstreamDir, gitDir)

@@ -729,6 +742,15 @@ func (b *BuildManager) syncWorker() {
 		}

 		b.parseWG.Wait()
+		b.buildWG.Wait()
+
+		for _, repo := range repos {
+			err = movePackagesLive(repo)
+			if err != nil {
+				log.Errorf("[%s] Error moving packages live: %v", repo, err)
+			}
+		}
+
 		time.Sleep(time.Duration(*checkInterval) * time.Minute)
 	}
 }
--- a/utils.go
+++ b/utils.go
@@ -37,6 +37,7 @@ const (
 	upstreamDir    = "upstream"
 	chrootDir      = "chroot"
 	makepkgDir     = "makepkg"
+	waitingDir     = "to_be_moved"
 )

 var (
@@ -169,6 +170,11 @@ func (path Package) FullRepo() string {
 	return splitPath[len(splitPath)-4]
 }

+func (path Package) Version() string {
+	fNameSplit := strings.Split(filepath.Base(string(path)), "-")
+	return strings.Join(fNameSplit[len(fNameSplit)-3:len(fNameSplit)-2], "-")
+}
+
 func statusId2string(s dbpackage.Status) string {
 	switch s {
 	case dbpackage.StatusSkipped:
@@ -408,6 +414,42 @@ func (p *BuildPackage) prepareKernelPatches() error {
 	return nil
 }

+func movePackagesLive(fullRepo string) error {
+	pkgFiles, err := filepath.Glob(filepath.Join(conf.Basedir.Work, waitingDir, fullRepo, "*.pkg.tar.zst"))
+	if err != nil {
+		return err
+	}
+
+	toAdd := make([]*BuildPackage, 0)
+
+	for _, file := range pkgFiles {
+		pkg := Package(file)
+		dbpkg, err := pkg.DBPackage()
+		if err != nil {
+			return err
+		}
+
+		err = os.Rename(file, filepath.Join(conf.Basedir.Repo, fullRepo, "os", conf.Arch, filepath.Base(file)))
+		if err != nil {
+			return err
+		}
+		err = os.Rename(file+".sig", filepath.Join(conf.Basedir.Repo, fullRepo, "os", conf.Arch, filepath.Base(file)+".sig"))
+		if err != nil {
+			return err
+		}
+
+		toAdd = append(toAdd, &BuildPackage{
+			DbPackage: dbpkg,
+			Pkgbase:   dbpkg.Pkgbase,
+			PkgFiles:  []string{filepath.Join(conf.Basedir.Repo, fullRepo, "os", conf.Arch, filepath.Base(file))},
+			Version:   pkg.Version(),
+		})
+	}
+
+	buildManager.repoAdd[fullRepo] <- toAdd
+	return nil
+}
+
 func packages2slice(pkgs interface{}) []string {
 	switch v := pkgs.(type) {
 	case []srcinfo.Package:
@@ -828,6 +870,10 @@ func (p *BuildPackage) findPkgFiles() error {
 }

 func (p *BuildPackage) toDbPackage(create bool) {
+	if p.DbPackage != nil {
+		return
+	}
+
 	dbPkg, err := db.DbPackage.Query().Where(dbpackage.And(dbpackage.Pkgbase(p.Pkgbase), dbpackage.March(p.March), dbpackage.RepositoryEQ(p.Repo))).Only(context.Background())
 	if err != nil && create {
 		dbPkg = db.DbPackage.Create().SetPkgbase(p.Pkgbase).SetMarch(p.March).SetPackages(packages2slice(p.Srcinfo.Packages)).SetRepository(p.Repo).SaveX(context.Background())