diff --git a/main.go b/main.go index 5050237..35afe61 100644 --- a/main.go +++ b/main.go @@ -228,7 +228,8 @@ func (b *BuildManager) buildWorker(id int, march string) { check(err) for _, file := range copyFiles { - _, err = copyFile(file, filepath.Join(conf.Basedir.Repo, pkg.FullRepo, "os", conf.Arch, filepath.Base(file))) + check(os.MkdirAll(filepath.Join(conf.Basedir.Work, waitingDir, pkg.FullRepo), 755)) + _, err = copyFile(file, filepath.Join(conf.Basedir.Work, waitingDir, pkg.FullRepo, filepath.Base(file))) if err != nil { check(err) b.buildWG.Done() @@ -236,7 +237,7 @@ func (b *BuildManager) buildWorker(id int, march string) { } if filepath.Ext(file) != ".sig" { - pkg.PkgFiles = append(pkg.PkgFiles, filepath.Join(conf.Basedir.Repo, pkg.FullRepo, "os", conf.Arch, filepath.Base(file))) + pkg.PkgFiles = append(pkg.PkgFiles, filepath.Join(conf.Basedir.Work, waitingDir, pkg.FullRepo, filepath.Base(file))) } } @@ -245,18 +246,30 @@ func (b *BuildManager) buildWorker(id int, march string) { } if pkg.DbPackage.Lto != dbpackage.LtoDisabled && pkg.DbPackage.Lto != dbpackage.LtoAutoDisabled { - pkg.DbPackage.Update().SetStatus(dbpackage.StatusBuild).SetLto(dbpackage.LtoEnabled).SetBuildTimeStart(start).SetLastVersionBuild(pkg.Version).SetBuildTimeEnd(time.Now().UTC()).ExecX(context.Background()) + pkg.DbPackage.Update(). + SetStatus(dbpackage.StatusBuild). + SetLto(dbpackage.LtoEnabled). + SetBuildTimeStart(start). + SetLastVersionBuild(pkg.Version). + SetBuildTimeEnd(time.Now().UTC()). + SetHash(pkg.Hash). + ExecX(context.Background()) } else { - pkg.DbPackage.Update().SetStatus(dbpackage.StatusBuild).SetBuildTimeStart(start).SetBuildTimeEnd(time.Now().UTC()).SetLastVersionBuild(pkg.Version).ExecX(context.Background()) + pkg.DbPackage.Update(). + SetStatus(dbpackage.StatusBuild). + SetBuildTimeStart(start). + SetBuildTimeEnd(time.Now().UTC()). + SetLastVersionBuild(pkg.Version). + SetHash(pkg.Hash).ExecX(context.Background()) } log.Infof("[%s/%s/%s] Build successful (%s)", pkg.FullRepo, pkg.Pkgbase, pkg.Version, time.Since(start)) - b.repoAdd[pkg.FullRepo] <- []*BuildPackage{pkg} err = cleanBuildDir(buildDir) if err != nil { log.Warningf("[%s/%s/%s] Error removing builddir: %v", pkg.FullRepo, pkg.Pkgbase, pkg.Version, err) } + b.buildWG.Done() } } } @@ -542,28 +555,30 @@ func (b *BuildManager) repoWorker(repo string) { for { select { case pkgL := <-b.repoAdd[repo]: + toAdd := make([]string, 0) for _, pkg := range pkgL { - args := []string{"-s", "-v", "-p", "-n", filepath.Join(conf.Basedir.Repo, pkg.FullRepo, "os", conf.Arch, pkg.FullRepo) + ".db.tar.xz"} - args = append(args, pkg.PkgFiles...) - cmd := exec.Command("repo-add", args...) - res, err := cmd.CombinedOutput() - log.Debug(string(res)) - if err != nil && cmd.ProcessState.ExitCode() != 1 { - log.Panicf("%s while repo-add: %v", string(res), err) - } + toAdd = append(toAdd, pkg.PkgFiles...) + } + args := []string{"-s", "-v", "-p", "-n", filepath.Join(conf.Basedir.Repo, repo, "os", conf.Arch, repo) + ".db.tar.xz"} + args = append(args, toAdd...) + cmd := exec.Command("repo-add", args...) + res, err := cmd.CombinedOutput() + log.Debug(string(res)) + if err != nil && cmd.ProcessState.ExitCode() != 1 { + log.Panicf("%s while repo-add: %v", string(res), err) + } + + for _, pkg := range pkgL { pkg.toDbPackage(true) pkg.DbPackage = pkg.DbPackage.Update().SetStatus(dbpackage.StatusLatest).ClearSkipReason().SetRepoVersion(pkg.Version).SetHash(pkg.Hash).SaveX(context.Background()) - - cmd = exec.Command("paccache", - "-rc", filepath.Join(conf.Basedir.Repo, pkg.FullRepo, "os", conf.Arch), - "-k", "1") - res, err = cmd.CombinedOutput() - log.Debug(string(res)) - check(err) - updateLastUpdated() - b.buildWG.Done() } + + cmd = exec.Command("paccache", "-rc", filepath.Join(conf.Basedir.Repo, repo, "os", conf.Arch), "-k", "1") + res, err = cmd.CombinedOutput() + log.Debug(string(res)) + check(err) + updateLastUpdated() case pkgL := <-b.repoPurge[repo]: for _, pkg := range pkgL { if _, err := os.Stat(filepath.Join(conf.Basedir.Repo, pkg.FullRepo, "os", conf.Arch, pkg.FullRepo) + ".db.tar.xz"); err != nil { @@ -616,8 +631,6 @@ func (b *BuildManager) syncWorker() { } for { - b.buildWG.Wait() - for gitDir, gitURL := range conf.Svn2git { gitPath := filepath.Join(conf.Basedir.Work, upstreamDir, gitDir) @@ -729,6 +742,15 @@ func (b *BuildManager) syncWorker() { } b.parseWG.Wait() + b.buildWG.Wait() + + for _, repo := range repos { + err = movePackagesLive(repo) + if err != nil { + log.Errorf("[%s] Error moving packages live: %v", repo, err) + } + } + time.Sleep(time.Duration(*checkInterval) * time.Minute) } } diff --git a/utils.go b/utils.go index 5b936c1..bd9ae1d 100644 --- a/utils.go +++ b/utils.go @@ -37,6 +37,7 @@ const ( upstreamDir = "upstream" chrootDir = "chroot" makepkgDir = "makepkg" + waitingDir = "to_be_moved" ) var ( @@ -169,6 +170,11 @@ func (path Package) FullRepo() string { return splitPath[len(splitPath)-4] } +func (path Package) Version() string { + fNameSplit := strings.Split(filepath.Base(string(path)), "-") + return strings.Join(fNameSplit[len(fNameSplit)-3:len(fNameSplit)-2], "-") +} + func statusId2string(s dbpackage.Status) string { switch s { case dbpackage.StatusSkipped: @@ -408,6 +414,42 @@ func (p *BuildPackage) prepareKernelPatches() error { return nil } +func movePackagesLive(fullRepo string) error { + pkgFiles, err := filepath.Glob(filepath.Join(conf.Basedir.Work, waitingDir, fullRepo, "*.pkg.tar.zst")) + if err != nil { + return err + } + + toAdd := make([]*BuildPackage, 0) + + for _, file := range pkgFiles { + pkg := Package(file) + dbpkg, err := pkg.DBPackage() + if err != nil { + return err + } + + err = os.Rename(file, filepath.Join(conf.Basedir.Repo, fullRepo, "os", conf.Arch, filepath.Base(file))) + if err != nil { + return err + } + err = os.Rename(file+".sig", filepath.Join(conf.Basedir.Repo, fullRepo, "os", conf.Arch, filepath.Base(file)+".sig")) + if err != nil { + return err + } + + toAdd = append(toAdd, &BuildPackage{ + DbPackage: dbpkg, + Pkgbase: dbpkg.Pkgbase, + PkgFiles: []string{filepath.Join(conf.Basedir.Repo, fullRepo, "os", conf.Arch, filepath.Base(file))}, + Version: pkg.Version(), + }) + } + + buildManager.repoAdd[fullRepo] <- toAdd + return nil +} + func packages2slice(pkgs interface{}) []string { switch v := pkgs.(type) { case []srcinfo.Package: @@ -828,6 +870,10 @@ func (p *BuildPackage) findPkgFiles() error { } func (p *BuildPackage) toDbPackage(create bool) { + if p.DbPackage != nil { + return + } + dbPkg, err := db.DbPackage.Query().Where(dbpackage.And(dbpackage.Pkgbase(p.Pkgbase), dbpackage.March(p.March), dbpackage.RepositoryEQ(p.Repo))).Only(context.Background()) if err != nil && create { dbPkg = db.DbPackage.Create().SetPkgbase(p.Pkgbase).SetMarch(p.March).SetPackages(packages2slice(p.Srcinfo.Packages)).SetRepository(p.Repo).SaveX(context.Background())