added automatic LTO fail detection for some cases based on build output

ent/dbpackage/dbpackage.go

@@ -143,6 +143,7 @@ const (
 	LtoEnabled      Lto = "enabled"
 	LtoUnknown      Lto = "unknown"
 	LtoDisabled     Lto = "disabled"
+	LtoAutoDisabled Lto = "auto_disabled"
 )
 
 func (l Lto) String() string {
@@ -152,7 +153,7 @@ func (l Lto) String() string {
 // LtoValidator is a validator for the "lto" field enum values. It is called by the builders before save.
 func LtoValidator(l Lto) error {
 	switch l {
-	case LtoEnabled, LtoUnknown, LtoDisabled:
+	case LtoEnabled, LtoUnknown, LtoDisabled, LtoAutoDisabled:
 		return nil
 	default:
 		return fmt.Errorf("dbpackage: invalid enum value for lto field: %q", l)

ent/migrate/schema.go

@@ -23,7 +23,7 @@ var (
 		{Name: "build_time_end", Type: field.TypeTime, Nullable: true},
 		{Name: "updated", Type: field.TypeTime, Nullable: true},
 		{Name: "hash", Type: field.TypeString, Nullable: true},
-		{Name: "lto", Type: field.TypeEnum, Nullable: true, Enums: []string{"enabled", "unknown", "disabled"}, Default: "unknown"},
+		{Name: "lto", Type: field.TypeEnum, Nullable: true, Enums: []string{"enabled", "unknown", "disabled", "auto_disabled"}, Default: "unknown"},
 	}
 	// DbPackagesTable holds the schema information for the "db_packages" table.
 	DbPackagesTable = &schema.Table{

ent/schema/dbpackage.go

@@ -25,7 +25,7 @@ func (DbPackage) Fields() []ent.Field {
 		field.Time("build_time_end").Optional(),
 		field.Time("updated").Optional(),
 		field.String("hash").Optional(),
-		field.Enum("lto").Values("enabled", "unknown", "disabled").Default("unknown").Optional(),
+		field.Enum("lto").Values("enabled", "unknown", "disabled", "auto_disabled").Default("unknown").Optional(),
 	}
 }
 

main.go

@@ -61,7 +61,7 @@ func (b *BuildManager) buildWorker(id int) {
 			pkg.toDbPackage(true)
 			pkg.DbPackage = pkg.DbPackage.Update().SetStatus(dbpackage.StatusBuilding).ClearSkipReason().SaveX(context.Background())
 
-			err := importKeys(pkg)
+			err := pkg.importKeys()
 			if err != nil {
 				log.Warningf("[%s/%s] Failed to import pgp keys: %v", pkg.FullRepo, pkg.Pkgbase, err)
 			}
@@ -84,15 +84,12 @@ func (b *BuildManager) buildWorker(id int) {
 			}
 
 			pkg.PkgFiles = []string{}
-			ltoDisabled := false
 
 			// default to LTO
 			makepkgFile := "makepkg-%s-lto.conf"
-			if contains(conf.Blacklist.LTO, pkg.Pkgbase) {
+			if pkg.DbPackage.Lto == dbpackage.LtoDisabled || pkg.DbPackage.Lto == dbpackage.LtoAutoDisabled {
 				// use non-lto makepkg.conf if LTO is blacklisted for this package
 				makepkgFile = "makepkg-%s.conf"
-				ltoDisabled = true
-				pkg.DbPackage.Update().SetLto(dbpackage.LtoDisabled).ExecX(context.Background())
 			}
 			cmd := exec.Command("sh", "-c",
 				"cd "+filepath.Dir(pkg.Pkgbuild)+"&&makechrootpkg -c -D "+conf.Basedir.Makepkg+" -l worker-"+strconv.Itoa(id)+" -r "+conf.Basedir.Chroot+" -- "+
@@ -124,6 +121,14 @@ func (b *BuildManager) buildWorker(id int) {
 					continue
 				}
 
+				if pkg.DbPackage.Lto != dbpackage.LtoAutoDisabled && pkg.DbPackage.Lto != dbpackage.LtoDisabled && reLdError.Match(out.Bytes()) {
+					log.Infof("[%s/%s] ld error detected, disabling LTO", pkg.FullRepo, pkg.Pkgbase)
+					pkg.DbPackage.Update().SetStatus(dbpackage.StatusQueued).SetSkipReason("non-LTO rebuild").SetLto(dbpackage.LtoAutoDisabled).ExecX(context.Background())
+					gitClean(pkg)
+					b.buildWG.Done()
+					continue
+				}
+
 				log.Warningf("[%s/%s] Build failed (%d)", pkg.FullRepo, pkg.Pkgbase, cmd.ProcessState.ExitCode())
 
 				check(os.MkdirAll(filepath.Join(conf.Basedir.Repo, "logs"), 0755))
@@ -181,10 +186,10 @@ func (b *BuildManager) buildWorker(id int) {
 				check(os.Remove(filepath.Join(conf.Basedir.Repo, "logs", pkg.Pkgbase+".log")))
 			}
 
-			if !ltoDisabled {
+			if pkg.DbPackage.Lto != dbpackage.LtoDisabled && pkg.DbPackage.Lto != dbpackage.LtoAutoDisabled {
 				pkg.DbPackage.Update().SetStatus(dbpackage.StatusBuild).SetLto(dbpackage.LtoEnabled).SetBuildTimeStart(start).SetBuildTimeEnd(time.Now().UTC()).ExecX(context.Background())
 			} else {
-				pkg.DbPackage.Update().SetStatus(dbpackage.StatusBuild).SetLto(dbpackage.LtoDisabled).SetBuildTimeStart(start).SetBuildTimeEnd(time.Now().UTC()).ExecX(context.Background())
+				pkg.DbPackage.Update().SetStatus(dbpackage.StatusBuild).SetBuildTimeStart(start).SetBuildTimeEnd(time.Now().UTC()).ExecX(context.Background())
 			}
 
 			log.Infof("[%s/%s] Build successful (%s)", pkg.FullRepo, pkg.Pkgbase, time.Now().Sub(start))
@@ -241,9 +246,9 @@ func (b *BuildManager) parseWorker() {
 			}
 
 			if skipping {
-				pkg.DbPackage.Update().SetUpdated(time.Now()).SetVersion(pkg.Version).
+				pkg.DbPackage = pkg.DbPackage.Update().SetUpdated(time.Now()).SetVersion(pkg.Version).
 					SetPackages(packages2slice(pkg.Srcinfo.Packages)).SetStatus(pkg.DbPackage.Status).
-					SetSkipReason(pkg.DbPackage.SkipReason).SetHash(pkg.Hash).ExecX(context.Background())
+					SetSkipReason(pkg.DbPackage.SkipReason).SetHash(pkg.Hash).SaveX(context.Background())
 				b.repoPurge[pkg.FullRepo] <- pkg
 				b.parseWG.Done()
 				continue
@@ -251,6 +256,10 @@ func (b *BuildManager) parseWorker() {
 				pkg.DbPackage = pkg.DbPackage.Update().SetUpdated(time.Now()).SetPackages(packages2slice(pkg.Srcinfo.Packages)).SetVersion(pkg.Version).SaveX(context.Background())
 			}
 
+			if contains(conf.Blacklist.LTO, pkg.Pkgbase) {
+				pkg.DbPackage = pkg.DbPackage.Update().SetLto(dbpackage.LtoDisabled).SaveX(context.Background())
+			}
+
 			repoVer, err := pkg.repoVersion()
 			if err != nil {
 				pkg.DbPackage = pkg.DbPackage.Update().ClearRepoVersion().SaveX(context.Background())
@@ -322,6 +331,7 @@ func (b *BuildManager) htmlWorker() {
 		LTO             bool
 		LTOUnknown      bool
 		LTODisabled     bool
+		LTOAutoDisabled bool
 	}
 
 	type Repo struct {
@@ -394,6 +404,8 @@ func (b *BuildManager) htmlWorker() {
 						addPkg.LTO = true
 					case dbpackage.LtoDisabled:
 						addPkg.LTODisabled = true
+					case dbpackage.LtoAutoDisabled:
+						addPkg.LTOAutoDisabled = true
 					}
 
 					addRepo.Packages = append(addRepo.Packages, addPkg)

tpl/packages.html

@@ -82,6 +82,9 @@
                                                   title="build with LTO"></i>{{end}}
                                 {{if $pkg.LTODisabled}}<i class="fa fa-times" style="color: var(--bs-danger)"
                                                           title="LTO explicitly disabled"></i>{{end}}
+                                {{if $pkg.LTOAutoDisabled}}<i class="fa fa-times-circle-o"
+                                                              style="color: var(--bs-danger)"
+                                                              title="LTO automatically disabled"></i>{{end}}
                                 {{if $pkg.LTOUnknown}}<i class="fa fa-hourglass-o" title="not build with LTO yet"></i>{{end}}
                             </td>
                             <td>{{$pkg.Svn2GitVersion}}</td>

utils.go

@@ -39,6 +39,7 @@ var (
 	rePkgSource = regexp.MustCompile(`(?msU)^source.*=.*\((.+)\)$`)
 	rePkgSum    = regexp.MustCompile(`(?msU)^sha256sums.*=.*\((.+)\)$`)
 	rePkgFile   = regexp.MustCompile(`^(.+)(?:-.+){2}-(?:x86_64|any)\.pkg\.tar\.zst(?:\.sig)*$`)
+	reLdError   = regexp.MustCompile(`(?mi)^\s*collect2: error: ld returned (\d+) exit status$`)
 )
 
 type BuildPackage struct {
@@ -354,10 +355,10 @@ func packages2slice(pkgs interface{}) []string {
 	}
 }
 
-func importKeys(pkg *BuildPackage) error {
+func (p *BuildPackage) importKeys() error {
-	if pkg.Srcinfo.ValidPGPKeys != nil {
+	if p.Srcinfo.ValidPGPKeys != nil {
 		args := []string{"--keyserver", "keyserver.ubuntu.com", "--recv-keys"}
-		args = append(args, pkg.Srcinfo.ValidPGPKeys...)
+		args = append(args, p.Srcinfo.ValidPGPKeys...)
 		cmd := exec.Command("gpg", args...)
 		_, err := cmd.CombinedOutput()