SunRed/ALHP.GO
1
0
Fork 0
forked from ALHP/ALHP.GO
Code Issues Pull Requests Packages Projects Releases Wiki Activity

added automatic LTO fail detection for some cases based on build output

Browse Source
This commit is contained in:
Giovanni Harting
2021-11-25 01:50:49 +01:00
parent d5e9832b33
commit 743b08bba1
6 changed files with 48 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
ent/dbpackage/dbpackage.go
View File

@@ -140,9 +140,10 @@ const DefaultLto = LtoUnknown
// Lto values. // Lto values.
const ( const (
LtoEnabled Lto = "enabled" LtoEnabled Lto = "enabled"
LtoUnknown Lto = "unknown" LtoUnknown Lto = "unknown"
LtoDisabled Lto = "disabled" LtoDisabled Lto = "disabled"
LtoAutoDisabled Lto = "auto_disabled"
) )
func (l Lto) String() string { func (l Lto) String() string {
@@ -152,7 +153,7 @@ func (l Lto) String() string {
// LtoValidator is a validator for the "lto" field enum values. It is called by the builders before save. // LtoValidator is a validator for the "lto" field enum values. It is called by the builders before save.
func LtoValidator(l Lto) error { func LtoValidator(l Lto) error {
switch l { switch l {
case LtoEnabled, LtoUnknown, LtoDisabled: case LtoEnabled, LtoUnknown, LtoDisabled, LtoAutoDisabled:
return nil return nil
default: default:
return fmt.Errorf("dbpackage: invalid enum value for lto field: %q", l) return fmt.Errorf("dbpackage: invalid enum value for lto field: %q", l)

2
ent/migrate/schema.go
View File

@@ -23,7 +23,7 @@ var (
{Name: "build_time_end", Type: field.TypeTime, Nullable: true}, {Name: "build_time_end", Type: field.TypeTime, Nullable: true},
{Name: "updated", Type: field.TypeTime, Nullable: true}, {Name: "updated", Type: field.TypeTime, Nullable: true},
{Name: "hash", Type: field.TypeString, Nullable: true}, {Name: "hash", Type: field.TypeString, Nullable: true},
{Name: "lto", Type: field.TypeEnum, Nullable: true, Enums: []string{"enabled", "unknown", "disabled"}, Default: "unknown"}, {Name: "lto", Type: field.TypeEnum, Nullable: true, Enums: []string{"enabled", "unknown", "disabled", "auto_disabled"}, Default: "unknown"},
} }
// DbPackagesTable holds the schema information for the "db_packages" table. // DbPackagesTable holds the schema information for the "db_packages" table.
DbPackagesTable = &schema.Table{ DbPackagesTable = &schema.Table{

2
ent/schema/dbpackage.go
View File

@@ -25,7 +25,7 @@ func (DbPackage) Fields() []ent.Field {
field.Time("build_time_end").Optional(), field.Time("build_time_end").Optional(),
field.Time("updated").Optional(), field.Time("updated").Optional(),
field.String("hash").Optional(), field.String("hash").Optional(),
field.Enum("lto").Values("enabled", "unknown", "disabled").Default("unknown").Optional(), field.Enum("lto").Values("enabled", "unknown", "disabled", "auto_disabled").Default("unknown").Optional(),
} }
} }

56
main.go
View File

@@ -61,7 +61,7 @@ func (b *BuildManager) buildWorker(id int) {
pkg.toDbPackage(true) pkg.toDbPackage(true)
pkg.DbPackage = pkg.DbPackage.Update().SetStatus(dbpackage.StatusBuilding).ClearSkipReason().SaveX(context.Background()) pkg.DbPackage = pkg.DbPackage.Update().SetStatus(dbpackage.StatusBuilding).ClearSkipReason().SaveX(context.Background())
err := importKeys(pkg) err := pkg.importKeys()
if err != nil { if err != nil {
log.Warningf("[%s/%s] Failed to import pgp keys: %v", pkg.FullRepo, pkg.Pkgbase, err) log.Warningf("[%s/%s] Failed to import pgp keys: %v", pkg.FullRepo, pkg.Pkgbase, err)
} }
@@ -84,15 +84,12 @@ func (b *BuildManager) buildWorker(id int) {
} }
pkg.PkgFiles = []string{} pkg.PkgFiles = []string{}
ltoDisabled := false
// default to LTO // default to LTO
makepkgFile := "makepkg-%s-lto.conf" makepkgFile := "makepkg-%s-lto.conf"
if contains(conf.Blacklist.LTO, pkg.Pkgbase) { if pkg.DbPackage.Lto == dbpackage.LtoDisabled || pkg.DbPackage.Lto == dbpackage.LtoAutoDisabled {
// use non-lto makepkg.conf if LTO is blacklisted for this package // use non-lto makepkg.conf if LTO is blacklisted for this package
makepkgFile = "makepkg-%s.conf" makepkgFile = "makepkg-%s.conf"
ltoDisabled = true
pkg.DbPackage.Update().SetLto(dbpackage.LtoDisabled).ExecX(context.Background())
} }
cmd := exec.Command("sh", "-c", cmd := exec.Command("sh", "-c",
"cd "+filepath.Dir(pkg.Pkgbuild)+"&&makechrootpkg -c -D "+conf.Basedir.Makepkg+" -l worker-"+strconv.Itoa(id)+" -r "+conf.Basedir.Chroot+" -- "+ "cd "+filepath.Dir(pkg.Pkgbuild)+"&&makechrootpkg -c -D "+conf.Basedir.Makepkg+" -l worker-"+strconv.Itoa(id)+" -r "+conf.Basedir.Chroot+" -- "+
@@ -124,6 +121,14 @@ func (b *BuildManager) buildWorker(id int) {
continue continue
} }
if pkg.DbPackage.Lto != dbpackage.LtoAutoDisabled && pkg.DbPackage.Lto != dbpackage.LtoDisabled && reLdError.Match(out.Bytes()) {
log.Infof("[%s/%s] ld error detected, disabling LTO", pkg.FullRepo, pkg.Pkgbase)
pkg.DbPackage.Update().SetStatus(dbpackage.StatusQueued).SetSkipReason("non-LTO rebuild").SetLto(dbpackage.LtoAutoDisabled).ExecX(context.Background())
gitClean(pkg)
b.buildWG.Done()
continue
}
log.Warningf("[%s/%s] Build failed (%d)", pkg.FullRepo, pkg.Pkgbase, cmd.ProcessState.ExitCode()) log.Warningf("[%s/%s] Build failed (%d)", pkg.FullRepo, pkg.Pkgbase, cmd.ProcessState.ExitCode())
check(os.MkdirAll(filepath.Join(conf.Basedir.Repo, "logs"), 0755)) check(os.MkdirAll(filepath.Join(conf.Basedir.Repo, "logs"), 0755))
@@ -181,10 +186,10 @@ func (b *BuildManager) buildWorker(id int) {
check(os.Remove(filepath.Join(conf.Basedir.Repo, "logs", pkg.Pkgbase+".log"))) check(os.Remove(filepath.Join(conf.Basedir.Repo, "logs", pkg.Pkgbase+".log")))
} }
if !ltoDisabled { if pkg.DbPackage.Lto != dbpackage.LtoDisabled && pkg.DbPackage.Lto != dbpackage.LtoAutoDisabled {
pkg.DbPackage.Update().SetStatus(dbpackage.StatusBuild).SetLto(dbpackage.LtoEnabled).SetBuildTimeStart(start).SetBuildTimeEnd(time.Now().UTC()).ExecX(context.Background()) pkg.DbPackage.Update().SetStatus(dbpackage.StatusBuild).SetLto(dbpackage.LtoEnabled).SetBuildTimeStart(start).SetBuildTimeEnd(time.Now().UTC()).ExecX(context.Background())
} else { } else {
pkg.DbPackage.Update().SetStatus(dbpackage.StatusBuild).SetLto(dbpackage.LtoDisabled).SetBuildTimeStart(start).SetBuildTimeEnd(time.Now().UTC()).ExecX(context.Background()) pkg.DbPackage.Update().SetStatus(dbpackage.StatusBuild).SetBuildTimeStart(start).SetBuildTimeEnd(time.Now().UTC()).ExecX(context.Background())
} }
log.Infof("[%s/%s] Build successful (%s)", pkg.FullRepo, pkg.Pkgbase, time.Now().Sub(start)) log.Infof("[%s/%s] Build successful (%s)", pkg.FullRepo, pkg.Pkgbase, time.Now().Sub(start))
@@ -241,9 +246,9 @@ func (b *BuildManager) parseWorker() {
} }
if skipping { if skipping {
pkg.DbPackage.Update().SetUpdated(time.Now()).SetVersion(pkg.Version). pkg.DbPackage = pkg.DbPackage.Update().SetUpdated(time.Now()).SetVersion(pkg.Version).
SetPackages(packages2slice(pkg.Srcinfo.Packages)).SetStatus(pkg.DbPackage.Status). SetPackages(packages2slice(pkg.Srcinfo.Packages)).SetStatus(pkg.DbPackage.Status).
SetSkipReason(pkg.DbPackage.SkipReason).SetHash(pkg.Hash).ExecX(context.Background()) SetSkipReason(pkg.DbPackage.SkipReason).SetHash(pkg.Hash).SaveX(context.Background())
b.repoPurge[pkg.FullRepo] <- pkg b.repoPurge[pkg.FullRepo] <- pkg
b.parseWG.Done() b.parseWG.Done()
continue continue
@@ -251,6 +256,10 @@ func (b *BuildManager) parseWorker() {
pkg.DbPackage = pkg.DbPackage.Update().SetUpdated(time.Now()).SetPackages(packages2slice(pkg.Srcinfo.Packages)).SetVersion(pkg.Version).SaveX(context.Background()) pkg.DbPackage = pkg.DbPackage.Update().SetUpdated(time.Now()).SetPackages(packages2slice(pkg.Srcinfo.Packages)).SetVersion(pkg.Version).SaveX(context.Background())
} }
if contains(conf.Blacklist.LTO, pkg.Pkgbase) {
pkg.DbPackage = pkg.DbPackage.Update().SetLto(dbpackage.LtoDisabled).SaveX(context.Background())
}
repoVer, err := pkg.repoVersion() repoVer, err := pkg.repoVersion()
if err != nil { if err != nil {
pkg.DbPackage = pkg.DbPackage.Update().ClearRepoVersion().SaveX(context.Background()) pkg.DbPackage = pkg.DbPackage.Update().ClearRepoVersion().SaveX(context.Background())
@@ -309,19 +318,20 @@ func (b *BuildManager) parseWorker() {
func (b *BuildManager) htmlWorker() { func (b *BuildManager) htmlWorker() {
type Pkg struct { type Pkg struct {
Pkgbase string Pkgbase string
Status string Status string
Class string Class string
Skip string Skip string
Version string Version string
Svn2GitVersion string Svn2GitVersion string
BuildDate string BuildDate string
BuildDuration time.Duration BuildDuration time.Duration
Checked string Checked string
Log string Log string
LTO bool LTO bool
LTOUnknown bool LTOUnknown bool
LTODisabled bool LTODisabled bool
LTOAutoDisabled bool
} }
type Repo struct { type Repo struct {
@@ -394,6 +404,8 @@ func (b *BuildManager) htmlWorker() {
addPkg.LTO = true addPkg.LTO = true
case dbpackage.LtoDisabled: case dbpackage.LtoDisabled:
addPkg.LTODisabled = true addPkg.LTODisabled = true
case dbpackage.LtoAutoDisabled:
addPkg.LTOAutoDisabled = true
} }
addRepo.Packages = append(addRepo.Packages, addPkg) addRepo.Packages = append(addRepo.Packages, addPkg)

3
tpl/packages.html
View File

@@ -82,6 +82,9 @@
title="build with LTO"></i>{{end}} title="build with LTO"></i>{{end}}
{{if $pkg.LTODisabled}}<i class="fa fa-times" style="color: var(--bs-danger)" {{if $pkg.LTODisabled}}<i class="fa fa-times" style="color: var(--bs-danger)"
title="LTO explicitly disabled"></i>{{end}} title="LTO explicitly disabled"></i>{{end}}
{{if $pkg.LTOAutoDisabled}}<i class="fa fa-times-circle-o"
style="color: var(--bs-danger)"
title="LTO automatically disabled"></i>{{end}}
{{if $pkg.LTOUnknown}}<i class="fa fa-hourglass-o" title="not build with LTO yet"></i>{{end}} {{if $pkg.LTOUnknown}}<i class="fa fa-hourglass-o" title="not build with LTO yet"></i>{{end}}
</td> </td>
<td>{{$pkg.Svn2GitVersion}}</td> <td>{{$pkg.Svn2GitVersion}}</td>

7
utils.go
View File

@@ -39,6 +39,7 @@ var (
rePkgSource = regexp.MustCompile(`(?msU)^source.*=.*\((.+)\)$`) rePkgSource = regexp.MustCompile(`(?msU)^source.*=.*\((.+)\)$`)
rePkgSum = regexp.MustCompile(`(?msU)^sha256sums.*=.*\((.+)\)$`) rePkgSum = regexp.MustCompile(`(?msU)^sha256sums.*=.*\((.+)\)$`)
rePkgFile = regexp.MustCompile(`^(.+)(?:-.+){2}-(?:x86_64|any)\.pkg\.tar\.zst(?:\.sig)*$`) rePkgFile = regexp.MustCompile(`^(.+)(?:-.+){2}-(?:x86_64|any)\.pkg\.tar\.zst(?:\.sig)*$`)
reLdError = regexp.MustCompile(`(?mi)^\s*collect2: error: ld returned (\d+) exit status$`)
) )
type BuildPackage struct { type BuildPackage struct {
@@ -354,10 +355,10 @@ func packages2slice(pkgs interface{}) []string {
} }
} }
func importKeys(pkg *BuildPackage) error { func (p *BuildPackage) importKeys() error {
if pkg.Srcinfo.ValidPGPKeys != nil { if p.Srcinfo.ValidPGPKeys != nil {
args := []string{"--keyserver", "keyserver.ubuntu.com", "--recv-keys"} args := []string{"--keyserver", "keyserver.ubuntu.com", "--recv-keys"}
args = append(args, pkg.Srcinfo.ValidPGPKeys...) args = append(args, p.Srcinfo.ValidPGPKeys...)
cmd := exec.Command("gpg", args...) cmd := exec.Command("gpg", args...)
_, err := cmd.CombinedOutput() _, err := cmd.CombinedOutput()