ALHP/ALHP.GO
6
43
Fork 9
Code Issues 25 Pull Requests Projects Releases Wiki Activity

add database backend (sqlite only for now)

Browse Source
This commit is contained in:
Giovanni Harting
2021-07-11 20:02:12 +02:00
parent 9c8366372f
commit f14f9b706b
26 changed files with 6768 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

171
main.go
View File

@@ -1,11 +1,15 @@
package main
import (
"ALHP.go/ent"
"ALHP.go/ent/dbpackage"
"bufio"
"bytes"
"context"
"fmt"
"github.com/Jguer/go-alpm/v2"
"github.com/Morganamilo/go-srcinfo"
_ "github.com/mattn/go-sqlite3"
log "github.com/sirupsen/logrus"
"github.com/wercker/journalhook"
"github.com/yargevad/filepathx"
@@ -33,6 +37,15 @@ const (
orgChrootName = "root"
)
const (
UNKNOWN = iota
FAILED = iota
BUILD = iota
QUEUED = iota
BUILDING = iota
LATEST = iota
)
var (
conf = Conf{}
repos []string
@@ -40,6 +53,8 @@ var (
rePkgRel = regexp.MustCompile(`(?m)^pkgrel\s*=\s*(.+)$`)
rePkgFile = regexp.MustCompile(`^(.*)-.*-.*-(?:x86_64|any)\.pkg\.tar\.zst(?:\.sig)*$`)
buildManager BuildManager
db *ent.Client
dbLock sync.RWMutex
)
type BuildPackage struct {
@@ -64,10 +79,6 @@ type BuildManager struct {
failedMutex sync.RWMutex
buildProcesses []*os.Process
buildProcMutex sync.RWMutex
stats struct {
fullyBuild int
eligible int
}
}
type Conf struct {
@@ -75,7 +86,7 @@ type Conf struct {
Repos, March, Blacklist []string
Svn2git map[string]string
Basedir struct {
Repo, Chroot, Makepkg, Upstream string
Repo, Chroot, Makepkg, Upstream, Db string
}
Build struct {
Worker int
@@ -225,6 +236,15 @@ func importKeys(pkg *BuildPackage) {
}
}
func packages2string(pkgs []srcinfo.Package) []string {
var sPkgs []string
for _, p := range pkgs {
sPkgs = append(sPkgs, p.Pkgname)
}
return sPkgs
}
func increasePkgRel(pkg *BuildPackage) {
f, err := os.OpenFile(pkg.Pkgbuild, os.O_RDWR, os.ModePerm)
check(err)
@@ -275,6 +295,11 @@ func (b *BuildManager) buildWorker(id int) {
log.Infof("[%s/%s] Build starting", pkg.FullRepo, pkg.Pkgbase)
dbPkg := getDbPackage(pkg)
dbLock.Lock()
dbPkg.Update().SetStatus(BUILDING).SaveX(context.Background())
dbLock.Unlock()
importKeys(pkg)
increasePkgRel(pkg)
pkg.PkgFiles = []string{}
@@ -292,7 +317,7 @@ func (b *BuildManager) buildWorker(id int) {
b.buildProcesses = append(b.buildProcesses, cmd.Process)
b.buildProcMutex.Unlock()
err := cmd.Wait()
err = cmd.Wait()
b.buildProcMutex.Lock()
for i := range b.buildProcesses {
@@ -329,6 +354,11 @@ func (b *BuildManager) buildWorker(id int) {
check(os.MkdirAll(filepath.Join(conf.Basedir.Repo, "logs"), os.ModePerm))
check(os.WriteFile(filepath.Join(conf.Basedir.Repo, "logs", pkg.Pkgbase+".log"), out.Bytes(), os.ModePerm))
dbPkg := getDbPackage(pkg)
dbLock.Lock()
dbPkg.Update().SetStatus(FAILED).SetBuildTime(time.Now()).SetBuildDuration(uint64(time.Now().Sub(start).Milliseconds())).SaveX(context.Background())
dbLock.Unlock()
gitClean(pkg)
b.buildWG.Done()
continue
@@ -372,18 +402,35 @@ func (b *BuildManager) buildWorker(id int) {
pkg.PkgFiles = append(pkg.PkgFiles, filepath.Join(conf.Basedir.Repo, pkg.FullRepo, "os", conf.Arch, filepath.Base(file)))
}
}
b.repoAdd[pkg.FullRepo] <- pkg
if _, err := os.Stat(filepath.Join(conf.Basedir.Repo, "logs", pkg.Pkgbase+".log")); err == nil {
check(os.Remove(filepath.Join(conf.Basedir.Repo, "logs", pkg.Pkgbase+".log")))
}
gitClean(pkg)
dbPkg = getDbPackage(pkg)
dbLock.Lock()
dbPkg.Update().SetStatus(BUILD).SetBuildTime(time.Now()).SetBuildDuration(uint64(time.Now().Sub(start).Milliseconds())).SaveX(context.Background())
dbLock.Unlock()
log.Infof("[%s/%s] Build successful (%s)", pkg.FullRepo, pkg.Pkgbase, time.Now().Sub(start))
b.repoAdd[pkg.FullRepo] <- pkg
gitClean(pkg)
}
}
}
func getDbPackage(pkg *BuildPackage) *ent.DbPackage {
dbLock.Lock()
dbPkg, err := db.DbPackage.Query().Where(dbpackage.Pkgbase(pkg.Pkgbase)).Only(context.Background())
if err != nil {
dbPkg = db.DbPackage.Create().SetPkgbase(pkg.Pkgbase).SetMarch(pkg.March).SetPackages(packages2string(pkg.Srcinfo.Packages)).SetRepository(pkg.Repo).SaveX(context.Background())
}
dbLock.Unlock()
return dbPkg
}
func (b *BuildManager) parseWorker() {
for {
if b.exit {
@@ -406,31 +453,6 @@ func (b *BuildManager) parseWorker() {
continue
}
pkg.Srcinfo = info
if contains(info.Arch, "any") || contains(conf.Blacklist, info.Pkgbase) {
log.Infof("Skipped %s: blacklisted or any-Package", info.Pkgbase)
b.repoPurge[pkg.FullRepo] <- pkg
b.parseWG.Done()
continue
}
// Skip Haskell packages for now, as we are facing linking problems with them,
// most likely caused by not having a dependency tree implemented yet and building at random.
// https://git.harting.dev/anonfunc/ALHP.GO/issues/11
if contains(info.MakeDepends, "ghc") || contains(info.MakeDepends, "haskell-ghc") || contains(info.Depends, "ghc") || contains(info.Depends, "haskell-ghc") {
log.Infof("Skipped %s: haskell package", info.Pkgbase)
b.repoPurge[pkg.FullRepo] <- pkg
b.parseWG.Done()
continue
}
if isPkgFailed(pkg) {
log.Infof("Skipped %s: failed build", info.Pkgbase)
b.repoPurge[pkg.FullRepo] <- pkg
b.parseWG.Done()
continue
}
var pkgVer string
if pkg.Srcinfo.Epoch == "" {
pkgVer = pkg.Srcinfo.Pkgver + "-" + pkg.Srcinfo.Pkgrel
@@ -438,17 +460,64 @@ func (b *BuildManager) parseWorker() {
pkgVer = pkg.Srcinfo.Epoch + ":" + pkg.Srcinfo.Pkgver + "-" + pkg.Srcinfo.Pkgrel
}
repoVer := getVersionFromRepo(pkg)
if repoVer != "" && alpm.VerCmp(repoVer, pkgVer) > 0 {
log.Debugf("Skipped %s: Version in repo higher than in PKGBUILD (%s < %s)", info.Pkgbase, pkgVer, repoVer)
b.stats.eligible++
b.stats.fullyBuild++
dbPkg := getDbPackage(pkg)
dbLock.Lock()
dbPkg = dbPkg.Update().SetUpdated(time.Now()).SetVersion(pkgVer).SaveX(context.Background())
dbLock.Unlock()
skipping := false
if contains(info.Arch, "any") {
log.Infof("Skipped %s: any-Package", info.Pkgbase)
dbLock.Lock()
dbPkg = dbPkg.Update().SetSkipReason("arch = any").SaveX(context.Background())
dbLock.Unlock()
skipping = true
} else if contains(conf.Blacklist, info.Pkgbase) {
log.Infof("Skipped %s: blacklisted package", info.Pkgbase)
dbLock.Lock()
dbPkg = dbPkg.Update().SetSkipReason("blacklisted").SaveX(context.Background())
dbLock.Unlock()
skipping = true
} else if contains(info.MakeDepends, "ghc") || contains(info.MakeDepends, "haskell-ghc") || contains(info.Depends, "ghc") || contains(info.Depends, "haskell-ghc") {
// Skip Haskell packages for now, as we are facing linking problems with them,
// most likely caused by not having a dependency tree implemented yet and building at random.
// https://git.harting.dev/anonfunc/ALHP.GO/issues/11
log.Infof("Skipped %s: haskell package", info.Pkgbase)
dbLock.Lock()
dbPkg = dbPkg.Update().SetSkipReason("blacklisted (haskell)").SaveX(context.Background())
dbLock.Unlock()
skipping = true
} else if isPkgFailed(pkg) {
log.Infof("Skipped %s: failed build", info.Pkgbase)
dbLock.Lock()
dbPkg = dbPkg.Update().SetStatus(FAILED).SetSkipReason("failed").SaveX(context.Background())
dbLock.Unlock()
skipping = true
}
if skipping {
b.repoPurge[pkg.FullRepo] <- pkg
b.parseWG.Done()
continue
}
b.stats.eligible++
b.parseWG.Done()
repoVer := getVersionFromRepo(pkg)
dbLock.Lock()
dbPkg = dbPkg.Update().SetRepoVersion(repoVer).SaveX(context.Background())
dbLock.Unlock()
if repoVer != "" && alpm.VerCmp(repoVer, pkgVer) > 0 {
log.Debugf("Skipped %s: Version in repo higher than in PKGBUILD (%s < %s)", info.Pkgbase, pkgVer, repoVer)
dbLock.Lock()
dbPkg = dbPkg.Update().SetStatus(LATEST).SetSkipReason("").SaveX(context.Background())
dbLock.Unlock()
b.parseWG.Done()
continue
}
dbLock.Lock()
dbPkg = dbPkg.Update().SetStatus(QUEUED).SaveX(context.Background())
dbLock.Unlock()
b.build <- pkg
}
}
@@ -570,6 +639,11 @@ func (b *BuildManager) repoWorker(repo string) {
log.Panicf("%s while repo-add: %v", string(res), err)
}
dbPkg := getDbPackage(pkg)
dbLock.Lock()
dbPkg = dbPkg.Update().SetStatus(LATEST).SetSkipReason("").SaveX(context.Background())
dbLock.Unlock()
cmd = exec.Command("paccache",
"-rc", filepath.Join(conf.Basedir.Repo, pkg.FullRepo, "os", conf.Arch),
"-k", "1")
@@ -687,11 +761,6 @@ func (b *BuildManager) syncWorker() {
}
b.parseWG.Wait()
if b.stats.eligible != 0 {
log.Infof("Processed source-repos. %d packages elegible to be build, %d already fully build. Covering %f%% of offical-repo (buildable) packages.", b.stats.eligible, b.stats.fullyBuild, float32(b.stats.fullyBuild)/float32(b.stats.eligible)*100.0)
}
b.stats.fullyBuild = 0
b.stats.eligible = 0
time.Sleep(5 * time.Minute)
}
}
@@ -716,6 +785,18 @@ func main() {
log.Warningf("Failed to drop priority: %v", err)
}
db, err = ent.Open("sqlite3", "file:"+conf.Basedir.Db+"?_fk=1&cache=shared")
if err != nil {
log.Panicf("Failed to open database %s: %v", conf.Basedir.Db, err)
}
defer func(dbSQLite *ent.Client) {
check(dbSQLite.Close())
}(db)
if err := db.Schema.Create(context.Background()); err != nil {
log.Panicf("Automigrate failed: %v", err)
}
err = os.MkdirAll(conf.Basedir.Repo, os.ModePerm)
check(err)