From 74fbc994398c8e4bbf42c1aafe33e9411250cb0a Mon Sep 17 00:00:00 2001 From: Giovanni Harting <539@idlegandalf.com> Date: Thu, 10 Jun 2021 21:32:11 +0200 Subject: [PATCH] inital commit --- .gitignore | 155 +++++++++++++ README.md | 74 +++++++ config.yaml | 31 +++ go.mod | 11 + go.sum | 20 ++ main.go | 626 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 917 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 config.yaml create mode 100644 go.mod create mode 100644 go.sum create mode 100644 main.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6d9dc3d --- /dev/null +++ b/.gitignore @@ -0,0 +1,155 @@ +# Created by https://www.toptal.com/developers/gitignore/api/go,linux,intellij+all,windows +# Edit at https://www.toptal.com/developers/gitignore?templates=go,linux,intellij+all,windows + +### Go ### +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Dependency directories (remove the comment below to include it) +# vendor/ + +### Go Patch ### +/vendor/ +/Godeps/ + +### Intellij+all ### +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/**/usage.statistics.xml +.idea/**/dictionaries +.idea/**/shelf + +# Generated files +.idea/**/contentModel.xml + +# Sensitive or high-churn files +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/dbnavigator.xml + +# Gradle +.idea/**/gradle.xml +.idea/**/libraries + +# Gradle and Maven with auto-import +# When using Gradle or Maven with auto-import, you should exclude module files, +# since they will be recreated, and may cause churn. Uncomment if using +# auto-import. +# .idea/artifacts +# .idea/compiler.xml +# .idea/jarRepositories.xml +# .idea/modules.xml +# .idea/*.iml +# .idea/modules +# *.iml +# *.ipr + +# CMake +cmake-build-*/ + +# Mongo Explorer plugin +.idea/**/mongoSettings.xml + +# File-based project format +*.iws + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +# Editor-based Rest Client +.idea/httpRequests + +# Android studio 3.1+ serialized cache file +.idea/caches/build_file_checksums.ser + +### Intellij+all Patch ### +# Ignores the whole .idea folder and all .iml files +# See https://github.com/joeblau/gitignore.io/issues/186 and https://github.com/joeblau/gitignore.io/issues/360 + +.idea/ + +# Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-249601023 + +*.iml +modules.xml +.idea/misc.xml +*.ipr + +# Sonarlint plugin +.idea/sonarlint + +### Linux ### +*~ + +# temporary files which can be created if a process still has a handle open of a deleted file +.fuse_hidden* + +# KDE directory preferences +.directory + +# Linux trash folder which might appear on any partition or disk +.Trash-* + +# .nfs files are created when an open file is removed but is still being accessed +.nfs* + +### Windows ### +# Windows thumbnail cache files +Thumbs.db +Thumbs.db:encryptable +ehthumbs.db +ehthumbs_vista.db + +# Dump file +*.stackdump + +# Folder config file +[Dd]esktop.ini + +# Recycle Bin used on file shares +$RECYCLE.BIN/ + +# Windows Installer files +*.cab +*.msi +*.msix +*.msm +*.msp + +# Windows shortcuts +*.lnk + +# End of https://www.toptal.com/developers/gitignore/api/go,linux,intellij+all,windows diff --git a/README.md b/README.md new file mode 100644 index 0000000..d322674 --- /dev/null +++ b/README.md @@ -0,0 +1,74 @@ +# alhp + +Build script for archlinux instructionset enabled repos. +All packages are build with -march= and -O3. Some packages will not build with -O3, they will just be provided from the official repos as usual. + +## Check your system for support + +**Important**: Before you enable any of these repos, check if your system supports x86-64-v3. You can do that with `/lib/ld-linux-x86-64.so.2 --help`. If you don't check beforehand you might be unable to boot your system anymore and need to downgrade any package that you may have upgraded. + +Example output snippet for a system supporting up to `x86-64-v3`: + +``` +Subdirectories of glibc-hwcaps directories, in priority order: + x86-64-v4 + x86-64-v3 (supported, searched) + x86-64-v2 (supported, searched) +``` + +## Enable Repos + +To enable these complement repos you need to add them above the regular repos in `/etc/pacman.conf` + +### Example pacman.conf + +```editorconfig +[core-x86-64-v3] +Server = https://alhp.harting.dev/$repo/os/$arch/ + +[extra-x86-64-v3] +Server = https://alhp.harting.dev/$repo/os/$arch/ + +[community-x86-64-v3] +Server = https://alhp.harting.dev/$repo/os/$arch/ + +[core] +Include = /etc/pacman.d/mirrorlist + +[extra] +Include = /etc/pacman.d/mirrorlist + +[community] +Include = /etc/pacman.d/mirrorlist +``` + +Replace `x86-64-v3` with your cpu-set. More information about all available options on [this gcc page](https://gcc.gnu.org/onlinedocs/gcc/x86-Options.html). +Currently, alhp.harting.dev only builds for `x86-64-v3` (list is subject to change). +You can see all available repositories [here](https://alhp.harting.dev/). + +After finished adding the repos to `pacman.conf` you need to import and sign the used pgp key: + +Import: +``` +pacman-key --keyserver keyserver.ubuntu.com --recv-keys 0D4D2FDAF45468F3DDF59BEDE3D0D2CD3952E298 +``` + +Local sign: +``` +pacman-key --lsign-key 0D4D2FDAF45468F3DDF59BEDE3D0D2CD3952E298 +``` + +Update package database: +``` +pacman -Sy +``` + +## Replace packages +Following command reinstalls all packages found in the repo **extra-x86-64-v3** that are already installed. +Replace `extra-x86-64-v3` with whatever repo you want to install. + +```shell script +pacman -S $(pacman -Sl x86-64-v3 | grep installed | cut -f 2 -d " " | perl -pe 's/\R/ /g;') +``` + +This is only needed once, new updates are coming from this new repo then, as usual. \ No newline at end of file diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000..8919ddc --- /dev/null +++ b/config.yaml @@ -0,0 +1,31 @@ +arch: x86_64 +repos: + - core + - extra + - community + +svn2git: + upstream-core-extra: "https://github.com/archlinux/svntogit-packages.git" + upstream-community: "https://github.com/archlinux/svntogit-community.git" + +basedir: + repo: /var/lib/alhp/repo/ + chroot: /var/lib/alhp/chroot/ + makepkg: /var/lib/alhp/makepkg/ + upstream: /var/lib/alhp/upstream/ + +march: + - x86-64-v3 + +blacklist: + - pacman + - tensorflow + - tensorflow-cuda + - gcc + +build: + worker: 4 + makej: 8 + +logging: + level: DEBUG \ No newline at end of file diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..b82effb --- /dev/null +++ b/go.mod @@ -0,0 +1,11 @@ +module ALHP.go + +go 1.16 + +require ( + github.com/Jguer/go-alpm/v2 v2.0.5 + github.com/Morganamilo/go-srcinfo v1.0.0 + github.com/sirupsen/logrus v1.8.1 + github.com/yargevad/filepathx v1.0.0 + gopkg.in/yaml.v2 v2.4.0 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..bce3db8 --- /dev/null +++ b/go.sum @@ -0,0 +1,20 @@ +github.com/Jguer/go-alpm/v2 v2.0.5 h1:1TZxkvCIfTOhjhxGy/Z1FNSeuY9DXBKF5qxUoj0IZ0A= +github.com/Jguer/go-alpm/v2 v2.0.5/go.mod h1:zU4iKCtNkDARfj5BrKJXYAQ5nIjtZbySfa0paboSmTQ= +github.com/Morganamilo/go-srcinfo v1.0.0 h1:Wh4nEF+HJWo+29hnxM18Q2hi+DUf0GejS13+Wg+dzmI= +github.com/Morganamilo/go-srcinfo v1.0.0/go.mod h1:MP6VGY1NNpVUmYIEgoM9acix95KQqIRyqQ0hCLsyYUY= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/yargevad/filepathx v1.0.0 h1:SYcT+N3tYGi+NvazubCNlvgIPbzAk7i7y2dwg3I5FYc= +github.com/yargevad/filepathx v1.0.0/go.mod h1:BprfX/gpYNJHJfc35GjRRpVcwWXS89gGulUIU5tK3tA= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= diff --git a/main.go b/main.go new file mode 100644 index 0000000..8872c54 --- /dev/null +++ b/main.go @@ -0,0 +1,626 @@ +package main + +import ( + "bufio" + "errors" + "fmt" + "github.com/Jguer/go-alpm/v2" + "github.com/Morganamilo/go-srcinfo" + log "github.com/sirupsen/logrus" + "github.com/yargevad/filepathx" + "gopkg.in/yaml.v2" + "io" + "os" + "os/exec" + "os/signal" + "path/filepath" + "regexp" + "strconv" + "strings" + "sync" + "syscall" + "time" +) + +const ( + pacmanConf = "/usr/share/devtools/pacman-extra.conf" + makepkgConf = "/usr/share/devtools/makepkg-x86_64.conf" + logDir = "logs" + orgChrootName = "root" +) + +var ( + conf = Conf{} + repos []string + reMarch = regexp.MustCompile(`(-march=)(.+?) `) + rePkgRel = regexp.MustCompile(`(?m)^pkgrel\s*=\s*(.+)$`) + rePkgFile = regexp.MustCompile(`^(.*)-.*-.*-(?:x86_64|any)\.pkg\.tar\.zst(?:\.sig)*$`) + buildManager BuildManager +) + +type BuildPackage struct { + Pkgbase string + Pkgbuild string + Srcinfo *srcinfo.Srcinfo + PkgFiles []string + Repo string + March string + FullRepo string +} + +type BuildManager struct { + toBuild chan *BuildPackage + toParse chan *BuildPackage + toPurge chan *BuildPackage + toRepoAdd chan *BuildPackage + exit bool + wg sync.WaitGroup + failedMutex sync.RWMutex +} + +type Conf struct { + Arch string + Repos, March, Blacklist []string + Svn2git map[string]string + Basedir struct { + Repo, Chroot, Makepkg, Upstream string + } + Build struct { + Worker int + Makej int + } + Logging struct { + Level string + } +} + +func check(e error) { + if e != nil { + panic(e) + } +} + +func contains(s []string, str string) bool { + if i := find(s, str); i != -1 { + return true + } + + return false +} + +func find(s []string, str string) int { + for i, v := range s { + if v == str { + return i + } + } + + return -1 +} + +func copyFile(src, dst string) (int64, error) { + sourceFileStat, err := os.Stat(src) + if err != nil { + return 0, err + } + + if !sourceFileStat.Mode().IsRegular() { + return 0, fmt.Errorf("%s is not a regular file", src) + } + + source, err := os.Open(src) + if err != nil { + return 0, err + } + defer source.Close() + + destination, err := os.Create(dst) + if err != nil { + return 0, err + } + defer destination.Close() + nBytes, err := io.Copy(destination, source) + return nBytes, err +} + +//goland:noinspection SpellCheckingInspection +func setupMakepkg(march string) { + lMakepkg := filepath.Join(conf.Basedir.Makepkg, fmt.Sprintf("makepkg-%s.conf", march)) + + if _, err := os.Stat(lMakepkg); errors.Is(err, os.ErrNotExist) { + check(os.MkdirAll(conf.Basedir.Makepkg, os.ModePerm)) + t, err := os.ReadFile(makepkgConf) + check(err) + makepkgStr := string(t) + + makepkgStr = strings.ReplaceAll(makepkgStr, "-mtune=generic", "") + makepkgStr = strings.ReplaceAll(makepkgStr, "-O2", "-O3") + makepkgStr = strings.ReplaceAll(makepkgStr, "#MAKEFLAGS=\"-j2\"", "MAKEFLAGS=\"-j"+strconv.Itoa(conf.Build.Makej)+"\"") + makepkgStr = reMarch.ReplaceAllString(makepkgStr, "${1}"+march) + + check(os.WriteFile(lMakepkg, []byte(makepkgStr), os.ModePerm)) + } +} + +func syncMarchs() { + files, err := os.ReadDir(conf.Basedir.Repo) + check(err) + + var eRepos []string + for _, file := range files { + if file.Name() != "." && file.Name() != logDir && file.IsDir() { + eRepos = append(eRepos, file.Name()) + } + } + + for _, march := range conf.March { + setupMakepkg(march) + for _, repo := range conf.Repos { + tRepo := fmt.Sprintf("%s-%s", repo, march) + repos = append(repos, tRepo) + + if _, err := os.Stat(filepath.Join(filepath.Join(conf.Basedir.Repo, tRepo, "os", conf.Arch))); os.IsNotExist(err) { + log.Debugf("Creating path %s", filepath.Join(conf.Basedir.Repo, tRepo, "os", conf.Arch)) + check(os.MkdirAll(filepath.Join(conf.Basedir.Repo, tRepo, "os", conf.Arch), os.ModePerm)) + } + + if i := find(eRepos, tRepo); i != -1 { + eRepos = append(eRepos[:i], eRepos[i+1:]...) + } + } + } + + log.Infof("Repos: %s", repos) + + for _, repo := range eRepos { + log.Infof("Removing old repo %s", repo) + check(os.RemoveAll(filepath.Join(conf.Basedir.Repo, repo))) + } +} + +func importKeys(pkg *BuildPackage) { + if pkg.Srcinfo.ValidPGPKeys != nil { + args := []string{"--keyserver", "keyserver.ubuntu.com", "--recv-keys"} + args = append(args, pkg.Srcinfo.ValidPGPKeys...) + cmd := backgroundCmd("gpg", args...) + res, err := cmd.CombinedOutput() + log.Debug(string(res)) + + if err != nil { + log.Warningf("Unable to import keys: %s", string(res)) + } + } +} + +func increasePkgRel(pkg *BuildPackage) { + f, err := os.OpenFile(pkg.Pkgbuild, os.O_RDWR, os.ModePerm) + check(err) + defer f.Close() + + fStr, err := io.ReadAll(f) + check(err) + + nStr := rePkgRel.ReplaceAllLiteralString(string(fStr), "pkgrel="+pkg.Srcinfo.Pkgrel+".1") + _, err = f.Seek(0, 0) + check(err) + check(f.Truncate(0)) + + _, err = f.WriteString(nStr) + check(err) +} + +func gitClean(pkg *BuildPackage) { + cmd := backgroundCmd("sh", "-c", "cd "+filepath.Dir(pkg.Pkgbuild)+"&&git clean -xdff") + res, err := cmd.CombinedOutput() + log.Debug(string(res)) + check(err) +} + +func (b *BuildManager) buildWorker(id int) { + for { + select { + case pkg := <-b.toBuild: + if b.exit { + continue + } else { + b.wg.Add(1) + } + + start := time.Now() + + log.Infof("[%s/%s] Build starting", pkg.FullRepo, pkg.Pkgbase) + + importKeys(pkg) + increasePkgRel(pkg) + pkg.PkgFiles = []string{} + + cmd := backgroundCmd("sh", "-c", + "cd "+filepath.Dir(pkg.Pkgbuild)+"&&makechrootpkg -c -D "+conf.Basedir.Makepkg+" -l worker-"+strconv.Itoa(id)+" -r "+conf.Basedir.Chroot+" -- "+ + "--config "+filepath.Join(conf.Basedir.Makepkg, fmt.Sprintf("makepkg-%s.conf", pkg.March))) + res, err := cmd.CombinedOutput() + if err != nil { + log.Warningf("[%s/%s] Build failed, exit code %d", pkg.FullRepo, pkg.Pkgbase, cmd.ProcessState.ExitCode()) + + b.failedMutex.Lock() + f, err := os.OpenFile(filepath.Join(conf.Basedir.Repo, pkg.FullRepo+"_failed.txt"), os.O_WRONLY|os.O_APPEND|os.O_CREATE, os.ModePerm) + check(err) + + if pkg.Srcinfo.Epoch != "" { + _, err := f.WriteString(fmt.Sprintf("%s==%s:%s-%s\n", pkg.Pkgbase, pkg.Srcinfo.Epoch, pkg.Srcinfo.Pkgver, pkg.Srcinfo.Pkgrel)) + check(err) + } else { + _, err := f.WriteString(fmt.Sprintf("%s==%s-%s\n", pkg.Pkgbase, pkg.Srcinfo.Pkgver, pkg.Srcinfo.Pkgrel)) + check(err) + } + f.Close() + b.failedMutex.Unlock() + + check(os.MkdirAll(filepath.Join(conf.Basedir.Repo, "logs"), os.ModePerm)) + check(os.WriteFile(filepath.Join(conf.Basedir.Repo, "logs", pkg.Pkgbase+".log"), res, os.ModePerm)) + + gitClean(pkg) + b.wg.Done() + continue + } + + pkgFiles, err := filepath.Glob(filepath.Join(filepath.Dir(pkg.Pkgbuild), "*.pkg.tar.zst")) + check(err) + log.Debug(pkgFiles) + + if len(pkgFiles) == 0 { + log.Warningf("No packages found after building %s. Abort build.", pkg.Pkgbase) + + gitClean(pkg) + b.wg.Done() + continue + } + + for _, file := range pkgFiles { + cmd = backgroundCmd("gpg", "--batch", "--detach-sign", file) + res, err := cmd.CombinedOutput() + log.Debug(string(res)) + if err != nil { + log.Warningf("Failed to sign %s: %s", pkg.Pkgbase, err) + b.wg.Done() + continue + } + } + + copyFiles, err := filepath.Glob(filepath.Join(filepath.Dir(pkg.Pkgbuild), "*.pkg.tar.zst*")) + check(err) + + for _, file := range copyFiles { + _, err = copyFile(file, filepath.Join(conf.Basedir.Repo, pkg.FullRepo, "os", conf.Arch, filepath.Base(file))) + if err != nil { + check(err) + b.wg.Done() + continue + } + + if filepath.Ext(file) != ".sig" { + pkg.PkgFiles = append(pkg.PkgFiles, filepath.Join(conf.Basedir.Repo, pkg.FullRepo, "os", conf.Arch, filepath.Base(file))) + } + } + b.toRepoAdd <- pkg + + if _, err := os.Stat(filepath.Join(conf.Basedir.Repo, "logs", pkg.Pkgbase+".log")); err == nil { + check(os.Remove(filepath.Join(conf.Basedir.Repo, "logs", pkg.Pkgbase+".log"))) + } + + gitClean(pkg) + log.Infof("[%s/%s] Build successful (%s)", pkg.FullRepo, pkg.Pkgbase, time.Now().Sub(start)) + } + } +} + +func (b *BuildManager) parseWorker() { + for { + if b.exit { + return + } + select { + case pkg := <-b.toParse: + cmd := backgroundCmd("sh", "-c", "cd "+filepath.Dir(pkg.Pkgbuild)+"&&"+"makepkg --printsrcinfo") + res, err := cmd.Output() + if err != nil { + log.Warningf("Failed generate SRCINFO for %s: %s", pkg.Pkgbase, err) + continue + } + + info, err := srcinfo.Parse(string(res)) + if err != nil { + log.Warningf("Failed to parse SRCINFO for %s: %s", pkg.Pkgbase, err) + continue + } + pkg.Srcinfo = info + + if contains(info.Arch, "any") || contains(conf.Blacklist, info.Pkgbase) { + log.Infof("Skipped %s: blacklisted or any-Package", info.Pkgbase) + b.toPurge <- pkg + continue + } + + if isPkgFailed(pkg) { + log.Infof("Skipped %s: failed build", info.Pkgbase) + b.toPurge <- pkg + continue + } + + var pkgVer string + if pkg.Srcinfo.Epoch == "" { + pkgVer = pkg.Srcinfo.Pkgver + "-" + pkg.Srcinfo.Pkgrel + } else { + pkgVer = pkg.Srcinfo.Epoch + ":" + pkg.Srcinfo.Pkgver + "-" + pkg.Srcinfo.Pkgrel + } + + repoVer := getVersionFromRepo(pkg) + if repoVer != "" && alpm.VerCmp(repoVer, pkgVer) > 0 { + log.Debugf("Skipped %s: Version in repo higher than in PKGBUILD (%s < %s)", info.Pkgbase, pkgVer, repoVer) + continue + } + + b.toBuild <- pkg + } + } +} + +func findPkgFiles(pkg *BuildPackage) { + pkgs, err := os.ReadDir(filepath.Join(conf.Basedir.Repo, pkg.FullRepo, "os", conf.Arch)) + check(err) + + var fPkg []string + for _, file := range pkgs { + if !file.IsDir() && !strings.HasSuffix(file.Name(), ".sig") { + matches := rePkgFile.FindStringSubmatch(file.Name()) + + var realPkgs []string + for _, realPkg := range pkg.Srcinfo.Packages { + realPkgs = append(realPkgs, realPkg.Pkgname) + } + + if len(matches) > 1 && contains(realPkgs, matches[1]) { + fPkg = append(fPkg, filepath.Join(conf.Basedir.Repo, pkg.FullRepo, "os", conf.Arch, file.Name())) + } + } + } + + pkg.PkgFiles = fPkg +} + +func getVersionFromRepo(pkg *BuildPackage) string { + findPkgFiles(pkg) + + if len(pkg.PkgFiles) == 0 { + return "" + } + + fNameSplit := strings.Split(pkg.PkgFiles[0], "-") + return fNameSplit[len(fNameSplit)-3] + "-" + fNameSplit[len(fNameSplit)-2] +} + +func isPkgFailed(pkg *BuildPackage) bool { + buildManager.failedMutex.Lock() + defer buildManager.failedMutex.Unlock() + + file, err := os.OpenFile(filepath.Join(conf.Basedir.Repo, pkg.FullRepo+"_failed.txt"), os.O_RDWR|os.O_CREATE, os.ModePerm) + check(err) + defer file.Close() + + failed := false + var newContent []string + scanner := bufio.NewScanner(file) + for scanner.Scan() { + splitPkg := strings.Split(scanner.Text(), "==") + + if splitPkg[0] == pkg.Pkgbase { + var pkgVer string + if pkg.Srcinfo.Epoch == "" { + pkgVer = pkg.Srcinfo.Pkgver + "-" + pkg.Srcinfo.Pkgrel + } else { + pkgVer = pkg.Srcinfo.Epoch + ":" + pkg.Srcinfo.Pkgver + "-" + pkg.Srcinfo.Pkgrel + } + + if alpm.VerCmp(splitPkg[1], pkgVer) < 0 { + failed = false + } else { + failed = true + newContent = append(newContent, scanner.Text()) + } + } else { + newContent = append(newContent, scanner.Text()) + } + } + + check(scanner.Err()) + + _, err = file.Seek(0, 0) + check(err) + check(file.Truncate(0)) + _, err = file.WriteString(strings.Join(newContent, "\n")) + check(err) + + return failed +} + +func setupChroot() { + if _, err := os.Stat(filepath.Join(conf.Basedir.Chroot, orgChrootName)); err == nil { + //goland:noinspection SpellCheckingInspection + cmd := backgroundCmd("arch-nspawn", filepath.Join(conf.Basedir.Chroot, orgChrootName), "pacman", "-Syuu", "--noconfirm") + res, err := cmd.CombinedOutput() + log.Debug(string(res)) + check(err) + } else if os.IsNotExist(err) { + err := os.MkdirAll(conf.Basedir.Chroot, os.ModePerm) + check(err) + + cmd := backgroundCmd("mkarchroot", "-C", pacmanConf, filepath.Join(conf.Basedir.Chroot, orgChrootName), "base-devel") + res, err := cmd.CombinedOutput() + log.Debug(string(res)) + check(err) + } else { + check(err) + } +} + +func (b *BuildManager) repoWorker() { + for { + select { + case pkg := <-b.toRepoAdd: + args := []string{"-s", "-v", "-p", "-n", filepath.Join(conf.Basedir.Repo, pkg.FullRepo, "os", conf.Arch, pkg.FullRepo) + ".db.tar.xz"} + args = append(args, pkg.PkgFiles...) + cmd := backgroundCmd("repo-add", args...) + res, err := cmd.CombinedOutput() + log.Debug(string(res)) + check(err) + + cmd = backgroundCmd("paccache", + "-rc", filepath.Join(conf.Basedir.Repo, pkg.FullRepo, "os", conf.Arch), + "-k", "1") + res, err = cmd.CombinedOutput() + log.Debug(string(res)) + check(err) + b.wg.Done() + case pkg := <-b.toPurge: + if _, err := os.Stat(filepath.Join(conf.Basedir.Repo, pkg.FullRepo, "os", conf.Arch, pkg.FullRepo) + ".db.tar.xz"); err != nil { + continue + } + if len(pkg.PkgFiles) == 0 { + findPkgFiles(pkg) + } + + var realPkgs []string + for _, realPkg := range pkg.Srcinfo.Packages { + realPkgs = append(realPkgs, realPkg.Pkgname) + } + + args := []string{"-s", "-v", filepath.Join(conf.Basedir.Repo, pkg.FullRepo, "os", conf.Arch, pkg.FullRepo) + ".db.tar.xz"} + args = append(args, realPkgs...) + cmd := backgroundCmd("repo-remove", args...) + res, err := cmd.CombinedOutput() + log.Debug(string(res)) + if err != nil && cmd.ProcessState.ExitCode() == 1 { + log.Debugf("Deleteing package %s failed: Package not found in database", pkg.Pkgbase) + continue + } + + for _, file := range pkg.PkgFiles { + check(os.Remove(file)) + check(os.Remove(file + ".sig")) + } + } + } +} + +func backgroundCmd(name string, arg ...string) *exec.Cmd { + cmd := exec.Command(name, arg...) + cmd.SysProcAttr = &syscall.SysProcAttr{ + Foreground: false, + Setsid: true, + } + return cmd +} + +func (b *BuildManager) syncWorker() { + check(os.MkdirAll(conf.Basedir.Upstream, os.ModePerm)) + + for i := 0; i < conf.Build.Worker; i++ { + go b.buildWorker(i) + go b.parseWorker() + } + + for { + b.wg.Wait() + for gitDir, gitURL := range conf.Svn2git { + gitPath := filepath.Join(conf.Basedir.Upstream, gitDir) + + if _, err := os.Stat(gitPath); os.IsNotExist(err) { + cmd := backgroundCmd("git", "clone", "--depth=1", gitURL, gitPath) + res, err := cmd.CombinedOutput() + log.Debug(string(res)) + check(err) + } else if err == nil { + cmd := backgroundCmd("sh", "-c", "cd "+gitPath+" && git clean -xdff") + res, err := cmd.CombinedOutput() + log.Debug(string(res)) + check(err) + + cmd = backgroundCmd("sh", "-c", "cd "+gitPath+" && git reset --hard") + res, err = cmd.CombinedOutput() + log.Debug(string(res)) + check(err) + + cmd = backgroundCmd("sh", "-c", "cd "+gitPath+" && git pull") + res, err = cmd.CombinedOutput() + log.Debug(string(res)) + check(err) + } + } + + pkgBuilds, err := filepathx.Glob(filepath.Join(conf.Basedir.Upstream, "/**/PKGBUILD")) + check(err) + + for _, pkgbuild := range pkgBuilds { + if b.exit { + return + } + + sPkgbuild := strings.Split(pkgbuild, "/") + repo := sPkgbuild[len(sPkgbuild)-2] + + if repo == "trunk" || !contains(conf.Repos, strings.Split(repo, "-")[0]) || strings.Contains(repo, "i686") { + continue + } + + for _, march := range conf.March { + b.toParse <- &BuildPackage{ + Pkgbuild: pkgbuild, + Pkgbase: sPkgbuild[len(sPkgbuild)-4], + Repo: strings.Split(repo, "-")[0], + March: march, + FullRepo: strings.Split(repo, "-")[0] + "-" + march, + } + } + } + + time.Sleep(5 * time.Minute) + } +} + +func main() { + killSignals := make(chan os.Signal, 1) + signal.Notify(killSignals, syscall.SIGINT, syscall.SIGTERM) + + confStr, err := os.ReadFile("config.yaml") + check(err) + + err = yaml.Unmarshal(confStr, &conf) + check(err) + + lvl, err := log.ParseLevel(conf.Logging.Level) + check(err) + log.SetLevel(lvl) + + err = os.MkdirAll(conf.Basedir.Repo, os.ModePerm) + check(err) + + buildManager = BuildManager{ + toBuild: make(chan *BuildPackage), + toParse: make(chan *BuildPackage, conf.Build.Worker), + toPurge: make(chan *BuildPackage), + toRepoAdd: make(chan *BuildPackage, conf.Build.Worker), + exit: false, + wg: sync.WaitGroup{}, + failedMutex: sync.RWMutex{}, + } + + setupChroot() + syncMarchs() + + go buildManager.repoWorker() + go buildManager.syncWorker() + + <-killSignals + + buildManager.exit = true + buildManager.wg.Wait() +}