forked from ALHP/ALHP.GO
inital commit
This commit is contained in:
155
.gitignore
vendored
Normal file
155
.gitignore
vendored
Normal file
@@ -0,0 +1,155 @@
|
|||||||
|
# Created by https://www.toptal.com/developers/gitignore/api/go,linux,intellij+all,windows
|
||||||
|
# Edit at https://www.toptal.com/developers/gitignore?templates=go,linux,intellij+all,windows
|
||||||
|
|
||||||
|
### Go ###
|
||||||
|
# Binaries for programs and plugins
|
||||||
|
*.exe
|
||||||
|
*.exe~
|
||||||
|
*.dll
|
||||||
|
*.so
|
||||||
|
*.dylib
|
||||||
|
|
||||||
|
# Test binary, built with `go test -c`
|
||||||
|
*.test
|
||||||
|
|
||||||
|
# Output of the go coverage tool, specifically when used with LiteIDE
|
||||||
|
*.out
|
||||||
|
|
||||||
|
# Dependency directories (remove the comment below to include it)
|
||||||
|
# vendor/
|
||||||
|
|
||||||
|
### Go Patch ###
|
||||||
|
/vendor/
|
||||||
|
/Godeps/
|
||||||
|
|
||||||
|
### Intellij+all ###
|
||||||
|
# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider
|
||||||
|
# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
|
||||||
|
|
||||||
|
# User-specific stuff
|
||||||
|
.idea/**/workspace.xml
|
||||||
|
.idea/**/tasks.xml
|
||||||
|
.idea/**/usage.statistics.xml
|
||||||
|
.idea/**/dictionaries
|
||||||
|
.idea/**/shelf
|
||||||
|
|
||||||
|
# Generated files
|
||||||
|
.idea/**/contentModel.xml
|
||||||
|
|
||||||
|
# Sensitive or high-churn files
|
||||||
|
.idea/**/dataSources/
|
||||||
|
.idea/**/dataSources.ids
|
||||||
|
.idea/**/dataSources.local.xml
|
||||||
|
.idea/**/sqlDataSources.xml
|
||||||
|
.idea/**/dynamic.xml
|
||||||
|
.idea/**/uiDesigner.xml
|
||||||
|
.idea/**/dbnavigator.xml
|
||||||
|
|
||||||
|
# Gradle
|
||||||
|
.idea/**/gradle.xml
|
||||||
|
.idea/**/libraries
|
||||||
|
|
||||||
|
# Gradle and Maven with auto-import
|
||||||
|
# When using Gradle or Maven with auto-import, you should exclude module files,
|
||||||
|
# since they will be recreated, and may cause churn. Uncomment if using
|
||||||
|
# auto-import.
|
||||||
|
# .idea/artifacts
|
||||||
|
# .idea/compiler.xml
|
||||||
|
# .idea/jarRepositories.xml
|
||||||
|
# .idea/modules.xml
|
||||||
|
# .idea/*.iml
|
||||||
|
# .idea/modules
|
||||||
|
# *.iml
|
||||||
|
# *.ipr
|
||||||
|
|
||||||
|
# CMake
|
||||||
|
cmake-build-*/
|
||||||
|
|
||||||
|
# Mongo Explorer plugin
|
||||||
|
.idea/**/mongoSettings.xml
|
||||||
|
|
||||||
|
# File-based project format
|
||||||
|
*.iws
|
||||||
|
|
||||||
|
# IntelliJ
|
||||||
|
out/
|
||||||
|
|
||||||
|
# mpeltonen/sbt-idea plugin
|
||||||
|
.idea_modules/
|
||||||
|
|
||||||
|
# JIRA plugin
|
||||||
|
atlassian-ide-plugin.xml
|
||||||
|
|
||||||
|
# Cursive Clojure plugin
|
||||||
|
.idea/replstate.xml
|
||||||
|
|
||||||
|
# Crashlytics plugin (for Android Studio and IntelliJ)
|
||||||
|
com_crashlytics_export_strings.xml
|
||||||
|
crashlytics.properties
|
||||||
|
crashlytics-build.properties
|
||||||
|
fabric.properties
|
||||||
|
|
||||||
|
# Editor-based Rest Client
|
||||||
|
.idea/httpRequests
|
||||||
|
|
||||||
|
# Android studio 3.1+ serialized cache file
|
||||||
|
.idea/caches/build_file_checksums.ser
|
||||||
|
|
||||||
|
### Intellij+all Patch ###
|
||||||
|
# Ignores the whole .idea folder and all .iml files
|
||||||
|
# See https://github.com/joeblau/gitignore.io/issues/186 and https://github.com/joeblau/gitignore.io/issues/360
|
||||||
|
|
||||||
|
.idea/
|
||||||
|
|
||||||
|
# Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-249601023
|
||||||
|
|
||||||
|
*.iml
|
||||||
|
modules.xml
|
||||||
|
.idea/misc.xml
|
||||||
|
*.ipr
|
||||||
|
|
||||||
|
# Sonarlint plugin
|
||||||
|
.idea/sonarlint
|
||||||
|
|
||||||
|
### Linux ###
|
||||||
|
*~
|
||||||
|
|
||||||
|
# temporary files which can be created if a process still has a handle open of a deleted file
|
||||||
|
.fuse_hidden*
|
||||||
|
|
||||||
|
# KDE directory preferences
|
||||||
|
.directory
|
||||||
|
|
||||||
|
# Linux trash folder which might appear on any partition or disk
|
||||||
|
.Trash-*
|
||||||
|
|
||||||
|
# .nfs files are created when an open file is removed but is still being accessed
|
||||||
|
.nfs*
|
||||||
|
|
||||||
|
### Windows ###
|
||||||
|
# Windows thumbnail cache files
|
||||||
|
Thumbs.db
|
||||||
|
Thumbs.db:encryptable
|
||||||
|
ehthumbs.db
|
||||||
|
ehthumbs_vista.db
|
||||||
|
|
||||||
|
# Dump file
|
||||||
|
*.stackdump
|
||||||
|
|
||||||
|
# Folder config file
|
||||||
|
[Dd]esktop.ini
|
||||||
|
|
||||||
|
# Recycle Bin used on file shares
|
||||||
|
$RECYCLE.BIN/
|
||||||
|
|
||||||
|
# Windows Installer files
|
||||||
|
*.cab
|
||||||
|
*.msi
|
||||||
|
*.msix
|
||||||
|
*.msm
|
||||||
|
*.msp
|
||||||
|
|
||||||
|
# Windows shortcuts
|
||||||
|
*.lnk
|
||||||
|
|
||||||
|
# End of https://www.toptal.com/developers/gitignore/api/go,linux,intellij+all,windows
|
||||||
74
README.md
Normal file
74
README.md
Normal file
@@ -0,0 +1,74 @@
|
|||||||
|
# alhp
|
||||||
|
|
||||||
|
Build script for archlinux instructionset enabled repos.
|
||||||
|
All packages are build with -march=<cpu-set> and -O3. Some packages will not build with -O3, they will just be provided from the official repos as usual.
|
||||||
|
|
||||||
|
## Check your system for support
|
||||||
|
|
||||||
|
**Important**: Before you enable any of these repos, check if your system supports x86-64-v3. You can do that with `/lib/ld-linux-x86-64.so.2 --help`. If you don't check beforehand you might be unable to boot your system anymore and need to downgrade any package that you may have upgraded.
|
||||||
|
|
||||||
|
Example output snippet for a system supporting up to `x86-64-v3`:
|
||||||
|
|
||||||
|
```
|
||||||
|
Subdirectories of glibc-hwcaps directories, in priority order:
|
||||||
|
x86-64-v4
|
||||||
|
x86-64-v3 (supported, searched)
|
||||||
|
x86-64-v2 (supported, searched)
|
||||||
|
```
|
||||||
|
|
||||||
|
## Enable Repos
|
||||||
|
|
||||||
|
To enable these complement repos you need to add them above the regular repos in `/etc/pacman.conf`
|
||||||
|
|
||||||
|
### Example pacman.conf
|
||||||
|
|
||||||
|
```editorconfig
|
||||||
|
[core-x86-64-v3]
|
||||||
|
Server = https://alhp.harting.dev/$repo/os/$arch/
|
||||||
|
|
||||||
|
[extra-x86-64-v3]
|
||||||
|
Server = https://alhp.harting.dev/$repo/os/$arch/
|
||||||
|
|
||||||
|
[community-x86-64-v3]
|
||||||
|
Server = https://alhp.harting.dev/$repo/os/$arch/
|
||||||
|
|
||||||
|
[core]
|
||||||
|
Include = /etc/pacman.d/mirrorlist
|
||||||
|
|
||||||
|
[extra]
|
||||||
|
Include = /etc/pacman.d/mirrorlist
|
||||||
|
|
||||||
|
[community]
|
||||||
|
Include = /etc/pacman.d/mirrorlist
|
||||||
|
```
|
||||||
|
|
||||||
|
Replace `x86-64-v3` with your cpu-set. More information about all available options on [this gcc page](https://gcc.gnu.org/onlinedocs/gcc/x86-Options.html).
|
||||||
|
Currently, alhp.harting.dev only builds for `x86-64-v3` (list is subject to change).
|
||||||
|
You can see all available repositories [here](https://alhp.harting.dev/).
|
||||||
|
|
||||||
|
After finished adding the repos to `pacman.conf` you need to import and sign the used pgp key:
|
||||||
|
|
||||||
|
Import:
|
||||||
|
```
|
||||||
|
pacman-key --keyserver keyserver.ubuntu.com --recv-keys 0D4D2FDAF45468F3DDF59BEDE3D0D2CD3952E298
|
||||||
|
```
|
||||||
|
|
||||||
|
Local sign:
|
||||||
|
```
|
||||||
|
pacman-key --lsign-key 0D4D2FDAF45468F3DDF59BEDE3D0D2CD3952E298
|
||||||
|
```
|
||||||
|
|
||||||
|
Update package database:
|
||||||
|
```
|
||||||
|
pacman -Sy
|
||||||
|
```
|
||||||
|
|
||||||
|
## Replace packages
|
||||||
|
Following command reinstalls all packages found in the repo **extra-x86-64-v3** that are already installed.
|
||||||
|
Replace `extra-x86-64-v3` with whatever repo you want to install.
|
||||||
|
|
||||||
|
```shell script
|
||||||
|
pacman -S $(pacman -Sl x86-64-v3 | grep installed | cut -f 2 -d " " | perl -pe 's/\R/ /g;')
|
||||||
|
```
|
||||||
|
|
||||||
|
This is only needed once, new updates are coming from this new repo then, as usual.
|
||||||
31
config.yaml
Normal file
31
config.yaml
Normal file
@@ -0,0 +1,31 @@
|
|||||||
|
arch: x86_64
|
||||||
|
repos:
|
||||||
|
- core
|
||||||
|
- extra
|
||||||
|
- community
|
||||||
|
|
||||||
|
svn2git:
|
||||||
|
upstream-core-extra: "https://github.com/archlinux/svntogit-packages.git"
|
||||||
|
upstream-community: "https://github.com/archlinux/svntogit-community.git"
|
||||||
|
|
||||||
|
basedir:
|
||||||
|
repo: /var/lib/alhp/repo/
|
||||||
|
chroot: /var/lib/alhp/chroot/
|
||||||
|
makepkg: /var/lib/alhp/makepkg/
|
||||||
|
upstream: /var/lib/alhp/upstream/
|
||||||
|
|
||||||
|
march:
|
||||||
|
- x86-64-v3
|
||||||
|
|
||||||
|
blacklist:
|
||||||
|
- pacman
|
||||||
|
- tensorflow
|
||||||
|
- tensorflow-cuda
|
||||||
|
- gcc
|
||||||
|
|
||||||
|
build:
|
||||||
|
worker: 4
|
||||||
|
makej: 8
|
||||||
|
|
||||||
|
logging:
|
||||||
|
level: DEBUG
|
||||||
11
go.mod
Normal file
11
go.mod
Normal file
@@ -0,0 +1,11 @@
|
|||||||
|
module ALHP.go
|
||||||
|
|
||||||
|
go 1.16
|
||||||
|
|
||||||
|
require (
|
||||||
|
github.com/Jguer/go-alpm/v2 v2.0.5
|
||||||
|
github.com/Morganamilo/go-srcinfo v1.0.0
|
||||||
|
github.com/sirupsen/logrus v1.8.1
|
||||||
|
github.com/yargevad/filepathx v1.0.0
|
||||||
|
gopkg.in/yaml.v2 v2.4.0
|
||||||
|
)
|
||||||
20
go.sum
Normal file
20
go.sum
Normal file
@@ -0,0 +1,20 @@
|
|||||||
|
github.com/Jguer/go-alpm/v2 v2.0.5 h1:1TZxkvCIfTOhjhxGy/Z1FNSeuY9DXBKF5qxUoj0IZ0A=
|
||||||
|
github.com/Jguer/go-alpm/v2 v2.0.5/go.mod h1:zU4iKCtNkDARfj5BrKJXYAQ5nIjtZbySfa0paboSmTQ=
|
||||||
|
github.com/Morganamilo/go-srcinfo v1.0.0 h1:Wh4nEF+HJWo+29hnxM18Q2hi+DUf0GejS13+Wg+dzmI=
|
||||||
|
github.com/Morganamilo/go-srcinfo v1.0.0/go.mod h1:MP6VGY1NNpVUmYIEgoM9acix95KQqIRyqQ0hCLsyYUY=
|
||||||
|
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||||
|
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
|
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||||
|
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||||
|
github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
|
||||||
|
github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
|
||||||
|
github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
|
||||||
|
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||||
|
github.com/yargevad/filepathx v1.0.0 h1:SYcT+N3tYGi+NvazubCNlvgIPbzAk7i7y2dwg3I5FYc=
|
||||||
|
github.com/yargevad/filepathx v1.0.0/go.mod h1:BprfX/gpYNJHJfc35GjRRpVcwWXS89gGulUIU5tK3tA=
|
||||||
|
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4=
|
||||||
|
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||||
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||||
|
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
|
||||||
|
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
|
||||||
626
main.go
Normal file
626
main.go
Normal file
@@ -0,0 +1,626 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"github.com/Jguer/go-alpm/v2"
|
||||||
|
"github.com/Morganamilo/go-srcinfo"
|
||||||
|
log "github.com/sirupsen/logrus"
|
||||||
|
"github.com/yargevad/filepathx"
|
||||||
|
"gopkg.in/yaml.v2"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"os/signal"
|
||||||
|
"path/filepath"
|
||||||
|
"regexp"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"syscall"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
pacmanConf = "/usr/share/devtools/pacman-extra.conf"
|
||||||
|
makepkgConf = "/usr/share/devtools/makepkg-x86_64.conf"
|
||||||
|
logDir = "logs"
|
||||||
|
orgChrootName = "root"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
conf = Conf{}
|
||||||
|
repos []string
|
||||||
|
reMarch = regexp.MustCompile(`(-march=)(.+?) `)
|
||||||
|
rePkgRel = regexp.MustCompile(`(?m)^pkgrel\s*=\s*(.+)$`)
|
||||||
|
rePkgFile = regexp.MustCompile(`^(.*)-.*-.*-(?:x86_64|any)\.pkg\.tar\.zst(?:\.sig)*$`)
|
||||||
|
buildManager BuildManager
|
||||||
|
)
|
||||||
|
|
||||||
|
type BuildPackage struct {
|
||||||
|
Pkgbase string
|
||||||
|
Pkgbuild string
|
||||||
|
Srcinfo *srcinfo.Srcinfo
|
||||||
|
PkgFiles []string
|
||||||
|
Repo string
|
||||||
|
March string
|
||||||
|
FullRepo string
|
||||||
|
}
|
||||||
|
|
||||||
|
type BuildManager struct {
|
||||||
|
toBuild chan *BuildPackage
|
||||||
|
toParse chan *BuildPackage
|
||||||
|
toPurge chan *BuildPackage
|
||||||
|
toRepoAdd chan *BuildPackage
|
||||||
|
exit bool
|
||||||
|
wg sync.WaitGroup
|
||||||
|
failedMutex sync.RWMutex
|
||||||
|
}
|
||||||
|
|
||||||
|
type Conf struct {
|
||||||
|
Arch string
|
||||||
|
Repos, March, Blacklist []string
|
||||||
|
Svn2git map[string]string
|
||||||
|
Basedir struct {
|
||||||
|
Repo, Chroot, Makepkg, Upstream string
|
||||||
|
}
|
||||||
|
Build struct {
|
||||||
|
Worker int
|
||||||
|
Makej int
|
||||||
|
}
|
||||||
|
Logging struct {
|
||||||
|
Level string
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func check(e error) {
|
||||||
|
if e != nil {
|
||||||
|
panic(e)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func contains(s []string, str string) bool {
|
||||||
|
if i := find(s, str); i != -1 {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func find(s []string, str string) int {
|
||||||
|
for i, v := range s {
|
||||||
|
if v == str {
|
||||||
|
return i
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return -1
|
||||||
|
}
|
||||||
|
|
||||||
|
func copyFile(src, dst string) (int64, error) {
|
||||||
|
sourceFileStat, err := os.Stat(src)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if !sourceFileStat.Mode().IsRegular() {
|
||||||
|
return 0, fmt.Errorf("%s is not a regular file", src)
|
||||||
|
}
|
||||||
|
|
||||||
|
source, err := os.Open(src)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
defer source.Close()
|
||||||
|
|
||||||
|
destination, err := os.Create(dst)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
defer destination.Close()
|
||||||
|
nBytes, err := io.Copy(destination, source)
|
||||||
|
return nBytes, err
|
||||||
|
}
|
||||||
|
|
||||||
|
//goland:noinspection SpellCheckingInspection
|
||||||
|
func setupMakepkg(march string) {
|
||||||
|
lMakepkg := filepath.Join(conf.Basedir.Makepkg, fmt.Sprintf("makepkg-%s.conf", march))
|
||||||
|
|
||||||
|
if _, err := os.Stat(lMakepkg); errors.Is(err, os.ErrNotExist) {
|
||||||
|
check(os.MkdirAll(conf.Basedir.Makepkg, os.ModePerm))
|
||||||
|
t, err := os.ReadFile(makepkgConf)
|
||||||
|
check(err)
|
||||||
|
makepkgStr := string(t)
|
||||||
|
|
||||||
|
makepkgStr = strings.ReplaceAll(makepkgStr, "-mtune=generic", "")
|
||||||
|
makepkgStr = strings.ReplaceAll(makepkgStr, "-O2", "-O3")
|
||||||
|
makepkgStr = strings.ReplaceAll(makepkgStr, "#MAKEFLAGS=\"-j2\"", "MAKEFLAGS=\"-j"+strconv.Itoa(conf.Build.Makej)+"\"")
|
||||||
|
makepkgStr = reMarch.ReplaceAllString(makepkgStr, "${1}"+march)
|
||||||
|
|
||||||
|
check(os.WriteFile(lMakepkg, []byte(makepkgStr), os.ModePerm))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func syncMarchs() {
|
||||||
|
files, err := os.ReadDir(conf.Basedir.Repo)
|
||||||
|
check(err)
|
||||||
|
|
||||||
|
var eRepos []string
|
||||||
|
for _, file := range files {
|
||||||
|
if file.Name() != "." && file.Name() != logDir && file.IsDir() {
|
||||||
|
eRepos = append(eRepos, file.Name())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, march := range conf.March {
|
||||||
|
setupMakepkg(march)
|
||||||
|
for _, repo := range conf.Repos {
|
||||||
|
tRepo := fmt.Sprintf("%s-%s", repo, march)
|
||||||
|
repos = append(repos, tRepo)
|
||||||
|
|
||||||
|
if _, err := os.Stat(filepath.Join(filepath.Join(conf.Basedir.Repo, tRepo, "os", conf.Arch))); os.IsNotExist(err) {
|
||||||
|
log.Debugf("Creating path %s", filepath.Join(conf.Basedir.Repo, tRepo, "os", conf.Arch))
|
||||||
|
check(os.MkdirAll(filepath.Join(conf.Basedir.Repo, tRepo, "os", conf.Arch), os.ModePerm))
|
||||||
|
}
|
||||||
|
|
||||||
|
if i := find(eRepos, tRepo); i != -1 {
|
||||||
|
eRepos = append(eRepos[:i], eRepos[i+1:]...)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Infof("Repos: %s", repos)
|
||||||
|
|
||||||
|
for _, repo := range eRepos {
|
||||||
|
log.Infof("Removing old repo %s", repo)
|
||||||
|
check(os.RemoveAll(filepath.Join(conf.Basedir.Repo, repo)))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func importKeys(pkg *BuildPackage) {
|
||||||
|
if pkg.Srcinfo.ValidPGPKeys != nil {
|
||||||
|
args := []string{"--keyserver", "keyserver.ubuntu.com", "--recv-keys"}
|
||||||
|
args = append(args, pkg.Srcinfo.ValidPGPKeys...)
|
||||||
|
cmd := backgroundCmd("gpg", args...)
|
||||||
|
res, err := cmd.CombinedOutput()
|
||||||
|
log.Debug(string(res))
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
log.Warningf("Unable to import keys: %s", string(res))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func increasePkgRel(pkg *BuildPackage) {
|
||||||
|
f, err := os.OpenFile(pkg.Pkgbuild, os.O_RDWR, os.ModePerm)
|
||||||
|
check(err)
|
||||||
|
defer f.Close()
|
||||||
|
|
||||||
|
fStr, err := io.ReadAll(f)
|
||||||
|
check(err)
|
||||||
|
|
||||||
|
nStr := rePkgRel.ReplaceAllLiteralString(string(fStr), "pkgrel="+pkg.Srcinfo.Pkgrel+".1")
|
||||||
|
_, err = f.Seek(0, 0)
|
||||||
|
check(err)
|
||||||
|
check(f.Truncate(0))
|
||||||
|
|
||||||
|
_, err = f.WriteString(nStr)
|
||||||
|
check(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func gitClean(pkg *BuildPackage) {
|
||||||
|
cmd := backgroundCmd("sh", "-c", "cd "+filepath.Dir(pkg.Pkgbuild)+"&&git clean -xdff")
|
||||||
|
res, err := cmd.CombinedOutput()
|
||||||
|
log.Debug(string(res))
|
||||||
|
check(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *BuildManager) buildWorker(id int) {
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case pkg := <-b.toBuild:
|
||||||
|
if b.exit {
|
||||||
|
continue
|
||||||
|
} else {
|
||||||
|
b.wg.Add(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
start := time.Now()
|
||||||
|
|
||||||
|
log.Infof("[%s/%s] Build starting", pkg.FullRepo, pkg.Pkgbase)
|
||||||
|
|
||||||
|
importKeys(pkg)
|
||||||
|
increasePkgRel(pkg)
|
||||||
|
pkg.PkgFiles = []string{}
|
||||||
|
|
||||||
|
cmd := backgroundCmd("sh", "-c",
|
||||||
|
"cd "+filepath.Dir(pkg.Pkgbuild)+"&&makechrootpkg -c -D "+conf.Basedir.Makepkg+" -l worker-"+strconv.Itoa(id)+" -r "+conf.Basedir.Chroot+" -- "+
|
||||||
|
"--config "+filepath.Join(conf.Basedir.Makepkg, fmt.Sprintf("makepkg-%s.conf", pkg.March)))
|
||||||
|
res, err := cmd.CombinedOutput()
|
||||||
|
if err != nil {
|
||||||
|
log.Warningf("[%s/%s] Build failed, exit code %d", pkg.FullRepo, pkg.Pkgbase, cmd.ProcessState.ExitCode())
|
||||||
|
|
||||||
|
b.failedMutex.Lock()
|
||||||
|
f, err := os.OpenFile(filepath.Join(conf.Basedir.Repo, pkg.FullRepo+"_failed.txt"), os.O_WRONLY|os.O_APPEND|os.O_CREATE, os.ModePerm)
|
||||||
|
check(err)
|
||||||
|
|
||||||
|
if pkg.Srcinfo.Epoch != "" {
|
||||||
|
_, err := f.WriteString(fmt.Sprintf("%s==%s:%s-%s\n", pkg.Pkgbase, pkg.Srcinfo.Epoch, pkg.Srcinfo.Pkgver, pkg.Srcinfo.Pkgrel))
|
||||||
|
check(err)
|
||||||
|
} else {
|
||||||
|
_, err := f.WriteString(fmt.Sprintf("%s==%s-%s\n", pkg.Pkgbase, pkg.Srcinfo.Pkgver, pkg.Srcinfo.Pkgrel))
|
||||||
|
check(err)
|
||||||
|
}
|
||||||
|
f.Close()
|
||||||
|
b.failedMutex.Unlock()
|
||||||
|
|
||||||
|
check(os.MkdirAll(filepath.Join(conf.Basedir.Repo, "logs"), os.ModePerm))
|
||||||
|
check(os.WriteFile(filepath.Join(conf.Basedir.Repo, "logs", pkg.Pkgbase+".log"), res, os.ModePerm))
|
||||||
|
|
||||||
|
gitClean(pkg)
|
||||||
|
b.wg.Done()
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
pkgFiles, err := filepath.Glob(filepath.Join(filepath.Dir(pkg.Pkgbuild), "*.pkg.tar.zst"))
|
||||||
|
check(err)
|
||||||
|
log.Debug(pkgFiles)
|
||||||
|
|
||||||
|
if len(pkgFiles) == 0 {
|
||||||
|
log.Warningf("No packages found after building %s. Abort build.", pkg.Pkgbase)
|
||||||
|
|
||||||
|
gitClean(pkg)
|
||||||
|
b.wg.Done()
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, file := range pkgFiles {
|
||||||
|
cmd = backgroundCmd("gpg", "--batch", "--detach-sign", file)
|
||||||
|
res, err := cmd.CombinedOutput()
|
||||||
|
log.Debug(string(res))
|
||||||
|
if err != nil {
|
||||||
|
log.Warningf("Failed to sign %s: %s", pkg.Pkgbase, err)
|
||||||
|
b.wg.Done()
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
copyFiles, err := filepath.Glob(filepath.Join(filepath.Dir(pkg.Pkgbuild), "*.pkg.tar.zst*"))
|
||||||
|
check(err)
|
||||||
|
|
||||||
|
for _, file := range copyFiles {
|
||||||
|
_, err = copyFile(file, filepath.Join(conf.Basedir.Repo, pkg.FullRepo, "os", conf.Arch, filepath.Base(file)))
|
||||||
|
if err != nil {
|
||||||
|
check(err)
|
||||||
|
b.wg.Done()
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if filepath.Ext(file) != ".sig" {
|
||||||
|
pkg.PkgFiles = append(pkg.PkgFiles, filepath.Join(conf.Basedir.Repo, pkg.FullRepo, "os", conf.Arch, filepath.Base(file)))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
b.toRepoAdd <- pkg
|
||||||
|
|
||||||
|
if _, err := os.Stat(filepath.Join(conf.Basedir.Repo, "logs", pkg.Pkgbase+".log")); err == nil {
|
||||||
|
check(os.Remove(filepath.Join(conf.Basedir.Repo, "logs", pkg.Pkgbase+".log")))
|
||||||
|
}
|
||||||
|
|
||||||
|
gitClean(pkg)
|
||||||
|
log.Infof("[%s/%s] Build successful (%s)", pkg.FullRepo, pkg.Pkgbase, time.Now().Sub(start))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *BuildManager) parseWorker() {
|
||||||
|
for {
|
||||||
|
if b.exit {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case pkg := <-b.toParse:
|
||||||
|
cmd := backgroundCmd("sh", "-c", "cd "+filepath.Dir(pkg.Pkgbuild)+"&&"+"makepkg --printsrcinfo")
|
||||||
|
res, err := cmd.Output()
|
||||||
|
if err != nil {
|
||||||
|
log.Warningf("Failed generate SRCINFO for %s: %s", pkg.Pkgbase, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
info, err := srcinfo.Parse(string(res))
|
||||||
|
if err != nil {
|
||||||
|
log.Warningf("Failed to parse SRCINFO for %s: %s", pkg.Pkgbase, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
pkg.Srcinfo = info
|
||||||
|
|
||||||
|
if contains(info.Arch, "any") || contains(conf.Blacklist, info.Pkgbase) {
|
||||||
|
log.Infof("Skipped %s: blacklisted or any-Package", info.Pkgbase)
|
||||||
|
b.toPurge <- pkg
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if isPkgFailed(pkg) {
|
||||||
|
log.Infof("Skipped %s: failed build", info.Pkgbase)
|
||||||
|
b.toPurge <- pkg
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
var pkgVer string
|
||||||
|
if pkg.Srcinfo.Epoch == "" {
|
||||||
|
pkgVer = pkg.Srcinfo.Pkgver + "-" + pkg.Srcinfo.Pkgrel
|
||||||
|
} else {
|
||||||
|
pkgVer = pkg.Srcinfo.Epoch + ":" + pkg.Srcinfo.Pkgver + "-" + pkg.Srcinfo.Pkgrel
|
||||||
|
}
|
||||||
|
|
||||||
|
repoVer := getVersionFromRepo(pkg)
|
||||||
|
if repoVer != "" && alpm.VerCmp(repoVer, pkgVer) > 0 {
|
||||||
|
log.Debugf("Skipped %s: Version in repo higher than in PKGBUILD (%s < %s)", info.Pkgbase, pkgVer, repoVer)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
b.toBuild <- pkg
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func findPkgFiles(pkg *BuildPackage) {
|
||||||
|
pkgs, err := os.ReadDir(filepath.Join(conf.Basedir.Repo, pkg.FullRepo, "os", conf.Arch))
|
||||||
|
check(err)
|
||||||
|
|
||||||
|
var fPkg []string
|
||||||
|
for _, file := range pkgs {
|
||||||
|
if !file.IsDir() && !strings.HasSuffix(file.Name(), ".sig") {
|
||||||
|
matches := rePkgFile.FindStringSubmatch(file.Name())
|
||||||
|
|
||||||
|
var realPkgs []string
|
||||||
|
for _, realPkg := range pkg.Srcinfo.Packages {
|
||||||
|
realPkgs = append(realPkgs, realPkg.Pkgname)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(matches) > 1 && contains(realPkgs, matches[1]) {
|
||||||
|
fPkg = append(fPkg, filepath.Join(conf.Basedir.Repo, pkg.FullRepo, "os", conf.Arch, file.Name()))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pkg.PkgFiles = fPkg
|
||||||
|
}
|
||||||
|
|
||||||
|
func getVersionFromRepo(pkg *BuildPackage) string {
|
||||||
|
findPkgFiles(pkg)
|
||||||
|
|
||||||
|
if len(pkg.PkgFiles) == 0 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
fNameSplit := strings.Split(pkg.PkgFiles[0], "-")
|
||||||
|
return fNameSplit[len(fNameSplit)-3] + "-" + fNameSplit[len(fNameSplit)-2]
|
||||||
|
}
|
||||||
|
|
||||||
|
func isPkgFailed(pkg *BuildPackage) bool {
|
||||||
|
buildManager.failedMutex.Lock()
|
||||||
|
defer buildManager.failedMutex.Unlock()
|
||||||
|
|
||||||
|
file, err := os.OpenFile(filepath.Join(conf.Basedir.Repo, pkg.FullRepo+"_failed.txt"), os.O_RDWR|os.O_CREATE, os.ModePerm)
|
||||||
|
check(err)
|
||||||
|
defer file.Close()
|
||||||
|
|
||||||
|
failed := false
|
||||||
|
var newContent []string
|
||||||
|
scanner := bufio.NewScanner(file)
|
||||||
|
for scanner.Scan() {
|
||||||
|
splitPkg := strings.Split(scanner.Text(), "==")
|
||||||
|
|
||||||
|
if splitPkg[0] == pkg.Pkgbase {
|
||||||
|
var pkgVer string
|
||||||
|
if pkg.Srcinfo.Epoch == "" {
|
||||||
|
pkgVer = pkg.Srcinfo.Pkgver + "-" + pkg.Srcinfo.Pkgrel
|
||||||
|
} else {
|
||||||
|
pkgVer = pkg.Srcinfo.Epoch + ":" + pkg.Srcinfo.Pkgver + "-" + pkg.Srcinfo.Pkgrel
|
||||||
|
}
|
||||||
|
|
||||||
|
if alpm.VerCmp(splitPkg[1], pkgVer) < 0 {
|
||||||
|
failed = false
|
||||||
|
} else {
|
||||||
|
failed = true
|
||||||
|
newContent = append(newContent, scanner.Text())
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
newContent = append(newContent, scanner.Text())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
check(scanner.Err())
|
||||||
|
|
||||||
|
_, err = file.Seek(0, 0)
|
||||||
|
check(err)
|
||||||
|
check(file.Truncate(0))
|
||||||
|
_, err = file.WriteString(strings.Join(newContent, "\n"))
|
||||||
|
check(err)
|
||||||
|
|
||||||
|
return failed
|
||||||
|
}
|
||||||
|
|
||||||
|
func setupChroot() {
|
||||||
|
if _, err := os.Stat(filepath.Join(conf.Basedir.Chroot, orgChrootName)); err == nil {
|
||||||
|
//goland:noinspection SpellCheckingInspection
|
||||||
|
cmd := backgroundCmd("arch-nspawn", filepath.Join(conf.Basedir.Chroot, orgChrootName), "pacman", "-Syuu", "--noconfirm")
|
||||||
|
res, err := cmd.CombinedOutput()
|
||||||
|
log.Debug(string(res))
|
||||||
|
check(err)
|
||||||
|
} else if os.IsNotExist(err) {
|
||||||
|
err := os.MkdirAll(conf.Basedir.Chroot, os.ModePerm)
|
||||||
|
check(err)
|
||||||
|
|
||||||
|
cmd := backgroundCmd("mkarchroot", "-C", pacmanConf, filepath.Join(conf.Basedir.Chroot, orgChrootName), "base-devel")
|
||||||
|
res, err := cmd.CombinedOutput()
|
||||||
|
log.Debug(string(res))
|
||||||
|
check(err)
|
||||||
|
} else {
|
||||||
|
check(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *BuildManager) repoWorker() {
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case pkg := <-b.toRepoAdd:
|
||||||
|
args := []string{"-s", "-v", "-p", "-n", filepath.Join(conf.Basedir.Repo, pkg.FullRepo, "os", conf.Arch, pkg.FullRepo) + ".db.tar.xz"}
|
||||||
|
args = append(args, pkg.PkgFiles...)
|
||||||
|
cmd := backgroundCmd("repo-add", args...)
|
||||||
|
res, err := cmd.CombinedOutput()
|
||||||
|
log.Debug(string(res))
|
||||||
|
check(err)
|
||||||
|
|
||||||
|
cmd = backgroundCmd("paccache",
|
||||||
|
"-rc", filepath.Join(conf.Basedir.Repo, pkg.FullRepo, "os", conf.Arch),
|
||||||
|
"-k", "1")
|
||||||
|
res, err = cmd.CombinedOutput()
|
||||||
|
log.Debug(string(res))
|
||||||
|
check(err)
|
||||||
|
b.wg.Done()
|
||||||
|
case pkg := <-b.toPurge:
|
||||||
|
if _, err := os.Stat(filepath.Join(conf.Basedir.Repo, pkg.FullRepo, "os", conf.Arch, pkg.FullRepo) + ".db.tar.xz"); err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if len(pkg.PkgFiles) == 0 {
|
||||||
|
findPkgFiles(pkg)
|
||||||
|
}
|
||||||
|
|
||||||
|
var realPkgs []string
|
||||||
|
for _, realPkg := range pkg.Srcinfo.Packages {
|
||||||
|
realPkgs = append(realPkgs, realPkg.Pkgname)
|
||||||
|
}
|
||||||
|
|
||||||
|
args := []string{"-s", "-v", filepath.Join(conf.Basedir.Repo, pkg.FullRepo, "os", conf.Arch, pkg.FullRepo) + ".db.tar.xz"}
|
||||||
|
args = append(args, realPkgs...)
|
||||||
|
cmd := backgroundCmd("repo-remove", args...)
|
||||||
|
res, err := cmd.CombinedOutput()
|
||||||
|
log.Debug(string(res))
|
||||||
|
if err != nil && cmd.ProcessState.ExitCode() == 1 {
|
||||||
|
log.Debugf("Deleteing package %s failed: Package not found in database", pkg.Pkgbase)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, file := range pkg.PkgFiles {
|
||||||
|
check(os.Remove(file))
|
||||||
|
check(os.Remove(file + ".sig"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func backgroundCmd(name string, arg ...string) *exec.Cmd {
|
||||||
|
cmd := exec.Command(name, arg...)
|
||||||
|
cmd.SysProcAttr = &syscall.SysProcAttr{
|
||||||
|
Foreground: false,
|
||||||
|
Setsid: true,
|
||||||
|
}
|
||||||
|
return cmd
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *BuildManager) syncWorker() {
|
||||||
|
check(os.MkdirAll(conf.Basedir.Upstream, os.ModePerm))
|
||||||
|
|
||||||
|
for i := 0; i < conf.Build.Worker; i++ {
|
||||||
|
go b.buildWorker(i)
|
||||||
|
go b.parseWorker()
|
||||||
|
}
|
||||||
|
|
||||||
|
for {
|
||||||
|
b.wg.Wait()
|
||||||
|
for gitDir, gitURL := range conf.Svn2git {
|
||||||
|
gitPath := filepath.Join(conf.Basedir.Upstream, gitDir)
|
||||||
|
|
||||||
|
if _, err := os.Stat(gitPath); os.IsNotExist(err) {
|
||||||
|
cmd := backgroundCmd("git", "clone", "--depth=1", gitURL, gitPath)
|
||||||
|
res, err := cmd.CombinedOutput()
|
||||||
|
log.Debug(string(res))
|
||||||
|
check(err)
|
||||||
|
} else if err == nil {
|
||||||
|
cmd := backgroundCmd("sh", "-c", "cd "+gitPath+" && git clean -xdff")
|
||||||
|
res, err := cmd.CombinedOutput()
|
||||||
|
log.Debug(string(res))
|
||||||
|
check(err)
|
||||||
|
|
||||||
|
cmd = backgroundCmd("sh", "-c", "cd "+gitPath+" && git reset --hard")
|
||||||
|
res, err = cmd.CombinedOutput()
|
||||||
|
log.Debug(string(res))
|
||||||
|
check(err)
|
||||||
|
|
||||||
|
cmd = backgroundCmd("sh", "-c", "cd "+gitPath+" && git pull")
|
||||||
|
res, err = cmd.CombinedOutput()
|
||||||
|
log.Debug(string(res))
|
||||||
|
check(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pkgBuilds, err := filepathx.Glob(filepath.Join(conf.Basedir.Upstream, "/**/PKGBUILD"))
|
||||||
|
check(err)
|
||||||
|
|
||||||
|
for _, pkgbuild := range pkgBuilds {
|
||||||
|
if b.exit {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
sPkgbuild := strings.Split(pkgbuild, "/")
|
||||||
|
repo := sPkgbuild[len(sPkgbuild)-2]
|
||||||
|
|
||||||
|
if repo == "trunk" || !contains(conf.Repos, strings.Split(repo, "-")[0]) || strings.Contains(repo, "i686") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, march := range conf.March {
|
||||||
|
b.toParse <- &BuildPackage{
|
||||||
|
Pkgbuild: pkgbuild,
|
||||||
|
Pkgbase: sPkgbuild[len(sPkgbuild)-4],
|
||||||
|
Repo: strings.Split(repo, "-")[0],
|
||||||
|
March: march,
|
||||||
|
FullRepo: strings.Split(repo, "-")[0] + "-" + march,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
time.Sleep(5 * time.Minute)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
killSignals := make(chan os.Signal, 1)
|
||||||
|
signal.Notify(killSignals, syscall.SIGINT, syscall.SIGTERM)
|
||||||
|
|
||||||
|
confStr, err := os.ReadFile("config.yaml")
|
||||||
|
check(err)
|
||||||
|
|
||||||
|
err = yaml.Unmarshal(confStr, &conf)
|
||||||
|
check(err)
|
||||||
|
|
||||||
|
lvl, err := log.ParseLevel(conf.Logging.Level)
|
||||||
|
check(err)
|
||||||
|
log.SetLevel(lvl)
|
||||||
|
|
||||||
|
err = os.MkdirAll(conf.Basedir.Repo, os.ModePerm)
|
||||||
|
check(err)
|
||||||
|
|
||||||
|
buildManager = BuildManager{
|
||||||
|
toBuild: make(chan *BuildPackage),
|
||||||
|
toParse: make(chan *BuildPackage, conf.Build.Worker),
|
||||||
|
toPurge: make(chan *BuildPackage),
|
||||||
|
toRepoAdd: make(chan *BuildPackage, conf.Build.Worker),
|
||||||
|
exit: false,
|
||||||
|
wg: sync.WaitGroup{},
|
||||||
|
failedMutex: sync.RWMutex{},
|
||||||
|
}
|
||||||
|
|
||||||
|
setupChroot()
|
||||||
|
syncMarchs()
|
||||||
|
|
||||||
|
go buildManager.repoWorker()
|
||||||
|
go buildManager.syncWorker()
|
||||||
|
|
||||||
|
<-killSignals
|
||||||
|
|
||||||
|
buildManager.exit = true
|
||||||
|
buildManager.wg.Wait()
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user