From e28d85b10db874c67b5ead4e5596227f2e45a4d9 Mon Sep 17 00:00:00 2001 From: Giovanni Harting <539@idlegandalf.com> Date: Tue, 27 Jul 2021 02:43:30 +0200 Subject: [PATCH] Added pre-build dependency version check (#32) Fixes #15. Uses ALPM's `satisfies` to resolve dependencies and then compares resolved dependency from local sync database with svn2git's version. Reviewed-on: https://git.harting.dev/anonfunc/ALHP.GO/pulls/32 Co-authored-by: Giovanni Harting <539@idlegandalf.com> Co-committed-by: Giovanni Harting <539@idlegandalf.com> --- config.yaml | 16 +- go.mod | 2 +- go.sum | 19 +- main.go | 423 +++---------------------------------- utils.go | 591 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 639 insertions(+), 412 deletions(-) create mode 100644 utils.go diff --git a/config.yaml b/config.yaml index 978fa7b..3d252cf 100644 --- a/config.yaml +++ b/config.yaml @@ -19,10 +19,16 @@ march: - x86-64-v3 blacklist: - - pacman - - tensorflow - - tensorflow-cuda - - gcc + packages: + - pacman + - tensorflow + - tensorflow-cuda + - gcc + repo: + - testing + - i686 + - staging + - unstable build: worker: 4 @@ -39,4 +45,4 @@ status: unknown: "dark" logging: - level: DEBUG \ No newline at end of file + level: INFO \ No newline at end of file diff --git a/go.mod b/go.mod index e1912a8..d34b749 100644 --- a/go.mod +++ b/go.mod @@ -5,11 +5,11 @@ go 1.16 require ( entgo.io/ent v0.8.0 github.com/Jguer/go-alpm/v2 v2.0.5 + github.com/Morganamilo/go-pacmanconf v0.0.0-20210502114700-cff030e927a5 github.com/Morganamilo/go-srcinfo v1.0.0 github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect github.com/mattn/go-sqlite3 v1.14.6 github.com/sirupsen/logrus v1.8.1 github.com/wercker/journalhook v0.0.0-20180428041537-5d0a5ae867b3 - github.com/yargevad/filepathx v1.0.0 gopkg.in/yaml.v2 v2.4.0 ) diff --git a/go.sum b/go.sum index e29bc0b..96dcc6f 100644 --- a/go.sum +++ b/go.sum @@ -19,6 +19,8 @@ github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20O github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/Jguer/go-alpm/v2 v2.0.5 h1:1TZxkvCIfTOhjhxGy/Z1FNSeuY9DXBKF5qxUoj0IZ0A= github.com/Jguer/go-alpm/v2 v2.0.5/go.mod h1:zU4iKCtNkDARfj5BrKJXYAQ5nIjtZbySfa0paboSmTQ= +github.com/Morganamilo/go-pacmanconf v0.0.0-20210502114700-cff030e927a5 h1:TMscPjkb1ThXN32LuFY5bEYIcXZx3YlwzhS1GxNpn/c= +github.com/Morganamilo/go-pacmanconf v0.0.0-20210502114700-cff030e927a5/go.mod h1:Hk55m330jNiwxRodIlMCvw5iEyoRUCIY64W1p9D+tHc= github.com/Morganamilo/go-srcinfo v1.0.0 h1:Wh4nEF+HJWo+29hnxM18Q2hi+DUf0GejS13+Wg+dzmI= github.com/Morganamilo/go-srcinfo v1.0.0/go.mod h1:MP6VGY1NNpVUmYIEgoM9acix95KQqIRyqQ0hCLsyYUY= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= @@ -60,7 +62,6 @@ github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-openapi/inflect v0.19.0 h1:9jCH9scKIbHeV9m12SmPilScz6krDxKRasNNSNPXu/4= github.com/go-openapi/inflect v0.19.0/go.mod h1:lHpZVlpIQqLyKwJ4N+YSc9hchQy/i12fJykb83CRBH4= github.com/go-sql-driver/mysql v1.5.1-0.20200311113236-681ffa848bae/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -127,10 +128,6 @@ github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2p github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= -github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= -github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= -github.com/jinzhu/now v1.1.1 h1:g39TucaRWyV3dwDO++eEc6qf8TVIQ/Da48WmqjZ3i7E= -github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -151,9 +148,7 @@ github.com/lib/pq v1.10.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -github.com/mattn/go-sqlite3 v1.14.5/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGwNd0Lj+XmI= github.com/mattn/go-sqlite3 v1.14.6 h1:dNPt6NO46WmLVt2DLNpwczCmdV5boIZ6g/tlDrlRUbg= github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= @@ -173,7 +168,6 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= @@ -208,11 +202,9 @@ github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4k github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v1.1.3 h1:xghbfqPkxzxP3C/f3n5DdpAbdKLj4ZE4BWQI362l53M= github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -263,7 +255,6 @@ golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -334,12 +325,10 @@ golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= @@ -390,10 +379,6 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gorm.io/driver/sqlite v1.1.4 h1:PDzwYE+sI6De2+mxAneV9Xs11+ZyKV6oxD3wDGkaNvM= -gorm.io/driver/sqlite v1.1.4/go.mod h1:mJCeTFr7+crvS+TRnWc5Z3UvwxUN1BGBLMrf5LA9DYw= -gorm.io/gorm v1.20.7 h1:rMS4CL3pNmYq1V5/X+nHHjh1Dx6dnf27+Cai5zabo+M= -gorm.io/gorm v1.20.7/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/main.go b/main.go index 2582eaf..fe62188 100644 --- a/main.go +++ b/main.go @@ -4,7 +4,6 @@ import ( "ALHP.go/ent" "ALHP.go/ent/dbpackage" "ALHP.go/ent/migrate" - "bufio" "bytes" "context" "fmt" @@ -13,10 +12,8 @@ import ( _ "github.com/mattn/go-sqlite3" log "github.com/sirupsen/logrus" "github.com/wercker/journalhook" - "github.com/yargevad/filepathx" "gopkg.in/yaml.v2" "html/template" - "io" "math/rand" "os" "os/exec" @@ -24,7 +21,6 @@ import ( "path/filepath" "regexp" "runtime" - "sort" "strconv" "strings" "sync" @@ -39,19 +35,11 @@ const ( orgChrootName = "root" ) -const ( - SKIPPED = iota - FAILED = iota - BUILD = iota - QUEUED = iota - BUILDING = iota - LATEST = iota - UNKNOWN = iota -) - var ( conf = Conf{} repos []string + alpmHandle *alpm.Handle + alpmLock sync.RWMutex reMarch = regexp.MustCompile(`(-march=)(.+?) `) rePkgRel = regexp.MustCompile(`(?m)^pkgrel\s*=\s*(.+)$`) rePkgFile = regexp.MustCompile(`^(.*)-.*-.*-(?:x86_64|any)\.pkg\.tar\.zst(?:\.sig)*$`) @@ -60,231 +48,6 @@ var ( dbLock sync.RWMutex ) -type BuildPackage struct { - Pkgbase string - Pkgbuild string - Srcinfo *srcinfo.Srcinfo - PkgFiles []string - Repo string - March string - FullRepo string - Version string -} - -type BuildManager struct { - build chan *BuildPackage - parse chan *BuildPackage - repoPurge map[string]chan *BuildPackage - repoAdd map[string]chan *BuildPackage - exit bool - buildWG sync.WaitGroup - parseWG sync.WaitGroup - repoWG sync.WaitGroup - failedMutex sync.RWMutex - buildProcesses []*os.Process - buildProcMutex sync.RWMutex -} - -type Conf struct { - Arch string - Repos, March, Blacklist []string - Svn2git map[string]string - Basedir struct { - Repo, Chroot, Makepkg, Upstream, Db string - } - Build struct { - Worker int - Makej int - } - Logging struct { - Level string - } - Status struct { - Class struct { - Skipped, Queued, Latest, Failed, Signing, Building, Unknown string - } - } -} - -func check(e error) { - if e != nil { - panic(e) - } -} - -func contains(s interface{}, str string) bool { - switch v := s.(type) { - case []string: - if i := find(v, str); i != -1 { - return true - } - case []srcinfo.ArchString: - var n []string - for _, as := range v { - n = append(n, as.Value) - } - - if i := find(n, str); i != -1 { - return true - } - default: - return false - } - - return false -} - -func find(s []string, str string) int { - for i, v := range s { - if v == str { - return i - } - } - - return -1 -} - -func copyFile(src, dst string) (int64, error) { - sourceFileStat, err := os.Stat(src) - if err != nil { - return 0, err - } - - if !sourceFileStat.Mode().IsRegular() { - return 0, fmt.Errorf("%s is not a regular file", src) - } - - source, err := os.Open(src) - if err != nil { - return 0, err - } - defer func(source *os.File) { - check(source.Close()) - }(source) - - destination, err := os.Create(dst) - if err != nil { - return 0, err - } - defer func(destination *os.File) { - check(destination.Close()) - }(destination) - nBytes, err := io.Copy(destination, source) - return nBytes, err -} - -//goland:noinspection SpellCheckingInspection -func setupMakepkg(march string) { - lMakepkg := filepath.Join(conf.Basedir.Makepkg, fmt.Sprintf("makepkg-%s.conf", march)) - - check(os.MkdirAll(conf.Basedir.Makepkg, os.ModePerm)) - t, err := os.ReadFile(makepkgConf) - check(err) - makepkgStr := string(t) - - makepkgStr = strings.ReplaceAll(makepkgStr, "-mtune=generic", "") - makepkgStr = strings.ReplaceAll(makepkgStr, "-O2", "-O3") - makepkgStr = strings.ReplaceAll(makepkgStr, " check ", " !check ") - makepkgStr = strings.ReplaceAll(makepkgStr, " color ", " !color ") - makepkgStr = strings.ReplaceAll(makepkgStr, "#MAKEFLAGS=\"-j2\"", "MAKEFLAGS=\"-j"+strconv.Itoa(conf.Build.Makej)+"\"") - makepkgStr = reMarch.ReplaceAllString(makepkgStr, "${1}"+march) - - check(os.WriteFile(lMakepkg, []byte(makepkgStr), os.ModePerm)) -} - -func syncMarchs() { - files, err := os.ReadDir(conf.Basedir.Repo) - check(err) - - var eRepos []string - for _, file := range files { - if file.Name() != "." && file.Name() != logDir && file.IsDir() { - eRepos = append(eRepos, file.Name()) - } - } - - for _, march := range conf.March { - setupMakepkg(march) - for _, repo := range conf.Repos { - tRepo := fmt.Sprintf("%s-%s", repo, march) - repos = append(repos, tRepo) - buildManager.repoAdd[tRepo] = make(chan *BuildPackage, conf.Build.Worker) - buildManager.repoPurge[tRepo] = make(chan *BuildPackage, 10000) - go buildManager.repoWorker(tRepo) - - if _, err := os.Stat(filepath.Join(filepath.Join(conf.Basedir.Repo, tRepo, "os", conf.Arch))); os.IsNotExist(err) { - log.Debugf("Creating path %s", filepath.Join(conf.Basedir.Repo, tRepo, "os", conf.Arch)) - check(os.MkdirAll(filepath.Join(conf.Basedir.Repo, tRepo, "os", conf.Arch), os.ModePerm)) - } - - if i := find(eRepos, tRepo); i != -1 { - eRepos = append(eRepos[:i], eRepos[i+1:]...) - } - } - } - - log.Infof("Repos: %s", repos) - - for _, repo := range eRepos { - log.Infof("Removing old repo %s", repo) - check(os.RemoveAll(filepath.Join(conf.Basedir.Repo, repo))) - } -} - -func importKeys(pkg *BuildPackage) { - if pkg.Srcinfo.ValidPGPKeys != nil { - args := []string{"--keyserver", "keyserver.ubuntu.com", "--recv-keys"} - args = append(args, pkg.Srcinfo.ValidPGPKeys...) - cmd := exec.Command("gpg", args...) - res, err := cmd.CombinedOutput() - log.Debug(string(res)) - - if err != nil { - log.Warningf("Unable to import keys: %s", string(res)) - } - } -} - -func packages2string(pkgs []srcinfo.Package) []string { - var sPkgs []string - for _, p := range pkgs { - sPkgs = append(sPkgs, p.Pkgname) - } - - return sPkgs -} - -func increasePkgRel(pkg *BuildPackage) { - f, err := os.OpenFile(pkg.Pkgbuild, os.O_RDWR, os.ModePerm) - check(err) - defer func(f *os.File) { - check(f.Close()) - }(f) - - fStr, err := io.ReadAll(f) - check(err) - - nStr := rePkgRel.ReplaceAllLiteralString(string(fStr), "pkgrel="+pkg.Srcinfo.Pkgrel+".1") - _, err = f.Seek(0, 0) - check(err) - check(f.Truncate(0)) - - _, err = f.WriteString(nStr) - check(err) - - pkg.Version = pkg.Version + ".1" -} - -func gitClean(pkg *BuildPackage) { - cmd := exec.Command("sudo", "git_clean.sh", filepath.Dir(pkg.Pkgbuild)) - res, err := cmd.CombinedOutput() - if err != nil { - log.Warningf("git clean failed with %v:\n%s", err, res) - } else { - log.Debug(string(res)) - } -} - func (b *BuildManager) buildWorker(id int) { err := syscall.Setpriority(syscall.PRIO_PROCESS, 0, 18) if err != nil { @@ -310,8 +73,17 @@ func (b *BuildManager) buildWorker(id int) { dbPkg.Update().SetStatus(BUILDING).SaveX(context.Background()) dbLock.Unlock() - importKeys(pkg) - increasePkgRel(pkg) + err := importKeys(pkg) + if err != nil { + log.Warningf("[%s/%s] Failed to import pgp keys: %v", pkg.FullRepo, pkg.Pkgbase, err) + } + + err = increasePkgRel(pkg) + if err != nil { + log.Errorf("[%s/%s] Failed to increase pkgrel: %v", pkg.FullRepo, pkg.Pkgbase, err) + b.buildWG.Done() + continue + } pkg.PkgFiles = []string{} cmd := exec.Command("sh", "-c", @@ -351,13 +123,8 @@ func (b *BuildManager) buildWorker(id int) { f, err := os.OpenFile(filepath.Join(conf.Basedir.Repo, pkg.FullRepo+"_failed.txt"), os.O_WRONLY|os.O_APPEND|os.O_CREATE|os.O_SYNC, os.ModePerm) check(err) - if pkg.Srcinfo.Epoch != "" { - _, err := f.WriteString(fmt.Sprintf("%s==%s:%s-%s\n", pkg.Pkgbase, pkg.Srcinfo.Epoch, pkg.Srcinfo.Pkgver, pkg.Srcinfo.Pkgrel)) - check(err) - } else { - _, err := f.WriteString(fmt.Sprintf("%s==%s-%s\n", pkg.Pkgbase, pkg.Srcinfo.Pkgver, pkg.Srcinfo.Pkgrel)) - check(err) - } + _, err = f.WriteString(fmt.Sprintf("%s==%s\n", pkg.Pkgbase, constructVersion(pkg.Srcinfo.Pkgver, pkg.Srcinfo.Pkgrel, pkg.Srcinfo.Epoch))) + check(err) check(f.Close()) b.failedMutex.Unlock() @@ -430,17 +197,6 @@ func (b *BuildManager) buildWorker(id int) { } } -func getDbPackage(pkg *BuildPackage) *ent.DbPackage { - dbLock.Lock() - dbPkg, err := db.DbPackage.Query().Where(dbpackage.Pkgbase(pkg.Pkgbase)).Only(context.Background()) - if err != nil { - dbPkg = db.DbPackage.Create().SetPkgbase(pkg.Pkgbase).SetMarch(pkg.March).SetPackages(packages2string(pkg.Srcinfo.Packages)).SetRepository(pkg.Repo).SaveX(context.Background()) - } - dbLock.Unlock() - - return dbPkg -} - func (b *BuildManager) parseWorker() { for { if b.exit { @@ -463,11 +219,7 @@ func (b *BuildManager) parseWorker() { continue } pkg.Srcinfo = info - if pkg.Srcinfo.Epoch == "" { - pkg.Version = pkg.Srcinfo.Pkgver + "-" + pkg.Srcinfo.Pkgrel - } else { - pkg.Version = pkg.Srcinfo.Epoch + ":" + pkg.Srcinfo.Pkgver + "-" + pkg.Srcinfo.Pkgrel - } + pkg.Version = constructVersion(pkg.Srcinfo.Pkgver, pkg.Srcinfo.Pkgrel, pkg.Srcinfo.Epoch) dbPkg := getDbPackage(pkg) dbLock.Lock() @@ -481,7 +233,7 @@ func (b *BuildManager) parseWorker() { dbPkg = dbPkg.Update().SetStatus(SKIPPED).SetSkipReason("arch = any").SaveX(context.Background()) dbLock.Unlock() skipping = true - } else if contains(conf.Blacklist, info.Pkgbase) { + } else if contains(conf.Blacklist.Packages, info.Pkgbase) { log.Debugf("Skipped %s: blacklisted package", info.Pkgbase) dbLock.Lock() dbPkg = dbPkg.Update().SetStatus(SKIPPED).SetSkipReason("blacklisted").SaveX(context.Background()) @@ -527,114 +279,21 @@ func (b *BuildManager) parseWorker() { dbPkg = dbPkg.Update().SetStatus(QUEUED).SaveX(context.Background()) dbLock.Unlock() + isLatest, err := isMirrorLatest(alpmHandle, pkg) + check(err) + + if !isLatest { + log.Infof("Delayed %s: not all dependencies are up to date", info.Pkgbase) + b.parseWG.Done() + continue + } + b.parseWG.Done() b.build <- pkg } } } -func findPkgFiles(pkg *BuildPackage) { - pkgs, err := os.ReadDir(filepath.Join(conf.Basedir.Repo, pkg.FullRepo, "os", conf.Arch)) - check(err) - - var fPkg []string - for _, file := range pkgs { - if !file.IsDir() && !strings.HasSuffix(file.Name(), ".sig") { - matches := rePkgFile.FindStringSubmatch(file.Name()) - - var realPkgs []string - for _, realPkg := range pkg.Srcinfo.Packages { - realPkgs = append(realPkgs, realPkg.Pkgname) - } - - if len(matches) > 1 && contains(realPkgs, matches[1]) { - fPkg = append(fPkg, filepath.Join(conf.Basedir.Repo, pkg.FullRepo, "os", conf.Arch, file.Name())) - } - } - } - - pkg.PkgFiles = fPkg -} - -func getVersionFromRepo(pkg *BuildPackage) string { - findPkgFiles(pkg) - - if len(pkg.PkgFiles) == 0 { - return "" - } - - fNameSplit := strings.Split(pkg.PkgFiles[0], "-") - return fNameSplit[len(fNameSplit)-3] + "-" + fNameSplit[len(fNameSplit)-2] -} - -func isPkgFailed(pkg *BuildPackage) bool { - buildManager.failedMutex.Lock() - defer buildManager.failedMutex.Unlock() - - file, err := os.OpenFile(filepath.Join(conf.Basedir.Repo, pkg.FullRepo+"_failed.txt"), os.O_RDWR|os.O_CREATE|os.O_SYNC, os.ModePerm) - check(err) - defer func(file *os.File) { - check(file.Close()) - }(file) - - failed := false - var newContent []string - scanner := bufio.NewScanner(file) - for scanner.Scan() { - line := scanner.Text() - splitPkg := strings.Split(line, "==") - - if splitPkg[0] == pkg.Pkgbase { - var pkgVer string - if pkg.Srcinfo.Epoch == "" { - pkgVer = pkg.Srcinfo.Pkgver + "-" + pkg.Srcinfo.Pkgrel - } else { - pkgVer = pkg.Srcinfo.Epoch + ":" + pkg.Srcinfo.Pkgver + "-" + pkg.Srcinfo.Pkgrel - } - - // try to build new versions of previously failed packages - if alpm.VerCmp(splitPkg[1], pkgVer) < 0 { - failed = false - } else { - failed = true - newContent = append(newContent, line+"\n") - } - } else { - newContent = append(newContent, line+"\n") - } - } - - check(scanner.Err()) - sort.Strings(newContent) - - _, err = file.Seek(0, 0) - check(err) - check(file.Truncate(0)) - _, err = file.WriteString(strings.Join(newContent, "")) - check(err) - - return failed -} - -func statusId2string(status int) (string, string) { - switch status { - case SKIPPED: - return "SKIPPED", "table-" + conf.Status.Class.Skipped - case QUEUED: - return "QUEUED", "table-" + conf.Status.Class.Queued - case LATEST: - return "LATEST", "table-" + conf.Status.Class.Latest - case FAILED: - return "FAILED", "table-" + conf.Status.Class.Failed - case BUILD: - return "SIGNING", "table-" + conf.Status.Class.Signing - case BUILDING: - return "BUILDING", "table-" + conf.Status.Class.Building - default: - return "UNKNOWN", "table-" + conf.Status.Class.Unknown - } -} - func (b *BuildManager) htmlWorker() { type Pkg struct { Pkgbase string @@ -731,26 +390,6 @@ func (b *BuildManager) htmlWorker() { } } -func setupChroot() { - if _, err := os.Stat(filepath.Join(conf.Basedir.Chroot, orgChrootName)); err == nil { - //goland:noinspection SpellCheckingInspection - cmd := exec.Command("arch-nspawn", filepath.Join(conf.Basedir.Chroot, orgChrootName), "pacman", "-Syuu", "--noconfirm") - res, err := cmd.CombinedOutput() - log.Debug(string(res)) - check(err) - } else if os.IsNotExist(err) { - err := os.MkdirAll(conf.Basedir.Chroot, os.ModePerm) - check(err) - - cmd := exec.Command("mkarchroot", "-C", pacmanConf, filepath.Join(conf.Basedir.Chroot, orgChrootName), "base-devel") - res, err := cmd.CombinedOutput() - log.Debug(string(res)) - check(err) - } else { - check(err) - } -} - func (b *BuildManager) repoWorker(repo string) { for { select { @@ -874,7 +513,7 @@ func (b *BuildManager) syncWorker() { // fetch updates between sync runs setupChroot() - pkgBuilds, err := filepathx.Glob(filepath.Join(conf.Basedir.Upstream, "/**/PKGBUILD")) + pkgBuilds, err := Glob(filepath.Join(conf.Basedir.Upstream, "/**/PKGBUILD")) check(err) // Shuffle pkgbuilds to spread out long-running builds, otherwise pkgBuilds is alphabetically-sorted @@ -889,7 +528,7 @@ func (b *BuildManager) syncWorker() { sPkgbuild := strings.Split(pkgbuild, "/") repo := sPkgbuild[len(sPkgbuild)-2] - if repo == "trunk" || !contains(conf.Repos, strings.Split(repo, "-")[0]) || strings.Contains(repo, "i686") || strings.Contains(repo, "testing") || strings.Contains(repo, "staging") { + if repo == "trunk" || !contains(conf.Repos, strings.Split(repo, "-")[0]) || containsSubStr(repo, conf.Blacklist.Repo) { continue } @@ -956,6 +595,11 @@ func main() { setupChroot() syncMarchs() + alpmLock.Lock() + alpmHandle, err = initALPM(filepath.Join(conf.Basedir.Chroot, "root"), filepath.Join(conf.Basedir.Chroot, "/root/var/lib/pacman")) + check(err) + alpmLock.Unlock() + go buildManager.syncWorker() go buildManager.htmlWorker() @@ -972,4 +616,5 @@ func main() { buildManager.buildProcMutex.RUnlock() buildManager.buildWG.Wait() buildManager.repoWG.Wait() + check(alpmHandle.Release()) } diff --git a/utils.go b/utils.go new file mode 100644 index 0000000..d468c20 --- /dev/null +++ b/utils.go @@ -0,0 +1,591 @@ +package main + +import ( + "ALHP.go/ent" + "ALHP.go/ent/dbpackage" + "bufio" + "context" + "fmt" + "github.com/Jguer/go-alpm/v2" + paconf "github.com/Morganamilo/go-pacmanconf" + "github.com/Morganamilo/go-srcinfo" + log "github.com/sirupsen/logrus" + "io" + "io/fs" + "os" + "os/exec" + "path/filepath" + "sort" + "strconv" + "strings" + "sync" +) + +const ( + SKIPPED = iota + FAILED = iota + BUILD = iota + QUEUED = iota + BUILDING = iota + LATEST = iota + UNKNOWN = iota +) + +type BuildPackage struct { + Pkgbase string + Pkgbuild string + Srcinfo *srcinfo.Srcinfo + PkgFiles []string + Repo string + March string + FullRepo string + Version string +} + +type BuildManager struct { + build chan *BuildPackage + parse chan *BuildPackage + repoPurge map[string]chan *BuildPackage + repoAdd map[string]chan *BuildPackage + exit bool + buildWG sync.WaitGroup + parseWG sync.WaitGroup + repoWG sync.WaitGroup + failedMutex sync.RWMutex + buildProcesses []*os.Process + buildProcMutex sync.RWMutex +} + +type Conf struct { + Arch string + Repos, March []string + Svn2git map[string]string + Basedir struct { + Repo, Chroot, Makepkg, Upstream, Db string + } + Build struct { + Worker int + Makej int + } + Logging struct { + Level string + } + Status struct { + Class struct { + Skipped, Queued, Latest, Failed, Signing, Building, Unknown string + } + } + Blacklist struct { + Packages []string + Repo []string + } +} + +type Globs []string + +func check(e error) { + if e != nil { + panic(e) + } +} + +func containsSubStr(str string, subList []string) bool { + for _, checkStr := range subList { + if strings.Contains(str, checkStr) { + return true + } + } + return false +} + +func statusId2string(status int) (string, string) { + switch status { + case SKIPPED: + return "SKIPPED", "table-" + conf.Status.Class.Skipped + case QUEUED: + return "QUEUED", "table-" + conf.Status.Class.Queued + case LATEST: + return "LATEST", "table-" + conf.Status.Class.Latest + case FAILED: + return "FAILED", "table-" + conf.Status.Class.Failed + case BUILD: + return "SIGNING", "table-" + conf.Status.Class.Signing + case BUILDING: + return "BUILDING", "table-" + conf.Status.Class.Building + default: + return "UNKNOWN", "table-" + conf.Status.Class.Unknown + } +} + +func getVersionFromRepo(pkg *BuildPackage) string { + findPkgFiles(pkg) + + if len(pkg.PkgFiles) == 0 { + return "" + } + + fNameSplit := strings.Split(pkg.PkgFiles[0], "-") + return fNameSplit[len(fNameSplit)-3] + "-" + fNameSplit[len(fNameSplit)-2] +} + +func gitClean(pkg *BuildPackage) { + cmd := exec.Command("sudo", "git_clean.sh", filepath.Dir(pkg.Pkgbuild)) + res, err := cmd.CombinedOutput() + if err != nil { + log.Warningf("git clean failed with %v:\n%s", err, res) + } else { + log.Debug(string(res)) + } +} + +func increasePkgRel(pkg *BuildPackage) error { + f, err := os.OpenFile(pkg.Pkgbuild, os.O_RDWR, os.ModePerm) + if err != nil { + return err + } + + defer func(f *os.File) { + err := f.Close() + if err != nil { + panic(err) + } + }(f) + + fStr, err := io.ReadAll(f) + if err != nil { + return err + } + + nStr := rePkgRel.ReplaceAllLiteralString(string(fStr), "pkgrel="+pkg.Srcinfo.Pkgrel+".1") + _, err = f.Seek(0, 0) + if err != nil { + return err + } + err = f.Truncate(0) + if err != nil { + return err + } + + _, err = f.WriteString(nStr) + if err != nil { + return err + } + + pkg.Version = pkg.Version + ".1" + return nil +} + +func packages2slice(pkgs interface{}) []string { + switch v := pkgs.(type) { + case []srcinfo.Package: + var sPkgs []string + for _, p := range v { + sPkgs = append(sPkgs, p.Pkgname) + } + + return sPkgs + case []srcinfo.ArchString: + var sPkgs []string + for _, p := range v { + sPkgs = append(sPkgs, p.Value) + } + + return sPkgs + default: + return []string{} + } +} + +func importKeys(pkg *BuildPackage) error { + if pkg.Srcinfo.ValidPGPKeys != nil { + args := []string{"--keyserver", "keyserver.ubuntu.com", "--recv-keys"} + args = append(args, pkg.Srcinfo.ValidPGPKeys...) + cmd := exec.Command("gpg", args...) + _, err := cmd.CombinedOutput() + + return err + } + return nil +} + +func constructVersion(pkgver string, pkgrel string, epoch string) string { + if epoch == "" { + return pkgver + "-" + pkgrel + } else { + return epoch + ":" + pkgver + "-" + pkgrel + } +} + +func initALPM(root string, dbpath string) (*alpm.Handle, error) { + h, err := alpm.Initialize(root, dbpath) + if err != nil { + return nil, err + } + + PacmanConfig, _, err := paconf.ParseFile(filepath.Join(root, "/etc/pacman.conf")) + if err != nil { + return nil, err + } + + for _, repo := range PacmanConfig.Repos { + db, err := h.RegisterSyncDB(repo.Name, 0) + if err != nil { + return nil, err + } + db.SetServers(repo.Servers) + + if len(repo.Usage) == 0 { + db.SetUsage(alpm.UsageAll) + } + for _, usage := range repo.Usage { + switch usage { + case "Sync": + db.SetUsage(alpm.UsageSync) + case "Search": + db.SetUsage(alpm.UsageSearch) + case "Install": + db.SetUsage(alpm.UsageInstall) + case "Upgrade": + db.SetUsage(alpm.UsageUpgrade) + case "All": + db.SetUsage(alpm.UsageAll) + } + } + } + + return h, nil +} + +func getSVN2GITVersion(pkg *BuildPackage) (string, error) { + if pkg.Pkgbuild == "" && pkg.Pkgbase == "" { + return "", fmt.Errorf("invalid arguments") + } + + // upstream/upstream-core-extra/extra-cmake-modules/repos/extra-any/PKGBUILD + pkgBuilds, _ := Glob(filepath.Join(conf.Basedir.Upstream, "**/"+pkg.Pkgbase+"/repos/*/PKGBUILD")) + + var fPkgbuilds []string + for _, pkgbuild := range pkgBuilds { + sPkgbuild := strings.Split(pkgbuild, "/") + repo := sPkgbuild[len(sPkgbuild)-2] + + if repo == "trunk" || containsSubStr(repo, conf.Blacklist.Repo) { + continue + } + + if !contains(fPkgbuilds, pkgbuild) { + fPkgbuilds = append(fPkgbuilds, pkgbuild) + } + } + + if len(fPkgbuilds) > 1 { + return "", fmt.Errorf("%s: multiple PKGBUILD found: %s", pkg.Pkgbase, fPkgbuilds) + } else if len(fPkgbuilds) == 0 { + return "", fmt.Errorf("%s: no matching PKGBUILD found (searched: %s, canidates: %s)", pkg.Pkgbase, filepath.Join(conf.Basedir.Upstream, "**/"+pkg.Pkgbase+"/repos/*/PKGBUILD"), pkgBuilds) + } + + cmd := exec.Command("sh", "-c", "cd "+filepath.Dir(fPkgbuilds[0])+"&&"+"makepkg --printsrcinfo") + res, err := cmd.Output() + if err != nil { + return "", err + } + + info, err := srcinfo.Parse(string(res)) + if err != nil { + return "", err + } + + return constructVersion(info.Pkgver, info.Pkgrel, info.Epoch), nil +} + +func isPkgFailed(pkg *BuildPackage) bool { + buildManager.failedMutex.Lock() + defer buildManager.failedMutex.Unlock() + + file, err := os.OpenFile(filepath.Join(conf.Basedir.Repo, pkg.FullRepo+"_failed.txt"), os.O_RDWR|os.O_CREATE|os.O_SYNC, os.ModePerm) + check(err) + defer func(file *os.File) { + check(file.Close()) + }(file) + + failed := false + var newContent []string + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + splitPkg := strings.Split(line, "==") + + if splitPkg[0] == pkg.Pkgbase { + pkgVer := constructVersion(pkg.Srcinfo.Pkgver, pkg.Srcinfo.Pkgrel, pkg.Srcinfo.Epoch) + + // try to build new versions of previously failed packages + if alpm.VerCmp(splitPkg[1], pkgVer) < 0 { + failed = false + } else { + failed = true + newContent = append(newContent, line+"\n") + } + } else { + newContent = append(newContent, line+"\n") + } + } + + check(scanner.Err()) + sort.Strings(newContent) + + _, err = file.Seek(0, 0) + check(err) + check(file.Truncate(0)) + _, err = file.WriteString(strings.Join(newContent, "")) + check(err) + + return failed +} + +func setupChroot() { + if _, err := os.Stat(filepath.Join(conf.Basedir.Chroot, orgChrootName)); err == nil { + //goland:noinspection SpellCheckingInspection + cmd := exec.Command("arch-nspawn", filepath.Join(conf.Basedir.Chroot, orgChrootName), "pacman", "-Syuu", "--noconfirm") + res, err := cmd.CombinedOutput() + log.Debug(string(res)) + check(err) + } else if os.IsNotExist(err) { + err := os.MkdirAll(conf.Basedir.Chroot, os.ModePerm) + check(err) + + cmd := exec.Command("mkarchroot", "-C", pacmanConf, filepath.Join(conf.Basedir.Chroot, orgChrootName), "base-devel") + res, err := cmd.CombinedOutput() + log.Debug(string(res)) + check(err) + } else { + check(err) + } +} + +func findPkgFiles(pkg *BuildPackage) { + pkgs, err := os.ReadDir(filepath.Join(conf.Basedir.Repo, pkg.FullRepo, "os", conf.Arch)) + check(err) + + var fPkg []string + for _, file := range pkgs { + if !file.IsDir() && !strings.HasSuffix(file.Name(), ".sig") { + matches := rePkgFile.FindStringSubmatch(file.Name()) + + var realPkgs []string + for _, realPkg := range pkg.Srcinfo.Packages { + realPkgs = append(realPkgs, realPkg.Pkgname) + } + + if len(matches) > 1 && contains(realPkgs, matches[1]) { + fPkg = append(fPkg, filepath.Join(conf.Basedir.Repo, pkg.FullRepo, "os", conf.Arch, file.Name())) + } + } + } + + pkg.PkgFiles = fPkg +} + +func getDbPackage(pkg *BuildPackage) *ent.DbPackage { + dbLock.Lock() + dbPkg, err := db.DbPackage.Query().Where(dbpackage.Pkgbase(pkg.Pkgbase)).Only(context.Background()) + if err != nil { + dbPkg = db.DbPackage.Create().SetPkgbase(pkg.Pkgbase).SetMarch(pkg.March).SetPackages(packages2slice(pkg.Srcinfo.Packages)).SetRepository(pkg.Repo).SaveX(context.Background()) + } + dbLock.Unlock() + + return dbPkg +} + +func syncMarchs() { + files, err := os.ReadDir(conf.Basedir.Repo) + check(err) + + var eRepos []string + for _, file := range files { + if file.Name() != "." && file.Name() != logDir && file.IsDir() { + eRepos = append(eRepos, file.Name()) + } + } + + for _, march := range conf.March { + setupMakepkg(march) + for _, repo := range conf.Repos { + tRepo := fmt.Sprintf("%s-%s", repo, march) + repos = append(repos, tRepo) + buildManager.repoAdd[tRepo] = make(chan *BuildPackage, conf.Build.Worker) + buildManager.repoPurge[tRepo] = make(chan *BuildPackage, 10000) + go buildManager.repoWorker(tRepo) + + if _, err := os.Stat(filepath.Join(filepath.Join(conf.Basedir.Repo, tRepo, "os", conf.Arch))); os.IsNotExist(err) { + log.Debugf("Creating path %s", filepath.Join(conf.Basedir.Repo, tRepo, "os", conf.Arch)) + check(os.MkdirAll(filepath.Join(conf.Basedir.Repo, tRepo, "os", conf.Arch), os.ModePerm)) + } + + if i := find(eRepos, tRepo); i != -1 { + eRepos = append(eRepos[:i], eRepos[i+1:]...) + } + } + } + + log.Infof("Repos: %s", repos) + + for _, repo := range eRepos { + log.Infof("Removing old repo %s", repo) + check(os.RemoveAll(filepath.Join(conf.Basedir.Repo, repo))) + } +} + +//goland:noinspection SpellCheckingInspection +func setupMakepkg(march string) { + lMakepkg := filepath.Join(conf.Basedir.Makepkg, fmt.Sprintf("makepkg-%s.conf", march)) + + check(os.MkdirAll(conf.Basedir.Makepkg, os.ModePerm)) + t, err := os.ReadFile(makepkgConf) + check(err) + makepkgStr := string(t) + + makepkgStr = strings.ReplaceAll(makepkgStr, "-mtune=generic", "") + makepkgStr = strings.ReplaceAll(makepkgStr, "-O2", "-O3") + makepkgStr = strings.ReplaceAll(makepkgStr, " check ", " !check ") + makepkgStr = strings.ReplaceAll(makepkgStr, " color ", " !color ") + makepkgStr = strings.ReplaceAll(makepkgStr, "#MAKEFLAGS=\"-j2\"", "MAKEFLAGS=\"-j"+strconv.Itoa(conf.Build.Makej)+"\"") + makepkgStr = reMarch.ReplaceAllString(makepkgStr, "${1}"+march) + + check(os.WriteFile(lMakepkg, []byte(makepkgStr), os.ModePerm)) +} + +func isMirrorLatest(h *alpm.Handle, buildPkg *BuildPackage) (bool, error) { + alpmLock.Lock() + dbs, err := h.SyncDBs() + if err != nil { + return false, err + } + + allDepends := buildPkg.Srcinfo.Depends + allDepends = append(allDepends, buildPkg.Srcinfo.MakeDepends...) + + for _, dep := range allDepends { + pkg, err := dbs.FindSatisfier(dep.Value) + if err != nil { + return false, err + } + + svn2gitVer, err := getSVN2GITVersion(&BuildPackage{ + Pkgbase: pkg.Base(), + }) + if err != nil { + return false, err + } + + if svn2gitVer != "" && alpm.VerCmp(svn2gitVer, pkg.Version()) != 0 { + return false, nil + } + } + alpmLock.Unlock() + + return true, nil +} + +func contains(s interface{}, str string) bool { + switch v := s.(type) { + case []string: + if i := find(v, str); i != -1 { + return true + } + case []srcinfo.ArchString: + var n []string + for _, as := range v { + n = append(n, as.Value) + } + + if i := find(n, str); i != -1 { + return true + } + default: + return false + } + + return false +} + +func find(s []string, str string) int { + for i, v := range s { + if v == str { + return i + } + } + + return -1 +} + +func copyFile(src, dst string) (int64, error) { + sourceFileStat, err := os.Stat(src) + if err != nil { + return 0, err + } + + if !sourceFileStat.Mode().IsRegular() { + return 0, fmt.Errorf("%s is not a regular file", src) + } + + source, err := os.Open(src) + if err != nil { + return 0, err + } + defer func(source *os.File) { + check(source.Close()) + }(source) + + destination, err := os.Create(dst) + if err != nil { + return 0, err + } + defer func(destination *os.File) { + check(destination.Close()) + }(destination) + nBytes, err := io.Copy(destination, source) + return nBytes, err +} + +func Glob(pattern string) ([]string, error) { + if !strings.Contains(pattern, "**") { + return filepath.Glob(pattern) + } + return Globs(strings.Split(pattern, "**")).Expand() +} + +func (globs Globs) Expand() ([]string, error) { + var matches = []string{""} + for _, glob := range globs { + var hits []string + var hitMap = map[string]bool{} + for _, match := range matches { + paths, err := filepath.Glob(match + glob) + if err != nil { + return nil, err + } + for _, path := range paths { + err = filepath.WalkDir(path, func(path string, d os.DirEntry, err error) error { + if err != nil { + return fs.SkipDir + } + if _, ok := hitMap[path]; !ok { + hits = append(hits, path) + hitMap[path] = true + } + return nil + }) + if err != nil { + return nil, err + } + } + } + matches = hits + } + + if globs == nil && len(matches) > 0 && matches[0] == "" { + matches = matches[1:] + } + + return matches, nil +}