diff --git a/ent/dbpackage.go b/ent/dbpackage.go index 41cac22..f5123dc 100644 --- a/ent/dbpackage.go +++ b/ent/dbpackage.go @@ -59,6 +59,8 @@ type DbPackage struct { IoOut *int64 `json:"io_out,omitempty"` // Srcinfo holds the value of the "srcinfo" field. Srcinfo *string `json:"srcinfo,omitempty"` + // SrcinfoHash holds the value of the "srcinfo_hash" field. + SrcinfoHash string `json:"srcinfo_hash,omitempty"` } // scanValues returns the types for scanning values from sql.Rows. @@ -70,7 +72,7 @@ func (*DbPackage) scanValues(columns []string) ([]interface{}, error) { values[i] = new([]byte) case dbpackage.FieldID, dbpackage.FieldMaxRss, dbpackage.FieldUTime, dbpackage.FieldSTime, dbpackage.FieldIoIn, dbpackage.FieldIoOut: values[i] = new(sql.NullInt64) - case dbpackage.FieldPkgbase, dbpackage.FieldStatus, dbpackage.FieldSkipReason, dbpackage.FieldRepository, dbpackage.FieldMarch, dbpackage.FieldVersion, dbpackage.FieldRepoVersion, dbpackage.FieldHash, dbpackage.FieldLto, dbpackage.FieldLastVersionBuild, dbpackage.FieldDebugSymbols, dbpackage.FieldSrcinfo: + case dbpackage.FieldPkgbase, dbpackage.FieldStatus, dbpackage.FieldSkipReason, dbpackage.FieldRepository, dbpackage.FieldMarch, dbpackage.FieldVersion, dbpackage.FieldRepoVersion, dbpackage.FieldHash, dbpackage.FieldLto, dbpackage.FieldLastVersionBuild, dbpackage.FieldDebugSymbols, dbpackage.FieldSrcinfo, dbpackage.FieldSrcinfoHash: values[i] = new(sql.NullString) case dbpackage.FieldBuildTimeStart, dbpackage.FieldUpdated, dbpackage.FieldLastVerified: values[i] = new(sql.NullTime) @@ -229,6 +231,12 @@ func (dp *DbPackage) assignValues(columns []string, values []interface{}) error dp.Srcinfo = new(string) *dp.Srcinfo = value.String } + case dbpackage.FieldSrcinfoHash: + if value, ok := values[i].(*sql.NullString); !ok { + return fmt.Errorf("unexpected type %T for field srcinfo_hash", values[i]) + } else if value.Valid { + dp.SrcinfoHash = value.String + } } } return nil @@ -331,6 +339,9 @@ func (dp *DbPackage) String() string { builder.WriteString("srcinfo=") builder.WriteString(*v) } + builder.WriteString(", ") + builder.WriteString("srcinfo_hash=") + builder.WriteString(dp.SrcinfoHash) builder.WriteByte(')') return builder.String() } diff --git a/ent/dbpackage/dbpackage.go b/ent/dbpackage/dbpackage.go index fc2d5dd..d10abd2 100644 --- a/ent/dbpackage/dbpackage.go +++ b/ent/dbpackage/dbpackage.go @@ -53,6 +53,8 @@ const ( FieldIoOut = "io_out" // FieldSrcinfo holds the string denoting the srcinfo field in the database. FieldSrcinfo = "srcinfo" + // FieldSrcinfoHash holds the string denoting the srcinfo_hash field in the database. + FieldSrcinfoHash = "srcinfo_hash" // Table holds the table name of the dbpackage in the database. Table = "db_packages" ) @@ -81,6 +83,7 @@ var Columns = []string{ FieldIoIn, FieldIoOut, FieldSrcinfo, + FieldSrcinfoHash, } // ValidColumn reports if the column name is valid (part of the table columns). diff --git a/ent/dbpackage/where.go b/ent/dbpackage/where.go index 011e084..1efd706 100644 --- a/ent/dbpackage/where.go +++ b/ent/dbpackage/where.go @@ -192,6 +192,13 @@ func Srcinfo(v string) predicate.DbPackage { }) } +// SrcinfoHash applies equality check predicate on the "srcinfo_hash" field. It's identical to SrcinfoHashEQ. +func SrcinfoHash(v string) predicate.DbPackage { + return predicate.DbPackage(func(s *sql.Selector) { + s.Where(sql.EQ(s.C(FieldSrcinfoHash), v)) + }) +} + // PkgbaseEQ applies the EQ predicate on the "pkgbase" field. func PkgbaseEQ(v string) predicate.DbPackage { return predicate.DbPackage(func(s *sql.Selector) { @@ -1892,6 +1899,119 @@ func SrcinfoContainsFold(v string) predicate.DbPackage { }) } +// SrcinfoHashEQ applies the EQ predicate on the "srcinfo_hash" field. +func SrcinfoHashEQ(v string) predicate.DbPackage { + return predicate.DbPackage(func(s *sql.Selector) { + s.Where(sql.EQ(s.C(FieldSrcinfoHash), v)) + }) +} + +// SrcinfoHashNEQ applies the NEQ predicate on the "srcinfo_hash" field. +func SrcinfoHashNEQ(v string) predicate.DbPackage { + return predicate.DbPackage(func(s *sql.Selector) { + s.Where(sql.NEQ(s.C(FieldSrcinfoHash), v)) + }) +} + +// SrcinfoHashIn applies the In predicate on the "srcinfo_hash" field. +func SrcinfoHashIn(vs ...string) predicate.DbPackage { + v := make([]interface{}, len(vs)) + for i := range v { + v[i] = vs[i] + } + return predicate.DbPackage(func(s *sql.Selector) { + s.Where(sql.In(s.C(FieldSrcinfoHash), v...)) + }) +} + +// SrcinfoHashNotIn applies the NotIn predicate on the "srcinfo_hash" field. +func SrcinfoHashNotIn(vs ...string) predicate.DbPackage { + v := make([]interface{}, len(vs)) + for i := range v { + v[i] = vs[i] + } + return predicate.DbPackage(func(s *sql.Selector) { + s.Where(sql.NotIn(s.C(FieldSrcinfoHash), v...)) + }) +} + +// SrcinfoHashGT applies the GT predicate on the "srcinfo_hash" field. +func SrcinfoHashGT(v string) predicate.DbPackage { + return predicate.DbPackage(func(s *sql.Selector) { + s.Where(sql.GT(s.C(FieldSrcinfoHash), v)) + }) +} + +// SrcinfoHashGTE applies the GTE predicate on the "srcinfo_hash" field. +func SrcinfoHashGTE(v string) predicate.DbPackage { + return predicate.DbPackage(func(s *sql.Selector) { + s.Where(sql.GTE(s.C(FieldSrcinfoHash), v)) + }) +} + +// SrcinfoHashLT applies the LT predicate on the "srcinfo_hash" field. +func SrcinfoHashLT(v string) predicate.DbPackage { + return predicate.DbPackage(func(s *sql.Selector) { + s.Where(sql.LT(s.C(FieldSrcinfoHash), v)) + }) +} + +// SrcinfoHashLTE applies the LTE predicate on the "srcinfo_hash" field. +func SrcinfoHashLTE(v string) predicate.DbPackage { + return predicate.DbPackage(func(s *sql.Selector) { + s.Where(sql.LTE(s.C(FieldSrcinfoHash), v)) + }) +} + +// SrcinfoHashContains applies the Contains predicate on the "srcinfo_hash" field. +func SrcinfoHashContains(v string) predicate.DbPackage { + return predicate.DbPackage(func(s *sql.Selector) { + s.Where(sql.Contains(s.C(FieldSrcinfoHash), v)) + }) +} + +// SrcinfoHashHasPrefix applies the HasPrefix predicate on the "srcinfo_hash" field. +func SrcinfoHashHasPrefix(v string) predicate.DbPackage { + return predicate.DbPackage(func(s *sql.Selector) { + s.Where(sql.HasPrefix(s.C(FieldSrcinfoHash), v)) + }) +} + +// SrcinfoHashHasSuffix applies the HasSuffix predicate on the "srcinfo_hash" field. +func SrcinfoHashHasSuffix(v string) predicate.DbPackage { + return predicate.DbPackage(func(s *sql.Selector) { + s.Where(sql.HasSuffix(s.C(FieldSrcinfoHash), v)) + }) +} + +// SrcinfoHashIsNil applies the IsNil predicate on the "srcinfo_hash" field. +func SrcinfoHashIsNil() predicate.DbPackage { + return predicate.DbPackage(func(s *sql.Selector) { + s.Where(sql.IsNull(s.C(FieldSrcinfoHash))) + }) +} + +// SrcinfoHashNotNil applies the NotNil predicate on the "srcinfo_hash" field. +func SrcinfoHashNotNil() predicate.DbPackage { + return predicate.DbPackage(func(s *sql.Selector) { + s.Where(sql.NotNull(s.C(FieldSrcinfoHash))) + }) +} + +// SrcinfoHashEqualFold applies the EqualFold predicate on the "srcinfo_hash" field. +func SrcinfoHashEqualFold(v string) predicate.DbPackage { + return predicate.DbPackage(func(s *sql.Selector) { + s.Where(sql.EqualFold(s.C(FieldSrcinfoHash), v)) + }) +} + +// SrcinfoHashContainsFold applies the ContainsFold predicate on the "srcinfo_hash" field. +func SrcinfoHashContainsFold(v string) predicate.DbPackage { + return predicate.DbPackage(func(s *sql.Selector) { + s.Where(sql.ContainsFold(s.C(FieldSrcinfoHash), v)) + }) +} + // And groups predicates with the AND operator between them. func And(predicates ...predicate.DbPackage) predicate.DbPackage { return predicate.DbPackage(func(s *sql.Selector) { diff --git a/ent/dbpackage_create.go b/ent/dbpackage_create.go index dce00d3..e650cb6 100644 --- a/ent/dbpackage_create.go +++ b/ent/dbpackage_create.go @@ -282,6 +282,20 @@ func (dpc *DbPackageCreate) SetNillableSrcinfo(s *string) *DbPackageCreate { return dpc } +// SetSrcinfoHash sets the "srcinfo_hash" field. +func (dpc *DbPackageCreate) SetSrcinfoHash(s string) *DbPackageCreate { + dpc.mutation.SetSrcinfoHash(s) + return dpc +} + +// SetNillableSrcinfoHash sets the "srcinfo_hash" field if the given value is not nil. +func (dpc *DbPackageCreate) SetNillableSrcinfoHash(s *string) *DbPackageCreate { + if s != nil { + dpc.SetSrcinfoHash(*s) + } + return dpc +} + // Mutation returns the DbPackageMutation object of the builder. func (dpc *DbPackageCreate) Mutation() *DbPackageMutation { return dpc.mutation @@ -609,6 +623,14 @@ func (dpc *DbPackageCreate) createSpec() (*DbPackage, *sqlgraph.CreateSpec) { }) _node.Srcinfo = &value } + if value, ok := dpc.mutation.SrcinfoHash(); ok { + _spec.Fields = append(_spec.Fields, &sqlgraph.FieldSpec{ + Type: field.TypeString, + Value: value, + Column: dbpackage.FieldSrcinfoHash, + }) + _node.SrcinfoHash = value + } return _node, _spec } diff --git a/ent/dbpackage_update.go b/ent/dbpackage_update.go index 0cfc931..ff0d49c 100644 --- a/ent/dbpackage_update.go +++ b/ent/dbpackage_update.go @@ -422,6 +422,26 @@ func (dpu *DbPackageUpdate) ClearSrcinfo() *DbPackageUpdate { return dpu } +// SetSrcinfoHash sets the "srcinfo_hash" field. +func (dpu *DbPackageUpdate) SetSrcinfoHash(s string) *DbPackageUpdate { + dpu.mutation.SetSrcinfoHash(s) + return dpu +} + +// SetNillableSrcinfoHash sets the "srcinfo_hash" field if the given value is not nil. +func (dpu *DbPackageUpdate) SetNillableSrcinfoHash(s *string) *DbPackageUpdate { + if s != nil { + dpu.SetSrcinfoHash(*s) + } + return dpu +} + +// ClearSrcinfoHash clears the value of the "srcinfo_hash" field. +func (dpu *DbPackageUpdate) ClearSrcinfoHash() *DbPackageUpdate { + dpu.mutation.ClearSrcinfoHash() + return dpu +} + // Mutation returns the DbPackageMutation object of the builder. func (dpu *DbPackageUpdate) Mutation() *DbPackageMutation { return dpu.mutation @@ -812,6 +832,19 @@ func (dpu *DbPackageUpdate) sqlSave(ctx context.Context) (n int, err error) { Column: dbpackage.FieldSrcinfo, }) } + if value, ok := dpu.mutation.SrcinfoHash(); ok { + _spec.Fields.Set = append(_spec.Fields.Set, &sqlgraph.FieldSpec{ + Type: field.TypeString, + Value: value, + Column: dbpackage.FieldSrcinfoHash, + }) + } + if dpu.mutation.SrcinfoHashCleared() { + _spec.Fields.Clear = append(_spec.Fields.Clear, &sqlgraph.FieldSpec{ + Type: field.TypeString, + Column: dbpackage.FieldSrcinfoHash, + }) + } _spec.Modifiers = dpu.modifiers if n, err = sqlgraph.UpdateNodes(ctx, dpu.driver, _spec); err != nil { if _, ok := err.(*sqlgraph.NotFoundError); ok { @@ -1226,6 +1259,26 @@ func (dpuo *DbPackageUpdateOne) ClearSrcinfo() *DbPackageUpdateOne { return dpuo } +// SetSrcinfoHash sets the "srcinfo_hash" field. +func (dpuo *DbPackageUpdateOne) SetSrcinfoHash(s string) *DbPackageUpdateOne { + dpuo.mutation.SetSrcinfoHash(s) + return dpuo +} + +// SetNillableSrcinfoHash sets the "srcinfo_hash" field if the given value is not nil. +func (dpuo *DbPackageUpdateOne) SetNillableSrcinfoHash(s *string) *DbPackageUpdateOne { + if s != nil { + dpuo.SetSrcinfoHash(*s) + } + return dpuo +} + +// ClearSrcinfoHash clears the value of the "srcinfo_hash" field. +func (dpuo *DbPackageUpdateOne) ClearSrcinfoHash() *DbPackageUpdateOne { + dpuo.mutation.ClearSrcinfoHash() + return dpuo +} + // Mutation returns the DbPackageMutation object of the builder. func (dpuo *DbPackageUpdateOne) Mutation() *DbPackageMutation { return dpuo.mutation @@ -1646,6 +1699,19 @@ func (dpuo *DbPackageUpdateOne) sqlSave(ctx context.Context) (_node *DbPackage, Column: dbpackage.FieldSrcinfo, }) } + if value, ok := dpuo.mutation.SrcinfoHash(); ok { + _spec.Fields.Set = append(_spec.Fields.Set, &sqlgraph.FieldSpec{ + Type: field.TypeString, + Value: value, + Column: dbpackage.FieldSrcinfoHash, + }) + } + if dpuo.mutation.SrcinfoHashCleared() { + _spec.Fields.Clear = append(_spec.Fields.Clear, &sqlgraph.FieldSpec{ + Type: field.TypeString, + Column: dbpackage.FieldSrcinfoHash, + }) + } _spec.Modifiers = dpuo.modifiers _node = &DbPackage{config: dpuo.config} _spec.Assign = _node.assignValues diff --git a/ent/migrate/schema.go b/ent/migrate/schema.go index f06a1ae..6c6f603 100644 --- a/ent/migrate/schema.go +++ b/ent/migrate/schema.go @@ -32,6 +32,7 @@ var ( {Name: "io_in", Type: field.TypeInt64, Nullable: true}, {Name: "io_out", Type: field.TypeInt64, Nullable: true}, {Name: "srcinfo", Type: field.TypeString, Nullable: true, Size: 2147483647}, + {Name: "srcinfo_hash", Type: field.TypeString, Nullable: true}, } // DbPackagesTable holds the schema information for the "db_packages" table. DbPackagesTable = &schema.Table{ diff --git a/ent/mutation.go b/ent/mutation.go index aa65cd2..6f9b5d2 100644 --- a/ent/mutation.go +++ b/ent/mutation.go @@ -59,6 +59,7 @@ type DbPackageMutation struct { io_out *int64 addio_out *int64 srcinfo *string + srcinfo_hash *string clearedFields map[string]struct{} done bool oldValue func(context.Context) (*DbPackage, error) @@ -1258,6 +1259,55 @@ func (m *DbPackageMutation) ResetSrcinfo() { delete(m.clearedFields, dbpackage.FieldSrcinfo) } +// SetSrcinfoHash sets the "srcinfo_hash" field. +func (m *DbPackageMutation) SetSrcinfoHash(s string) { + m.srcinfo_hash = &s +} + +// SrcinfoHash returns the value of the "srcinfo_hash" field in the mutation. +func (m *DbPackageMutation) SrcinfoHash() (r string, exists bool) { + v := m.srcinfo_hash + if v == nil { + return + } + return *v, true +} + +// OldSrcinfoHash returns the old "srcinfo_hash" field's value of the DbPackage entity. +// If the DbPackage object wasn't provided to the builder, the object is fetched from the database. +// An error is returned if the mutation operation is not UpdateOne, or the database query fails. +func (m *DbPackageMutation) OldSrcinfoHash(ctx context.Context) (v string, err error) { + if !m.op.Is(OpUpdateOne) { + return v, errors.New("OldSrcinfoHash is only allowed on UpdateOne operations") + } + if m.id == nil || m.oldValue == nil { + return v, errors.New("OldSrcinfoHash requires an ID field in the mutation") + } + oldValue, err := m.oldValue(ctx) + if err != nil { + return v, fmt.Errorf("querying old value for OldSrcinfoHash: %w", err) + } + return oldValue.SrcinfoHash, nil +} + +// ClearSrcinfoHash clears the value of the "srcinfo_hash" field. +func (m *DbPackageMutation) ClearSrcinfoHash() { + m.srcinfo_hash = nil + m.clearedFields[dbpackage.FieldSrcinfoHash] = struct{}{} +} + +// SrcinfoHashCleared returns if the "srcinfo_hash" field was cleared in this mutation. +func (m *DbPackageMutation) SrcinfoHashCleared() bool { + _, ok := m.clearedFields[dbpackage.FieldSrcinfoHash] + return ok +} + +// ResetSrcinfoHash resets all changes to the "srcinfo_hash" field. +func (m *DbPackageMutation) ResetSrcinfoHash() { + m.srcinfo_hash = nil + delete(m.clearedFields, dbpackage.FieldSrcinfoHash) +} + // Where appends a list predicates to the DbPackageMutation builder. func (m *DbPackageMutation) Where(ps ...predicate.DbPackage) { m.predicates = append(m.predicates, ps...) @@ -1277,7 +1327,7 @@ func (m *DbPackageMutation) Type() string { // order to get all numeric fields that were incremented/decremented, call // AddedFields(). func (m *DbPackageMutation) Fields() []string { - fields := make([]string, 0, 21) + fields := make([]string, 0, 22) if m.pkgbase != nil { fields = append(fields, dbpackage.FieldPkgbase) } @@ -1341,6 +1391,9 @@ func (m *DbPackageMutation) Fields() []string { if m.srcinfo != nil { fields = append(fields, dbpackage.FieldSrcinfo) } + if m.srcinfo_hash != nil { + fields = append(fields, dbpackage.FieldSrcinfoHash) + } return fields } @@ -1391,6 +1444,8 @@ func (m *DbPackageMutation) Field(name string) (ent.Value, bool) { return m.IoOut() case dbpackage.FieldSrcinfo: return m.Srcinfo() + case dbpackage.FieldSrcinfoHash: + return m.SrcinfoHash() } return nil, false } @@ -1442,6 +1497,8 @@ func (m *DbPackageMutation) OldField(ctx context.Context, name string) (ent.Valu return m.OldIoOut(ctx) case dbpackage.FieldSrcinfo: return m.OldSrcinfo(ctx) + case dbpackage.FieldSrcinfoHash: + return m.OldSrcinfoHash(ctx) } return nil, fmt.Errorf("unknown DbPackage field %s", name) } @@ -1598,6 +1655,13 @@ func (m *DbPackageMutation) SetField(name string, value ent.Value) error { } m.SetSrcinfo(v) return nil + case dbpackage.FieldSrcinfoHash: + v, ok := value.(string) + if !ok { + return fmt.Errorf("unexpected type %T for field %s", value, name) + } + m.SetSrcinfoHash(v) + return nil } return fmt.Errorf("unknown DbPackage field %s", name) } @@ -1745,6 +1809,9 @@ func (m *DbPackageMutation) ClearedFields() []string { if m.FieldCleared(dbpackage.FieldSrcinfo) { fields = append(fields, dbpackage.FieldSrcinfo) } + if m.FieldCleared(dbpackage.FieldSrcinfoHash) { + fields = append(fields, dbpackage.FieldSrcinfoHash) + } return fields } @@ -1813,6 +1880,9 @@ func (m *DbPackageMutation) ClearField(name string) error { case dbpackage.FieldSrcinfo: m.ClearSrcinfo() return nil + case dbpackage.FieldSrcinfoHash: + m.ClearSrcinfoHash() + return nil } return fmt.Errorf("unknown DbPackage nullable field %s", name) } @@ -1884,6 +1954,9 @@ func (m *DbPackageMutation) ResetField(name string) error { case dbpackage.FieldSrcinfo: m.ResetSrcinfo() return nil + case dbpackage.FieldSrcinfoHash: + m.ResetSrcinfoHash() + return nil } return fmt.Errorf("unknown DbPackage field %s", name) } diff --git a/ent/schema/dbpackage.go b/ent/schema/dbpackage.go index ac1be86..589b883 100644 --- a/ent/schema/dbpackage.go +++ b/ent/schema/dbpackage.go @@ -34,6 +34,7 @@ func (DbPackage) Fields() []ent.Field { field.Int64("io_in").Optional().Nillable(), field.Int64("io_out").Optional().Nillable(), field.Text("srcinfo").Optional().Nillable(), + field.String("srcinfo_hash").Optional(), } } diff --git a/main.go b/main.go index f212d70..c0bd23a 100644 --- a/main.go +++ b/main.go @@ -21,7 +21,6 @@ import ( "os/exec" "os/signal" "path/filepath" - "runtime" "strings" "sync" "syscall" @@ -319,85 +318,6 @@ func (b *BuildManager) repoWorker(repo string) { } } -func (b *BuildManager) refreshSRCINFOs(ctx context.Context, path string) error { - pkgBuilds, err := Glob(path) - if err != nil { - return fmt.Errorf("error scanning for PKGBUILDs: %w", err) - } - - step := int(float32(len(pkgBuilds)) / float32(runtime.NumCPU())) - cur := 0 - wg := sync.WaitGroup{} - for i := 0; i < runtime.NumCPU(); i++ { - if i == runtime.NumCPU()-1 { - step = len(pkgBuilds) - cur - } - - wg.Add(1) - go func(pkgBuilds []string) { - defer wg.Done() - for _, pkgbuild := range pkgBuilds { - mPkgbuild := PKGBUILD(pkgbuild) - if mPkgbuild.FullRepo() == "trunk" || !Contains(conf.Repos, mPkgbuild.Repo()) || containsSubStr(mPkgbuild.FullRepo(), conf.Blacklist.Repo) { - continue - } - - for _, march := range conf.March { - dbPkg, dbErr := db.DbPackage.Query().Where( - dbpackage.And( - dbpackage.Pkgbase(mPkgbuild.PkgBase()), - dbpackage.RepositoryEQ(dbpackage.Repository(mPkgbuild.Repo())), - dbpackage.March(march), - ), - ).Only(context.Background()) - - if ent.IsNotFound(dbErr) { - log.Debugf("[%s/%s] Package not found in database", mPkgbuild.Repo(), mPkgbuild.PkgBase()) - } else if err != nil { - log.Errorf("[%s/%s] Problem querying db for package: %v", mPkgbuild.Repo(), mPkgbuild.PkgBase(), dbErr) - } - - // compare b3sum of PKGBUILD file to hash in database, only proceed if hash differs - // reduces the amount of PKGBUILDs that need to be parsed with makepkg, which is _really_ slow, significantly - b3s, err := b3sum(pkgbuild) - if err != nil { - log.Fatalf("Error hashing PKGBUILD: %v", err) - } - - if dbPkg != nil && b3s == dbPkg.Hash { - log.Debugf("[%s/%s] Skipped: PKGBUILD hash matches db (%s)", mPkgbuild.Repo(), mPkgbuild.PkgBase(), b3s) - continue - } else if dbPkg != nil && b3s != dbPkg.Hash { - log.Debugf("[%s/%s] srcinfo cleared", mPkgbuild.Repo(), mPkgbuild.PkgBase()) - dbPkg = dbPkg.Update().ClearSrcinfo().SaveX(context.Background()) - } - - proto := &ProtoPackage{ - Pkgbuild: pkgbuild, - Pkgbase: mPkgbuild.PkgBase(), - Repo: dbpackage.Repository(mPkgbuild.Repo()), - March: march, - FullRepo: mPkgbuild.Repo() + "-" + march, - Hash: b3s, - DbPackage: dbPkg, - } - - _, err = proto.isEligible(ctx) - if err != nil { - log.Infof("Unable to determine status for package %s: %v", proto.Pkgbase, err) - b.repoPurge[proto.FullRepo] <- []*ProtoPackage{proto} - } - } - } - }(pkgBuilds[cur : cur+step]) - - cur += step - } - - wg.Wait() - return nil -} - func (b *BuildManager) syncWorker(ctx context.Context) error { err := os.MkdirAll(filepath.Join(conf.Basedir.Work, upstreamDir), 0755) if err != nil { diff --git a/proto_package.go b/proto_package.go index 043be49..903e10c 100644 --- a/proto_package.go +++ b/proto_package.go @@ -715,7 +715,7 @@ func (p *ProtoPackage) genSrcinfo() error { p.Srcinfo = info if p.DbPackage != nil { - p.DbPackage = p.DbPackage.Update().SetSrcinfo(string(res)).SaveX(context.Background()) + p.DbPackage = p.DbPackage.Update().SetSrcinfoHash(p.Hash).SetSrcinfo(string(res)).SaveX(context.Background()) } return nil diff --git a/utils.go b/utils.go index d77e6ea..5a78cb9 100644 --- a/utils.go +++ b/utils.go @@ -19,6 +19,7 @@ import ( "os/exec" "path/filepath" "regexp" + "runtime" "sort" "strconv" "strings" @@ -117,6 +118,85 @@ func updateLastUpdated() error { return nil } +func (b *BuildManager) refreshSRCINFOs(ctx context.Context, path string) error { + pkgBuilds, err := Glob(path) + if err != nil { + return fmt.Errorf("error scanning for PKGBUILDs: %w", err) + } + + step := int(float32(len(pkgBuilds)) / float32(runtime.NumCPU())) + cur := 0 + wg := sync.WaitGroup{} + for i := 0; i < runtime.NumCPU(); i++ { + if i == runtime.NumCPU()-1 { + step = len(pkgBuilds) - cur + } + + wg.Add(1) + go func(pkgBuilds []string) { + defer wg.Done() + for _, pkgbuild := range pkgBuilds { + mPkgbuild := PKGBUILD(pkgbuild) + if mPkgbuild.FullRepo() == "trunk" || !Contains(conf.Repos, mPkgbuild.Repo()) || containsSubStr(mPkgbuild.FullRepo(), conf.Blacklist.Repo) { + continue + } + + for _, march := range conf.March { + dbPkg, dbErr := db.DbPackage.Query().Where( + dbpackage.And( + dbpackage.Pkgbase(mPkgbuild.PkgBase()), + dbpackage.RepositoryEQ(dbpackage.Repository(mPkgbuild.Repo())), + dbpackage.March(march), + ), + ).Only(context.Background()) + + if ent.IsNotFound(dbErr) { + log.Debugf("[%s/%s] Package not found in database", mPkgbuild.Repo(), mPkgbuild.PkgBase()) + } else if err != nil { + log.Errorf("[%s/%s] Problem querying db for package: %v", mPkgbuild.Repo(), mPkgbuild.PkgBase(), dbErr) + } + + // compare b3sum of PKGBUILD file to hash in database, only proceed if hash differs + // reduces the amount of PKGBUILDs that need to be parsed with makepkg, which is _really_ slow, significantly + b3s, err := b3sum(pkgbuild) + if err != nil { + log.Fatalf("Error hashing PKGBUILD: %v", err) + } + + if dbPkg != nil && b3s == dbPkg.Hash { + log.Debugf("[%s/%s] Skipped: PKGBUILD hash matches db (%s)", mPkgbuild.Repo(), mPkgbuild.PkgBase(), b3s) + continue + } else if dbPkg != nil && b3s != dbPkg.Hash && dbPkg.SrcinfoHash != b3s { + log.Debugf("[%s/%s] srcinfo cleared", mPkgbuild.Repo(), mPkgbuild.PkgBase()) + dbPkg = dbPkg.Update().ClearSrcinfo().SaveX(context.Background()) + } + + proto := &ProtoPackage{ + Pkgbuild: pkgbuild, + Pkgbase: mPkgbuild.PkgBase(), + Repo: dbpackage.Repository(mPkgbuild.Repo()), + March: march, + FullRepo: mPkgbuild.Repo() + "-" + march, + Hash: b3s, + DbPackage: dbPkg, + } + + _, err = proto.isEligible(ctx) + if err != nil { + log.Infof("Unable to determine status for package %s: %v", proto.Pkgbase, err) + b.repoPurge[proto.FullRepo] <- []*ProtoPackage{proto} + } + } + } + }(pkgBuilds[cur : cur+step]) + + cur += step + } + + wg.Wait() + return nil +} + func statusId2string(s dbpackage.Status) string { switch s { case dbpackage.StatusSkipped: