diff --git a/ent/dbpackage.go b/ent/dbpackage.go index a5c9bdf..3ef3d71 100644 --- a/ent/dbpackage.go +++ b/ent/dbpackage.go @@ -39,6 +39,8 @@ type DbPackage struct { BuildDuration uint64 `json:"build_duration,omitempty"` // Updated holds the value of the "updated" field. Updated time.Time `json:"updated,omitempty"` + // Hash holds the value of the "hash" field. + Hash string `json:"hash,omitempty"` } // scanValues returns the types for scanning values from sql.Rows. @@ -50,7 +52,7 @@ func (*DbPackage) scanValues(columns []string) ([]interface{}, error) { values[i] = new([]byte) case dbpackage.FieldID, dbpackage.FieldStatus, dbpackage.FieldBuildDuration: values[i] = new(sql.NullInt64) - case dbpackage.FieldPkgbase, dbpackage.FieldSkipReason, dbpackage.FieldRepository, dbpackage.FieldMarch, dbpackage.FieldVersion, dbpackage.FieldRepoVersion: + case dbpackage.FieldPkgbase, dbpackage.FieldSkipReason, dbpackage.FieldRepository, dbpackage.FieldMarch, dbpackage.FieldVersion, dbpackage.FieldRepoVersion, dbpackage.FieldHash: values[i] = new(sql.NullString) case dbpackage.FieldBuildTime, dbpackage.FieldUpdated: values[i] = new(sql.NullTime) @@ -144,6 +146,12 @@ func (dp *DbPackage) assignValues(columns []string, values []interface{}) error } else if value.Valid { dp.Updated = value.Time } + case dbpackage.FieldHash: + if value, ok := values[i].(*sql.NullString); !ok { + return fmt.Errorf("unexpected type %T for field hash", values[i]) + } else if value.Valid { + dp.Hash = value.String + } } } return nil @@ -194,6 +202,8 @@ func (dp *DbPackage) String() string { builder.WriteString(fmt.Sprintf("%v", dp.BuildDuration)) builder.WriteString(", updated=") builder.WriteString(dp.Updated.Format(time.ANSIC)) + builder.WriteString(", hash=") + builder.WriteString(dp.Hash) builder.WriteByte(')') return builder.String() } diff --git a/ent/dbpackage/dbpackage.go b/ent/dbpackage/dbpackage.go index da2727b..478f5a5 100644 --- a/ent/dbpackage/dbpackage.go +++ b/ent/dbpackage/dbpackage.go @@ -29,6 +29,8 @@ const ( FieldBuildDuration = "build_duration" // FieldUpdated holds the string denoting the updated field in the database. FieldUpdated = "updated" + // FieldHash holds the string denoting the hash field in the database. + FieldHash = "hash" // Table holds the table name of the dbpackage in the database. Table = "db_packages" ) @@ -47,6 +49,7 @@ var Columns = []string{ FieldBuildTime, FieldBuildDuration, FieldUpdated, + FieldHash, } // ValidColumn reports if the column name is valid (part of the table columns). diff --git a/ent/dbpackage/where.go b/ent/dbpackage/where.go index f29a038..1682b90 100644 --- a/ent/dbpackage/where.go +++ b/ent/dbpackage/where.go @@ -162,6 +162,13 @@ func Updated(v time.Time) predicate.DbPackage { }) } +// Hash applies equality check predicate on the "hash" field. It's identical to HashEQ. +func Hash(v string) predicate.DbPackage { + return predicate.DbPackage(func(s *sql.Selector) { + s.Where(sql.EQ(s.C(FieldHash), v)) + }) +} + // PkgbaseEQ applies the EQ predicate on the "pkgbase" field. func PkgbaseEQ(v string) predicate.DbPackage { return predicate.DbPackage(func(s *sql.Selector) { @@ -1230,6 +1237,131 @@ func UpdatedNotNil() predicate.DbPackage { }) } +// HashEQ applies the EQ predicate on the "hash" field. +func HashEQ(v string) predicate.DbPackage { + return predicate.DbPackage(func(s *sql.Selector) { + s.Where(sql.EQ(s.C(FieldHash), v)) + }) +} + +// HashNEQ applies the NEQ predicate on the "hash" field. +func HashNEQ(v string) predicate.DbPackage { + return predicate.DbPackage(func(s *sql.Selector) { + s.Where(sql.NEQ(s.C(FieldHash), v)) + }) +} + +// HashIn applies the In predicate on the "hash" field. +func HashIn(vs ...string) predicate.DbPackage { + v := make([]interface{}, len(vs)) + for i := range v { + v[i] = vs[i] + } + return predicate.DbPackage(func(s *sql.Selector) { + // if not arguments were provided, append the FALSE constants, + // since we can't apply "IN ()". This will make this predicate falsy. + if len(v) == 0 { + s.Where(sql.False()) + return + } + s.Where(sql.In(s.C(FieldHash), v...)) + }) +} + +// HashNotIn applies the NotIn predicate on the "hash" field. +func HashNotIn(vs ...string) predicate.DbPackage { + v := make([]interface{}, len(vs)) + for i := range v { + v[i] = vs[i] + } + return predicate.DbPackage(func(s *sql.Selector) { + // if not arguments were provided, append the FALSE constants, + // since we can't apply "IN ()". This will make this predicate falsy. + if len(v) == 0 { + s.Where(sql.False()) + return + } + s.Where(sql.NotIn(s.C(FieldHash), v...)) + }) +} + +// HashGT applies the GT predicate on the "hash" field. +func HashGT(v string) predicate.DbPackage { + return predicate.DbPackage(func(s *sql.Selector) { + s.Where(sql.GT(s.C(FieldHash), v)) + }) +} + +// HashGTE applies the GTE predicate on the "hash" field. +func HashGTE(v string) predicate.DbPackage { + return predicate.DbPackage(func(s *sql.Selector) { + s.Where(sql.GTE(s.C(FieldHash), v)) + }) +} + +// HashLT applies the LT predicate on the "hash" field. +func HashLT(v string) predicate.DbPackage { + return predicate.DbPackage(func(s *sql.Selector) { + s.Where(sql.LT(s.C(FieldHash), v)) + }) +} + +// HashLTE applies the LTE predicate on the "hash" field. +func HashLTE(v string) predicate.DbPackage { + return predicate.DbPackage(func(s *sql.Selector) { + s.Where(sql.LTE(s.C(FieldHash), v)) + }) +} + +// HashContains applies the Contains predicate on the "hash" field. +func HashContains(v string) predicate.DbPackage { + return predicate.DbPackage(func(s *sql.Selector) { + s.Where(sql.Contains(s.C(FieldHash), v)) + }) +} + +// HashHasPrefix applies the HasPrefix predicate on the "hash" field. +func HashHasPrefix(v string) predicate.DbPackage { + return predicate.DbPackage(func(s *sql.Selector) { + s.Where(sql.HasPrefix(s.C(FieldHash), v)) + }) +} + +// HashHasSuffix applies the HasSuffix predicate on the "hash" field. +func HashHasSuffix(v string) predicate.DbPackage { + return predicate.DbPackage(func(s *sql.Selector) { + s.Where(sql.HasSuffix(s.C(FieldHash), v)) + }) +} + +// HashIsNil applies the IsNil predicate on the "hash" field. +func HashIsNil() predicate.DbPackage { + return predicate.DbPackage(func(s *sql.Selector) { + s.Where(sql.IsNull(s.C(FieldHash))) + }) +} + +// HashNotNil applies the NotNil predicate on the "hash" field. +func HashNotNil() predicate.DbPackage { + return predicate.DbPackage(func(s *sql.Selector) { + s.Where(sql.NotNull(s.C(FieldHash))) + }) +} + +// HashEqualFold applies the EqualFold predicate on the "hash" field. +func HashEqualFold(v string) predicate.DbPackage { + return predicate.DbPackage(func(s *sql.Selector) { + s.Where(sql.EqualFold(s.C(FieldHash), v)) + }) +} + +// HashContainsFold applies the ContainsFold predicate on the "hash" field. +func HashContainsFold(v string) predicate.DbPackage { + return predicate.DbPackage(func(s *sql.Selector) { + s.Where(sql.ContainsFold(s.C(FieldHash), v)) + }) +} + // And groups predicates with the AND operator between them. func And(predicates ...predicate.DbPackage) predicate.DbPackage { return predicate.DbPackage(func(s *sql.Selector) { diff --git a/ent/dbpackage_create.go b/ent/dbpackage_create.go index 7ba1f0f..d4f2569 100644 --- a/ent/dbpackage_create.go +++ b/ent/dbpackage_create.go @@ -142,6 +142,20 @@ func (dpc *DbPackageCreate) SetNillableUpdated(t *time.Time) *DbPackageCreate { return dpc } +// SetHash sets the "hash" field. +func (dpc *DbPackageCreate) SetHash(s string) *DbPackageCreate { + dpc.mutation.SetHash(s) + return dpc +} + +// SetNillableHash sets the "hash" field if the given value is not nil. +func (dpc *DbPackageCreate) SetNillableHash(s *string) *DbPackageCreate { + if s != nil { + dpc.SetHash(*s) + } + return dpc +} + // Mutation returns the DbPackageMutation object of the builder. func (dpc *DbPackageCreate) Mutation() *DbPackageMutation { return dpc.mutation @@ -354,6 +368,14 @@ func (dpc *DbPackageCreate) createSpec() (*DbPackage, *sqlgraph.CreateSpec) { }) _node.Updated = value } + if value, ok := dpc.mutation.Hash(); ok { + _spec.Fields = append(_spec.Fields, &sqlgraph.FieldSpec{ + Type: field.TypeString, + Value: value, + Column: dbpackage.FieldHash, + }) + _node.Hash = value + } return _node, _spec } diff --git a/ent/dbpackage_update.go b/ent/dbpackage_update.go index 943757a..8ef1193 100644 --- a/ent/dbpackage_update.go +++ b/ent/dbpackage_update.go @@ -199,6 +199,26 @@ func (dpu *DbPackageUpdate) ClearUpdated() *DbPackageUpdate { return dpu } +// SetHash sets the "hash" field. +func (dpu *DbPackageUpdate) SetHash(s string) *DbPackageUpdate { + dpu.mutation.SetHash(s) + return dpu +} + +// SetNillableHash sets the "hash" field if the given value is not nil. +func (dpu *DbPackageUpdate) SetNillableHash(s *string) *DbPackageUpdate { + if s != nil { + dpu.SetHash(*s) + } + return dpu +} + +// ClearHash clears the value of the "hash" field. +func (dpu *DbPackageUpdate) ClearHash() *DbPackageUpdate { + dpu.mutation.ClearHash() + return dpu +} + // Mutation returns the DbPackageMutation object of the builder. func (dpu *DbPackageUpdate) Mutation() *DbPackageMutation { return dpu.mutation @@ -430,6 +450,19 @@ func (dpu *DbPackageUpdate) sqlSave(ctx context.Context) (n int, err error) { Column: dbpackage.FieldUpdated, }) } + if value, ok := dpu.mutation.Hash(); ok { + _spec.Fields.Set = append(_spec.Fields.Set, &sqlgraph.FieldSpec{ + Type: field.TypeString, + Value: value, + Column: dbpackage.FieldHash, + }) + } + if dpu.mutation.HashCleared() { + _spec.Fields.Clear = append(_spec.Fields.Clear, &sqlgraph.FieldSpec{ + Type: field.TypeString, + Column: dbpackage.FieldHash, + }) + } if n, err = sqlgraph.UpdateNodes(ctx, dpu.driver, _spec); err != nil { if _, ok := err.(*sqlgraph.NotFoundError); ok { err = &NotFoundError{dbpackage.Label} @@ -621,6 +654,26 @@ func (dpuo *DbPackageUpdateOne) ClearUpdated() *DbPackageUpdateOne { return dpuo } +// SetHash sets the "hash" field. +func (dpuo *DbPackageUpdateOne) SetHash(s string) *DbPackageUpdateOne { + dpuo.mutation.SetHash(s) + return dpuo +} + +// SetNillableHash sets the "hash" field if the given value is not nil. +func (dpuo *DbPackageUpdateOne) SetNillableHash(s *string) *DbPackageUpdateOne { + if s != nil { + dpuo.SetHash(*s) + } + return dpuo +} + +// ClearHash clears the value of the "hash" field. +func (dpuo *DbPackageUpdateOne) ClearHash() *DbPackageUpdateOne { + dpuo.mutation.ClearHash() + return dpuo +} + // Mutation returns the DbPackageMutation object of the builder. func (dpuo *DbPackageUpdateOne) Mutation() *DbPackageMutation { return dpuo.mutation @@ -876,6 +929,19 @@ func (dpuo *DbPackageUpdateOne) sqlSave(ctx context.Context) (_node *DbPackage, Column: dbpackage.FieldUpdated, }) } + if value, ok := dpuo.mutation.Hash(); ok { + _spec.Fields.Set = append(_spec.Fields.Set, &sqlgraph.FieldSpec{ + Type: field.TypeString, + Value: value, + Column: dbpackage.FieldHash, + }) + } + if dpuo.mutation.HashCleared() { + _spec.Fields.Clear = append(_spec.Fields.Clear, &sqlgraph.FieldSpec{ + Type: field.TypeString, + Column: dbpackage.FieldHash, + }) + } _node = &DbPackage{config: dpuo.config} _spec.Assign = _node.assignValues _spec.ScanValues = _node.scanValues diff --git a/ent/migrate/schema.go b/ent/migrate/schema.go index 8f97c2b..e038541 100644 --- a/ent/migrate/schema.go +++ b/ent/migrate/schema.go @@ -22,6 +22,7 @@ var ( {Name: "build_time", Type: field.TypeTime, Nullable: true}, {Name: "build_duration", Type: field.TypeUint64, Nullable: true}, {Name: "updated", Type: field.TypeTime, Nullable: true}, + {Name: "hash", Type: field.TypeString, Nullable: true}, } // DbPackagesTable holds the schema information for the "db_packages" table. DbPackagesTable = &schema.Table{ diff --git a/ent/mutation.go b/ent/mutation.go index c906d74..8023e8b 100644 --- a/ent/mutation.go +++ b/ent/mutation.go @@ -45,6 +45,7 @@ type DbPackageMutation struct { build_duration *uint64 addbuild_duration *uint64 updated *time.Time + hash *string clearedFields map[string]struct{} done bool oldValue func(context.Context) (*DbPackage, error) @@ -658,6 +659,55 @@ func (m *DbPackageMutation) ResetUpdated() { delete(m.clearedFields, dbpackage.FieldUpdated) } +// SetHash sets the "hash" field. +func (m *DbPackageMutation) SetHash(s string) { + m.hash = &s +} + +// Hash returns the value of the "hash" field in the mutation. +func (m *DbPackageMutation) Hash() (r string, exists bool) { + v := m.hash + if v == nil { + return + } + return *v, true +} + +// OldHash returns the old "hash" field's value of the DbPackage entity. +// If the DbPackage object wasn't provided to the builder, the object is fetched from the database. +// An error is returned if the mutation operation is not UpdateOne, or the database query fails. +func (m *DbPackageMutation) OldHash(ctx context.Context) (v string, err error) { + if !m.op.Is(OpUpdateOne) { + return v, fmt.Errorf("OldHash is only allowed on UpdateOne operations") + } + if m.id == nil || m.oldValue == nil { + return v, fmt.Errorf("OldHash requires an ID field in the mutation") + } + oldValue, err := m.oldValue(ctx) + if err != nil { + return v, fmt.Errorf("querying old value for OldHash: %w", err) + } + return oldValue.Hash, nil +} + +// ClearHash clears the value of the "hash" field. +func (m *DbPackageMutation) ClearHash() { + m.hash = nil + m.clearedFields[dbpackage.FieldHash] = struct{}{} +} + +// HashCleared returns if the "hash" field was cleared in this mutation. +func (m *DbPackageMutation) HashCleared() bool { + _, ok := m.clearedFields[dbpackage.FieldHash] + return ok +} + +// ResetHash resets all changes to the "hash" field. +func (m *DbPackageMutation) ResetHash() { + m.hash = nil + delete(m.clearedFields, dbpackage.FieldHash) +} + // Op returns the operation name. func (m *DbPackageMutation) Op() Op { return m.op @@ -672,7 +722,7 @@ func (m *DbPackageMutation) Type() string { // order to get all numeric fields that were incremented/decremented, call // AddedFields(). func (m *DbPackageMutation) Fields() []string { - fields := make([]string, 0, 11) + fields := make([]string, 0, 12) if m.pkgbase != nil { fields = append(fields, dbpackage.FieldPkgbase) } @@ -706,6 +756,9 @@ func (m *DbPackageMutation) Fields() []string { if m.updated != nil { fields = append(fields, dbpackage.FieldUpdated) } + if m.hash != nil { + fields = append(fields, dbpackage.FieldHash) + } return fields } @@ -736,6 +789,8 @@ func (m *DbPackageMutation) Field(name string) (ent.Value, bool) { return m.BuildDuration() case dbpackage.FieldUpdated: return m.Updated() + case dbpackage.FieldHash: + return m.Hash() } return nil, false } @@ -767,6 +822,8 @@ func (m *DbPackageMutation) OldField(ctx context.Context, name string) (ent.Valu return m.OldBuildDuration(ctx) case dbpackage.FieldUpdated: return m.OldUpdated(ctx) + case dbpackage.FieldHash: + return m.OldHash(ctx) } return nil, fmt.Errorf("unknown DbPackage field %s", name) } @@ -853,6 +910,13 @@ func (m *DbPackageMutation) SetField(name string, value ent.Value) error { } m.SetUpdated(v) return nil + case dbpackage.FieldHash: + v, ok := value.(string) + if !ok { + return fmt.Errorf("unexpected type %T for field %s", value, name) + } + m.SetHash(v) + return nil } return fmt.Errorf("unknown DbPackage field %s", name) } @@ -931,6 +995,9 @@ func (m *DbPackageMutation) ClearedFields() []string { if m.FieldCleared(dbpackage.FieldUpdated) { fields = append(fields, dbpackage.FieldUpdated) } + if m.FieldCleared(dbpackage.FieldHash) { + fields = append(fields, dbpackage.FieldHash) + } return fields } @@ -966,6 +1033,9 @@ func (m *DbPackageMutation) ClearField(name string) error { case dbpackage.FieldUpdated: m.ClearUpdated() return nil + case dbpackage.FieldHash: + m.ClearHash() + return nil } return fmt.Errorf("unknown DbPackage nullable field %s", name) } @@ -1007,6 +1077,9 @@ func (m *DbPackageMutation) ResetField(name string) error { case dbpackage.FieldUpdated: m.ResetUpdated() return nil + case dbpackage.FieldHash: + m.ResetHash() + return nil } return fmt.Errorf("unknown DbPackage field %s", name) } diff --git a/ent/schema/dbpackage.go b/ent/schema/dbpackage.go index 46d3f40..c5e898e 100644 --- a/ent/schema/dbpackage.go +++ b/ent/schema/dbpackage.go @@ -24,6 +24,7 @@ func (DbPackage) Fields() []ent.Field { field.Time("build_time").Optional(), field.Uint64("build_duration").Positive().Optional(), field.Time("updated").Optional(), + field.String("hash").Optional(), } } diff --git a/go.mod b/go.mod index d34b749..6ed1354 100644 --- a/go.mod +++ b/go.mod @@ -12,4 +12,5 @@ require ( github.com/sirupsen/logrus v1.8.1 github.com/wercker/journalhook v0.0.0-20180428041537-5d0a5ae867b3 gopkg.in/yaml.v2 v2.4.0 + lukechampine.com/blake3 v1.1.5 ) diff --git a/go.sum b/go.sum index 96dcc6f..2d13804 100644 --- a/go.sum +++ b/go.sum @@ -62,6 +62,7 @@ github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-openapi/inflect v0.19.0 h1:9jCH9scKIbHeV9m12SmPilScz6krDxKRasNNSNPXu/4= github.com/go-openapi/inflect v0.19.0/go.mod h1:lHpZVlpIQqLyKwJ4N+YSc9hchQy/i12fJykb83CRBH4= github.com/go-sql-driver/mysql v1.5.1-0.20200311113236-681ffa848bae/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -137,6 +138,8 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s= +github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= @@ -148,6 +151,7 @@ github.com/lib/pq v1.10.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-sqlite3 v1.14.6 h1:dNPt6NO46WmLVt2DLNpwczCmdV5boIZ6g/tlDrlRUbg= github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= @@ -168,6 +172,7 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= @@ -202,9 +207,11 @@ github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4k github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v1.1.3 h1:xghbfqPkxzxP3C/f3n5DdpAbdKLj4ZE4BWQI362l53M= github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -220,8 +227,6 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1 github.com/wercker/journalhook v0.0.0-20180428041537-5d0a5ae867b3 h1:shC1HB1UogxN5Ech3Yqaaxj1X/P656PPCB4RbojIJqc= github.com/wercker/journalhook v0.0.0-20180428041537-5d0a5ae867b3/go.mod h1:XCsSkdKK4gwBMNrOCZWww0pX6AOt+2gYc5Z6jBRrNVg= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/yargevad/filepathx v1.0.0 h1:SYcT+N3tYGi+NvazubCNlvgIPbzAk7i7y2dwg3I5FYc= -github.com/yargevad/filepathx v1.0.0/go.mod h1:BprfX/gpYNJHJfc35GjRRpVcwWXS89gGulUIU5tK3tA= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= @@ -255,6 +260,7 @@ golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -325,10 +331,12 @@ golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= @@ -384,4 +392,6 @@ honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +lukechampine.com/blake3 v1.1.5 h1:hsACfxWvLdGmjYbWGrumQIphOvO+ZruZehWtgd2fxoM= +lukechampine.com/blake3 v1.1.5/go.mod h1:hE8RpzdO8ttZ7446CXEwDP1eu2V4z7stv0Urj1El20g= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= diff --git a/main.go b/main.go index 6f659b3..10fc1c7 100644 --- a/main.go +++ b/main.go @@ -11,10 +11,8 @@ import ( "github.com/Morganamilo/go-srcinfo" _ "github.com/mattn/go-sqlite3" log "github.com/sirupsen/logrus" - "github.com/wercker/journalhook" "gopkg.in/yaml.v2" "html/template" - "math/rand" "os" "os/exec" "os/signal" @@ -222,13 +220,13 @@ func (b *BuildManager) parseWorker() { if contains(info.Arch, "any") { log.Debugf("Skipped %s: any-Package", info.Pkgbase) dbLock.Lock() - dbPkg = dbPkg.Update().SetStatus(SKIPPED).SetSkipReason("arch = any").SaveX(context.Background()) + dbPkg = dbPkg.Update().SetStatus(SKIPPED).SetSkipReason("arch = any").SetHash(pkg.Hash).SaveX(context.Background()) dbLock.Unlock() skipping = true } else if contains(conf.Blacklist.Packages, info.Pkgbase) { log.Debugf("Skipped %s: blacklisted package", info.Pkgbase) dbLock.Lock() - dbPkg = dbPkg.Update().SetStatus(SKIPPED).SetSkipReason("blacklisted").SaveX(context.Background()) + dbPkg = dbPkg.Update().SetStatus(SKIPPED).SetSkipReason("blacklisted").SetHash(pkg.Hash).SaveX(context.Background()) dbLock.Unlock() skipping = true } else if contains(info.MakeDepends, "ghc") || contains(info.MakeDepends, "haskell-ghc") || contains(info.Depends, "ghc") || contains(info.Depends, "haskell-ghc") { @@ -237,13 +235,13 @@ func (b *BuildManager) parseWorker() { // https://git.harting.dev/anonfunc/ALHP.GO/issues/11 log.Debugf("Skipped %s: haskell package", info.Pkgbase) dbLock.Lock() - dbPkg = dbPkg.Update().SetStatus(SKIPPED).SetSkipReason("blacklisted (haskell)").SaveX(context.Background()) + dbPkg = dbPkg.Update().SetStatus(SKIPPED).SetSkipReason("blacklisted (haskell)").SetHash(pkg.Hash).SaveX(context.Background()) dbLock.Unlock() skipping = true } else if isPkgFailed(pkg) { log.Debugf("Skipped %s: failed build", info.Pkgbase) dbLock.Lock() - dbPkg = dbPkg.Update().SetStatus(FAILED).SetSkipReason("").SaveX(context.Background()) + dbPkg = dbPkg.Update().SetStatus(FAILED).SetSkipReason("").SetHash(pkg.Hash).SaveX(context.Background()) dbLock.Unlock() skipping = true } @@ -261,7 +259,7 @@ func (b *BuildManager) parseWorker() { if repoVer != "" && alpm.VerCmp(repoVer, pkg.Version) > 0 { log.Debugf("Skipped %s: Version in repo higher than in PKGBUILD (%s < %s)", info.Pkgbase, pkg.Version, repoVer) dbLock.Lock() - dbPkg = dbPkg.Update().SetStatus(LATEST).SetSkipReason("").SaveX(context.Background()) + dbPkg = dbPkg.Update().SetStatus(LATEST).SetSkipReason("").SetHash(pkg.Hash).SaveX(context.Background()) dbLock.Unlock() b.parseWG.Done() continue @@ -429,7 +427,7 @@ func (b *BuildManager) repoWorker(repo string) { dbPkg := getDbPackage(pkg) dbLock.Lock() - dbPkg = dbPkg.Update().SetStatus(LATEST).SetSkipReason("").SetRepoVersion(pkg.Version).SaveX(context.Background()) + dbPkg = dbPkg.Update().SetStatus(LATEST).SetSkipReason("").SetRepoVersion(pkg.Version).SetHash(pkg.Hash).SaveX(context.Background()) dbLock.Unlock() cmd = exec.Command("paccache", @@ -495,17 +493,6 @@ func (b *BuildManager) syncWorker() { for { b.buildWG.Wait() - /* TODO: Use `v` to print rudimentary stats - var v []struct { - Status int `json:"status"` - Count int `json:"count"` - } - - dbLock.RLock() - db.DbPackage.Query().GroupBy(dbpackage.FieldStatus).Aggregate(ent.Count()).ScanX(context.Background(), &v) - dbLock.RUnlock() - */ - for gitDir, gitURL := range conf.Svn2git { gitPath := filepath.Join(conf.Basedir.Upstream, gitDir) @@ -544,10 +531,6 @@ func (b *BuildManager) syncWorker() { pkgBuilds, err := Glob(filepath.Join(conf.Basedir.Upstream, "/**/PKGBUILD")) check(err) - // Shuffle pkgbuilds to spread out long-running builds, otherwise pkgBuilds is alphabetically-sorted - rand.Seed(time.Now().UnixNano()) - rand.Shuffle(len(pkgBuilds), func(i, j int) { pkgBuilds[i], pkgBuilds[j] = pkgBuilds[j], pkgBuilds[i] }) - for _, pkgbuild := range pkgBuilds { if b.exit { return @@ -560,6 +543,31 @@ func (b *BuildManager) syncWorker() { continue } + // compare b3sum of PKGBUILD file to hash in database, only proceed if hash differs + // reduces the amount of PKGBUILDs that need to be parsed with makepkg, which is _really_ slow, significantly + dbLock.RLock() + dbPkg, dbErr := db.DbPackage.Query().Where(dbpackage.And(dbpackage.Pkgbase(sPkgbuild[len(sPkgbuild)-4]), dbpackage.Repository(repo))).Only(context.Background()) + dbLock.RUnlock() + + if dbErr != nil { + switch dbErr.(type) { + case *ent.NotFoundError: + log.Debugf("[%s/%s] Package not found in database", repo, sPkgbuild[len(sPkgbuild)-4]) + break + default: + log.Errorf("[%s/%s] Problem querying db for package: %v", repo, sPkgbuild[len(sPkgbuild)-4], dbErr) + } + } + + b3s, err := b3sum(pkgbuild) + check(err) + + if dbPkg != nil && b3s == dbPkg.Hash { + log.Debugf("[%s/%s] Skipped: PKGBUILD hash matches db (%s)", repo, sPkgbuild[len(sPkgbuild)-4], b3s) + continue + } + + // send to parse for _, march := range conf.March { b.parseWG.Add(1) b.parse <- &BuildPackage{ @@ -568,6 +576,7 @@ func (b *BuildManager) syncWorker() { Repo: strings.Split(repo, "-")[0], March: march, FullRepo: strings.Split(repo, "-")[0] + "-" + march, + Hash: b3s, } } } @@ -590,7 +599,7 @@ func main() { lvl, err := log.ParseLevel(conf.Logging.Level) check(err) log.SetLevel(lvl) - journalhook.Enable() + // journalhook.Enable() err = syscall.Setpriority(syscall.PRIO_PROCESS, 0, 5) if err != nil { diff --git a/utils.go b/utils.go index d03aaba..703d37a 100644 --- a/utils.go +++ b/utils.go @@ -5,6 +5,7 @@ import ( "ALHP.go/ent/dbpackage" "bufio" "context" + "encoding/hex" "fmt" "github.com/Jguer/go-alpm/v2" paconf "github.com/Morganamilo/go-pacmanconf" @@ -12,6 +13,7 @@ import ( log "github.com/sirupsen/logrus" "io" "io/fs" + "lukechampine.com/blake3" "os" "os/exec" "path/filepath" @@ -44,6 +46,7 @@ type BuildPackage struct { March string FullRepo string Version string + Hash string } type BuildManager struct { @@ -100,6 +103,22 @@ func check(e error) { } } +func b3sum(filePath string) (string, error) { + file, err := os.Open(filePath) + if err != nil { + return "", err + } + defer func(file *os.File) { + check(file.Close()) + }(file) + + hash := blake3.New(32, nil) + if _, err := io.Copy(hash, file); err != nil { + return "", err + } + return hex.EncodeToString(hash.Sum(nil)), nil +} + func containsSubStr(str string, subList []string) bool { for _, checkStr := range subList { if strings.Contains(str, checkStr) {