strptrk/ALHP.GO
1
0
Fork 0
forked from ALHP/ALHP.GO
Code Pull Requests Activity

faster SRCINFO parsing + memory-based building

Browse Source 
Switched to parsing srcinfo with channels on all available cpus, speeding
up srcinfo-parsing and queue generation by a lot.

New memory-limit based building will max out the available memory while
not building the same packages at the same time for different marchs,
fixing some long-standing bugs like firefox not building at the same time
because the same ports are used for profile-based optimization. This
also drops the artificial delay on build-start, speeding up things even
more. This also means there is no hard-coded limit on how many packages
can be build at once anymore. As long as there is RAM available, builds will be
started.
This commit is contained in:
Giovanni Harting
2023-03-14 00:39:15 +01:00
parent 9baa7b2bcb
commit ece8c4c7d9
26 changed files with 1157 additions and 1114 deletions
Download Patch File Download Diff File Expand all files Collapse all files

67
ent/client.go
View File

@@ -8,12 +8,12 @@ import (
"fmt"
"log"
"git.harting.dev/ALHP/ALHP.GO/ent/migrate"
"git.harting.dev/ALHP/ALHP.GO/ent/dbpackage"
"somegit.dev/ALHP/ALHP.GO/ent/migrate"
"entgo.io/ent"
"entgo.io/ent/dialect"
"entgo.io/ent/dialect/sql"
"somegit.dev/ALHP/ALHP.GO/ent/dbpackage"
)
// Client is the client that holds all ent builders.
@@ -39,6 +39,55 @@ func (c *Client) init() {
c.DbPackage = NewDbPackageClient(c.config)
}
type (
// config is the configuration for the client and its builder.
config struct {
// driver used for executing database requests.
driver dialect.Driver
// debug enable a debug logging.
debug bool
// log used for logging on debug mode.
log func(...any)
// hooks to execute on mutations.
hooks *hooks
// interceptors to execute on queries.
inters *inters
}
// Option function to configure the client.
Option func(*config)
)
// options applies the options on the config object.
func (c *config) options(opts ...Option) {
for _, opt := range opts {
opt(c)
}
if c.debug {
c.driver = dialect.Debug(c.driver, c.log)
}
}
// Debug enables debug logging on the ent.Driver.
func Debug() Option {
return func(c *config) {
c.debug = true
}
}
// Log sets the logging function for debug mode.
func Log(fn func(...any)) Option {
return func(c *config) {
c.log = fn
}
}
// Driver configures the client driver.
func Driver(driver dialect.Driver) Option {
return func(c *config) {
c.driver = driver
}
}
// Open opens a database/sql.DB specified by the driver name and
// the data source name, and returns a new client attached to it.
// Optional parameters can be added for configuring the client.
@@ -154,7 +203,7 @@ func (c *DbPackageClient) Use(hooks ...Hook) {
c.hooks.DbPackage = append(c.hooks.DbPackage, hooks...)
}
// Use adds a list of query interceptors to the interceptors stack.
// Intercept adds a list of query interceptors to the interceptors stack.
// A call to `Intercept(f, g, h)` equals to `dbpackage.Intercept(f(g(h())))`.
func (c *DbPackageClient) Intercept(interceptors ...Interceptor) {
c.inters.DbPackage = append(c.inters.DbPackage, interceptors...)
@@ -255,3 +304,13 @@ func (c *DbPackageClient) mutate(ctx context.Context, m *DbPackageMutation) (Val
return nil, fmt.Errorf("ent: unknown DbPackage mutation op: %q", m.Op())
}
}
// hooks and interceptors per client, for fast access.
type (
hooks struct {
DbPackage []ent.Hook
}
inters struct {
DbPackage []ent.Interceptor
}
)

66
ent/config.go
View File

@@ -1,66 +0,0 @@
// Code generated by ent, DO NOT EDIT.
package ent
import (
"entgo.io/ent"
"entgo.io/ent/dialect"
)
// Option function to configure the client.
type Option func(*config)
// Config is the configuration for the client and its builder.
type config struct {
// driver used for executing database requests.
driver dialect.Driver
// debug enable a debug logging.
debug bool
// log used for logging on debug mode.
log func(...any)
// hooks to execute on mutations.
hooks *hooks
// interceptors to execute on queries.
inters *inters
}
// hooks and interceptors per client, for fast access.
type (
hooks struct {
DbPackage []ent.Hook
}
inters struct {
DbPackage []ent.Interceptor
}
)
// Options applies the options on the config object.
func (c *config) options(opts ...Option) {
for _, opt := range opts {
opt(c)
}
if c.debug {
c.driver = dialect.Debug(c.driver, c.log)
}
}
// Debug enables debug logging on the ent.Driver.
func Debug() Option {
return func(c *config) {
c.debug = true
}
}
// Log sets the logging function for debug mode.
func Log(fn func(...any)) Option {
return func(c *config) {
c.log = fn
}
}
// Driver configures the client driver.
func Driver(driver dialect.Driver) Option {
return func(c *config) {
c.driver = driver
}
}

33
ent/context.go
View File

@@ -1,33 +0,0 @@
// Code generated by ent, DO NOT EDIT.
package ent
import (
"context"
)
type clientCtxKey struct{}
// FromContext returns a Client stored inside a context, or nil if there isn't one.
func FromContext(ctx context.Context) *Client {
c, _ := ctx.Value(clientCtxKey{}).(*Client)
return c
}
// NewContext returns a new context with the given Client attached.
func NewContext(parent context.Context, c *Client) context.Context {
return context.WithValue(parent, clientCtxKey{}, c)
}
type txCtxKey struct{}
// TxFromContext returns a Tx stored inside a context, or nil if there isn't one.
func TxFromContext(ctx context.Context) *Tx {
tx, _ := ctx.Value(txCtxKey{}).(*Tx)
return tx
}
// NewTxContext returns a new context with the given Tx attached.
func NewTxContext(parent context.Context, tx *Tx) context.Context {
return context.WithValue(parent, txCtxKey{}, tx)
}

8
ent/dbpackage.go
View File

@@ -9,7 +9,7 @@ import (
"time"
"entgo.io/ent/dialect/sql"
"git.harting.dev/ALHP/ALHP.GO/ent/dbpackage"
"somegit.dev/ALHP/ALHP.GO/ent/dbpackage"
)
// DbPackage is the model entity for the DbPackage schema.
@@ -359,9 +359,3 @@ func (dp *DbPackage) String() string {
// DbPackages is a parsable slice of DbPackage.
type DbPackages []*DbPackage
func (dp DbPackages) config(cfg config) {
for _i := range dp {
dp[_i].config = cfg
}
}

2
ent/dbpackage/where.go
View File

@@ -6,7 +6,7 @@ import (
"time"
"entgo.io/ent/dialect/sql"
"git.harting.dev/ALHP/ALHP.GO/ent/predicate"
"somegit.dev/ALHP/ALHP.GO/ent/predicate"
)
// ID filters vertices based on their ID field.

10
ent/dbpackage_create.go
View File

@@ -10,7 +10,7 @@ import (
"entgo.io/ent/dialect/sql/sqlgraph"
"entgo.io/ent/schema/field"
"git.harting.dev/ALHP/ALHP.GO/ent/dbpackage"
"somegit.dev/ALHP/ALHP.GO/ent/dbpackage"
)
// DbPackageCreate is the builder for creating a DbPackage entity.
@@ -424,13 +424,7 @@ func (dpc *DbPackageCreate) sqlSave(ctx context.Context) (*DbPackage, error) {
func (dpc *DbPackageCreate) createSpec() (*DbPackage, *sqlgraph.CreateSpec) {
var (
_node = &DbPackage{config: dpc.config}
_spec = &sqlgraph.CreateSpec{
Table: dbpackage.Table,
ID: &sqlgraph.FieldSpec{
Type: field.TypeInt,
Column: dbpackage.FieldID,
},
}
_spec = sqlgraph.NewCreateSpec(dbpackage.Table, sqlgraph.NewFieldSpec(dbpackage.FieldID, field.TypeInt))
)
if value, ok := dpc.mutation.Pkgbase(); ok {
_spec.SetField(dbpackage.FieldPkgbase, field.TypeString, value)

24
ent/dbpackage_delete.go
View File

@@ -8,8 +8,8 @@ import (
"entgo.io/ent/dialect/sql"
"entgo.io/ent/dialect/sql/sqlgraph"
"entgo.io/ent/schema/field"
"git.harting.dev/ALHP/ALHP.GO/ent/dbpackage"
"git.harting.dev/ALHP/ALHP.GO/ent/predicate"
"somegit.dev/ALHP/ALHP.GO/ent/dbpackage"
"somegit.dev/ALHP/ALHP.GO/ent/predicate"
)
// DbPackageDelete is the builder for deleting a DbPackage entity.
@@ -40,15 +40,7 @@ func (dpd *DbPackageDelete) ExecX(ctx context.Context) int {
}
func (dpd *DbPackageDelete) sqlExec(ctx context.Context) (int, error) {
_spec := &sqlgraph.DeleteSpec{
Node: &sqlgraph.NodeSpec{
Table: dbpackage.Table,
ID: &sqlgraph.FieldSpec{
Type: field.TypeInt,
Column: dbpackage.FieldID,
},
},
}
_spec := sqlgraph.NewDeleteSpec(dbpackage.Table, sqlgraph.NewFieldSpec(dbpackage.FieldID, field.TypeInt))
if ps := dpd.mutation.predicates; len(ps) > 0 {
_spec.Predicate = func(selector *sql.Selector) {
for i := range ps {
@@ -69,6 +61,12 @@ type DbPackageDeleteOne struct {
dpd *DbPackageDelete
}
// Where appends a list predicates to the DbPackageDelete builder.
func (dpdo *DbPackageDeleteOne) Where(ps ...predicate.DbPackage) *DbPackageDeleteOne {
dpdo.dpd.mutation.Where(ps...)
return dpdo
}
// Exec executes the deletion query.
func (dpdo *DbPackageDeleteOne) Exec(ctx context.Context) error {
n, err := dpdo.dpd.Exec(ctx)
@@ -84,5 +82,7 @@ func (dpdo *DbPackageDeleteOne) Exec(ctx context.Context) error {
// ExecX is like Exec, but panics if an error occurs.
func (dpdo *DbPackageDeleteOne) ExecX(ctx context.Context) {
dpdo.dpd.ExecX(ctx)
if err := dpdo.Exec(ctx); err != nil {
panic(err)
}
}

28
ent/dbpackage_query.go
View File

@@ -10,8 +10,8 @@ import (
"entgo.io/ent/dialect/sql"
"entgo.io/ent/dialect/sql/sqlgraph"
"entgo.io/ent/schema/field"
"git.harting.dev/ALHP/ALHP.GO/ent/dbpackage"
"git.harting.dev/ALHP/ALHP.GO/ent/predicate"
"somegit.dev/ALHP/ALHP.GO/ent/dbpackage"
"somegit.dev/ALHP/ALHP.GO/ent/predicate"
)
// DbPackageQuery is the builder for querying DbPackage entities.
@@ -178,10 +178,12 @@ func (dpq *DbPackageQuery) AllX(ctx context.Context) []*DbPackage {
}
// IDs executes the query and returns a list of DbPackage IDs.
func (dpq *DbPackageQuery) IDs(ctx context.Context) ([]int, error) {
var ids []int
func (dpq *DbPackageQuery) IDs(ctx context.Context) (ids []int, err error) {
if dpq.ctx.Unique == nil && dpq.path != nil {
dpq.Unique(true)
}
ctx = setContextOp(ctx, dpq.ctx, "IDs")
if err := dpq.Select(dbpackage.FieldID).Scan(ctx, &ids); err != nil {
if err = dpq.Select(dbpackage.FieldID).Scan(ctx, &ids); err != nil {
return nil, err
}
return ids, nil
@@ -369,20 +371,12 @@ func (dpq *DbPackageQuery) sqlCount(ctx context.Context) (int, error) {
}
func (dpq *DbPackageQuery) querySpec() *sqlgraph.QuerySpec {
_spec := &sqlgraph.QuerySpec{
Node: &sqlgraph.NodeSpec{
Table: dbpackage.Table,
Columns: dbpackage.Columns,
ID: &sqlgraph.FieldSpec{
Type: field.TypeInt,
Column: dbpackage.FieldID,
},
},
From: dpq.sql,
Unique: true,
}
_spec := sqlgraph.NewQuerySpec(dbpackage.Table, dbpackage.Columns, sqlgraph.NewFieldSpec(dbpackage.FieldID, field.TypeInt))
_spec.From = dpq.sql
if unique := dpq.ctx.Unique; unique != nil {
_spec.Unique = *unique
} else if dpq.path != nil {
_spec.Unique = true
}
if fields := dpq.ctx.Fields; len(fields) > 0 {
_spec.Node.Columns = make([]string, 0, len(fields))

32
ent/dbpackage_update.go
View File

@@ -12,8 +12,8 @@ import (
"entgo.io/ent/dialect/sql/sqlgraph"
"entgo.io/ent/dialect/sql/sqljson"
"entgo.io/ent/schema/field"
"git.harting.dev/ALHP/ALHP.GO/ent/dbpackage"
"git.harting.dev/ALHP/ALHP.GO/ent/predicate"
"somegit.dev/ALHP/ALHP.GO/ent/dbpackage"
"somegit.dev/ALHP/ALHP.GO/ent/predicate"
)
// DbPackageUpdate is the builder for updating DbPackage entities.
@@ -536,16 +536,7 @@ func (dpu *DbPackageUpdate) sqlSave(ctx context.Context) (n int, err error) {
if err := dpu.check(); err != nil {
return n, err
}
_spec := &sqlgraph.UpdateSpec{
Node: &sqlgraph.NodeSpec{
Table: dbpackage.Table,
Columns: dbpackage.Columns,
ID: &sqlgraph.FieldSpec{
Type: field.TypeInt,
Column: dbpackage.FieldID,
},
},
}
_spec := sqlgraph.NewUpdateSpec(dbpackage.Table, dbpackage.Columns, sqlgraph.NewFieldSpec(dbpackage.FieldID, field.TypeInt))
if ps := dpu.mutation.predicates; len(ps) > 0 {
_spec.Predicate = func(selector *sql.Selector) {
for i := range ps {
@@ -1162,6 +1153,12 @@ func (dpuo *DbPackageUpdateOne) Mutation() *DbPackageMutation {
return dpuo.mutation
}
// Where appends a list predicates to the DbPackageUpdate builder.
func (dpuo *DbPackageUpdateOne) Where(ps ...predicate.DbPackage) *DbPackageUpdateOne {
dpuo.mutation.Where(ps...)
return dpuo
}
// Select allows selecting one or more fields (columns) of the returned entity.
// The default is selecting all fields defined in the entity schema.
func (dpuo *DbPackageUpdateOne) Select(field string, fields ...string) *DbPackageUpdateOne {
@@ -1231,16 +1228,7 @@ func (dpuo *DbPackageUpdateOne) sqlSave(ctx context.Context) (_node *DbPackage,
if err := dpuo.check(); err != nil {
return _node, err
}
_spec := &sqlgraph.UpdateSpec{
Node: &sqlgraph.NodeSpec{
Table: dbpackage.Table,
Columns: dbpackage.Columns,
ID: &sqlgraph.FieldSpec{
Type: field.TypeInt,
Column: dbpackage.FieldID,
},
},
}
_spec := sqlgraph.NewUpdateSpec(dbpackage.Table, dbpackage.Columns, sqlgraph.NewFieldSpec(dbpackage.FieldID, field.TypeInt))
id, ok := dpuo.mutation.ID()
if !ok {
return nil, &ValidationError{Name: "id", err: errors.New(`ent: missing "DbPackage.id" for update`)}

28
ent/ent.go
View File

@@ -11,7 +11,7 @@ import (
"entgo.io/ent"
"entgo.io/ent/dialect/sql"
"entgo.io/ent/dialect/sql/sqlgraph"
"git.harting.dev/ALHP/ALHP.GO/ent/dbpackage"
"somegit.dev/ALHP/ALHP.GO/ent/dbpackage"
)
// ent aliases to avoid import conflicts in user's code.
@@ -33,6 +33,32 @@ type (
MutateFunc = ent.MutateFunc
)
type clientCtxKey struct{}
// FromContext returns a Client stored inside a context, or nil if there isn't one.
func FromContext(ctx context.Context) *Client {
c, _ := ctx.Value(clientCtxKey{}).(*Client)
return c
}
// NewContext returns a new context with the given Client attached.
func NewContext(parent context.Context, c *Client) context.Context {
return context.WithValue(parent, clientCtxKey{}, c)
}
type txCtxKey struct{}
// TxFromContext returns a Tx stored inside a context, or nil if there isn't one.
func TxFromContext(ctx context.Context) *Tx {
tx, _ := ctx.Value(txCtxKey{}).(*Tx)
return tx
}
// NewTxContext returns a new context with the given Tx attached.
func NewTxContext(parent context.Context, tx *Tx) context.Context {
return context.WithValue(parent, txCtxKey{}, tx)
}
// OrderFunc applies an ordering on the sql selector.
type OrderFunc func(*sql.Selector)

6
ent/enttest/enttest.go
View File

@@ -5,12 +5,12 @@ package enttest
import (
"context"
"git.harting.dev/ALHP/ALHP.GO/ent"
"somegit.dev/ALHP/ALHP.GO/ent"
// required by schema hooks.
_ "git.harting.dev/ALHP/ALHP.GO/ent/runtime"
_ "somegit.dev/ALHP/ALHP.GO/ent/runtime"
"entgo.io/ent/dialect/sql/schema"
"git.harting.dev/ALHP/ALHP.GO/ent/migrate"
"somegit.dev/ALHP/ALHP.GO/ent/migrate"
)
type (

2
ent/hook/hook.go
View File

@@ -6,7 +6,7 @@ import (
"context"
"fmt"
"git.harting.dev/ALHP/ALHP.GO/ent"
"somegit.dev/ALHP/ALHP.GO/ent"
)
// The DbPackageFunc type is an adapter to allow the use of ordinary

5
ent/mutation.go
View File

@@ -9,11 +9,10 @@ import (
"sync"
"time"
"git.harting.dev/ALHP/ALHP.GO/ent/dbpackage"
"git.harting.dev/ALHP/ALHP.GO/ent/predicate"
"entgo.io/ent"
"entgo.io/ent/dialect/sql"
"somegit.dev/ALHP/ALHP.GO/ent/dbpackage"
"somegit.dev/ALHP/ALHP.GO/ent/predicate"
)
const (

4
ent/runtime.go
View File

@@ -3,8 +3,8 @@
package ent
import (
"git.harting.dev/ALHP/ALHP.GO/ent/dbpackage"
"git.harting.dev/ALHP/ALHP.GO/ent/schema"
"somegit.dev/ALHP/ALHP.GO/ent/dbpackage"
"somegit.dev/ALHP/ALHP.GO/ent/schema"
)
// The init function reads all schema descriptors with runtime code

6
ent/runtime/runtime.go
View File

@@ -2,9 +2,9 @@
package runtime
// The schema-stitching logic is generated in git.harting.dev/ALHP/ALHP.GO/ent/runtime.go
// The schema-stitching logic is generated in somegit.dev/ALHP/ALHP.GO/ent/runtime.go
const (
Version = "v0.11.6" // Version of ent codegen.
Sum = "h1:fMQwhuzbPv12AXdrAGyHoOcgh9r0D9F8WEsCRoUWxVc=" // Sum of ent codegen.
Version = "v0.11.9" // Version of ent codegen.
Sum = "h1:dbbCkAiPVTRBIJwoZctiSYjB7zxQIBOzVSU5H9VYIQI=" // Sum of ent codegen.
)

3
ent/schema/dbpackage.go
View File

@@ -15,7 +15,8 @@ func (DbPackage) Fields() []ent.Field {
return []ent.Field{
field.String("pkgbase").NotEmpty().Immutable(),
field.Strings("packages").Optional(),
field.Enum("status").Values("skipped", "failed", "build", "queued", "delayed", "building", "latest", "signing", "unknown").Default("unknown").Optional(),
field.Enum("status").Values("skipped", "failed", "build", "queued", "delayed", "building",
"latest", "signing", "unknown").Default("unknown").Optional(),
field.String("skip_reason").Optional(),
field.Enum("repository").Values("extra", "core", "community"),
field.String("march").NotEmpty().Immutable(),