From efd0f9a6af25b0a99f347a09cec94b2ceec40f0c Mon Sep 17 00:00:00 2001 From: Giovanni Harting <539@idlegandalf.com> Date: Mon, 25 Oct 2021 02:12:12 +0200 Subject: [PATCH] removed locks, switching to WAL for SQLite default --- config_example.yaml | 2 +- csgo/demo_loader.go | 17 ++-------- csgo/demo_parser.go | 19 +---------- go.sum | 7 ++++ main.go | 78 ++++++++++++--------------------------------- utils/utils.go | 66 ++++++++------------------------------ 6 files changed, 46 insertions(+), 143 deletions(-) diff --git a/config_example.yaml b/config_example.yaml index 4a6c5ba..3b87ef1 100644 --- a/config_example.yaml +++ b/config_example.yaml @@ -3,7 +3,7 @@ logging: db: driver: sqlite3 - connect_to: "file:csgowtfd.db?_fk=1&cache=shared" + connect_to: "file:csgowtfd.db?_journal_mode=WAL&_fk=1&cache=shared&_sync=NORMAL" parser: worker: 6 diff --git a/csgo/demo_loader.go b/csgo/demo_loader.go index 5df4623..ba795d0 100644 --- a/csgo/demo_loader.go +++ b/csgo/demo_loader.go @@ -18,7 +18,6 @@ import ( "io/ioutil" "math/rand" "os" - "sync" "time" ) @@ -34,7 +33,6 @@ type DemoMatchLoaderConfig struct { LoginKey string ServerList string Db *ent.Client - Lock *sync.RWMutex Worker int ApiKey string RateLimit ratelimit.Limiter @@ -51,7 +49,6 @@ type DemoMatchLoader struct { loginKey string serverList string db *ent.Client - lock *sync.RWMutex dp *DemoParser parseDemo chan *Demo parseMap map[string]bool @@ -171,12 +168,11 @@ func (d *DemoMatchLoader) Setup(config *DemoMatchLoaderConfig) error { d.loginKey = config.LoginKey d.sentryFile = config.Sentry d.serverList = config.ServerList - d.lock = config.Lock d.db = config.Db d.dp = &DemoParser{} d.parseMap = map[string]bool{} d.cache = config.Cache - err := d.dp.Setup(config.Db, config.Lock, config.Worker) + err := d.dp.Setup(config.Db, config.Worker) if err != nil { return err } @@ -357,9 +353,7 @@ func (d *DemoMatchLoader) gcWorker(apiKey string, rl ratelimit.Limiter) { continue } - d.lock.RLock() iMatch, err := d.db.Match.Get(context.Background(), matchId) - d.lock.RUnlock() if err != nil { switch e := err.(type) { case *ent.NotFoundError: @@ -399,10 +393,7 @@ func (d *DemoMatchLoader) gcWorker(apiKey string, rl ratelimit.Limiter) { var players []*ent.Player for _, accountId := range lastRound.GetReservation().GetAccountIds() { - tPlayer, err := utils.GetPlayer(&utils.DBWithLock{ - Client: d.db, - Lock: d.lock, - }, AccountId2SteamId(accountId), apiKey, rl) + tPlayer, err := utils.GetPlayer(d.db, AccountId2SteamId(accountId), apiKey, rl) if err != nil { log.Warningf("[DL] Unable to get player for steamid %d: %v", AccountId2SteamId(accountId), err) continue @@ -413,7 +404,6 @@ func (d *DemoMatchLoader) gcWorker(apiKey string, rl ratelimit.Limiter) { demo.Url = lastRound.GetMap() demo.MatchId = matchZero.GetMatchid() - d.lock.Lock() tMatch, err := d.db.Match.Create(). SetID(matchZero.GetMatchid()). AddPlayers(players...). @@ -426,7 +416,6 @@ func (d *DemoMatchLoader) gcWorker(apiKey string, rl ratelimit.Limiter) { SetScoreTeamB(int(lastRound.GetTeamScores()[1])). SetMatchResult(int(lastRound.GetMatchResult())). Save(context.Background()) - d.lock.Unlock() if err != nil { log.Warningf("[DL] Unable to create match %d: %v", matchZero.GetMatchid(), err) delete(d.parseMap, demo.ShareCode) @@ -468,7 +457,6 @@ func (d *DemoMatchLoader) gcWorker(apiKey string, rl ratelimit.Limiter) { } kills, deaths, assists, hs, score, mvp := playerStatsFromRound(lastRound, mPlayer) - d.lock.Lock() err := d.db.Stats.Create(). SetMatches(tMatch). SetPlayers(mPlayer). @@ -484,7 +472,6 @@ func (d *DemoMatchLoader) gcWorker(apiKey string, rl ratelimit.Limiter) { SetMk4(mk4). SetMk5(mk5). Exec(context.Background()) - d.lock.Unlock() if err != nil { log.Warningf("[DL] Unable to create stats for player %d in match %d: %v", mPlayer.ID, tMatch.ID, err) } diff --git a/csgo/demo_parser.go b/csgo/demo_parser.go index 8598e64..39312c1 100644 --- a/csgo/demo_parser.go +++ b/csgo/demo_parser.go @@ -14,7 +14,6 @@ import ( log "github.com/sirupsen/logrus" "io" "net/http" - "sync" "time" ) @@ -28,17 +27,15 @@ type DemoParser struct { demoQueue chan *Demo tempDir string db *ent.Client - lock *sync.RWMutex } type DemoNotFoundError struct { error } -func (p *DemoParser) Setup(db *ent.Client, lock *sync.RWMutex, worker int) error { +func (p *DemoParser) Setup(db *ent.Client, worker int) error { p.demoQueue = make(chan *Demo, 1000) p.db = db - p.lock = lock for i := 0; i < worker; i++ { go p.parseWorker() } @@ -70,13 +67,11 @@ func (p *DemoParser) downloadReplay(demo *Demo) (io.Reader, error) { } func (p *DemoParser) getDBPlayer(demo *Demo, demoPlayer *common.Player) (*ent.Stats, error) { - p.lock.RLock() tMatchPlayer, err := p.db.Stats.Query().WithMatches(func(q *ent.MatchQuery) { q.Where(match.ID(demo.MatchId)) }).WithPlayers(func(q *ent.PlayerQuery) { q.Where(player.ID(demoPlayer.SteamID64)) }).Only(context.Background()) - p.lock.RUnlock() if err != nil { return nil, err } @@ -108,9 +103,7 @@ func (p *DemoParser) parseWorker() { continue } - p.lock.RLock() tMatch, err := p.db.Match.Get(context.Background(), demo.MatchId) - p.lock.RUnlock() if err != nil { log.Errorf("[DP] Unable to get match %d: %v", demo.MatchId, err) continue @@ -139,9 +132,7 @@ func (p *DemoParser) parseWorker() { } downloadTime := time.Now().Sub(startTime) - p.lock.RLock() tStats, err := tMatch.QueryStats().WithPlayers().All(context.Background()) - p.lock.RUnlock() if err != nil { log.Errorf("[DP] Failed to find players for match %d: %v", demo.MatchId, err) continue @@ -294,9 +285,7 @@ func (p *DemoParser) parseWorker() { continue } - p.lock.Lock() err = tMatch.Update().SetMap(demoParser.Header().MapName).SetDemoParsed(true).Exec(context.Background()) - p.lock.Unlock() if err != nil { log.Errorf("[DP] Unable to update match %d in database: %v", demo.MatchId, err) continue @@ -307,7 +296,6 @@ func (p *DemoParser) parseWorker() { tMatchPlayer.Color = stats.ColorGrey } - p.lock.Lock() nMatchPLayer, err := tMatchPlayer.Update(). SetDmgTeam(tMatchPlayer.DmgTeam). SetDmgEnemy(tMatchPlayer.DmgEnemy). @@ -328,25 +316,20 @@ func (p *DemoParser) parseWorker() { SetDmgTeam(tMatchPlayer.DmgTeam). SetDmgEnemy(tMatchPlayer.DmgEnemy). Save(context.Background()) - p.lock.Unlock() if err != nil { log.Errorf("[DP] Unable to update stats %d in database: %v", tMatchPlayer.PlayerStats, err) continue } for _, eqDmg := range eqMap[tMatchPlayer.PlayerStats] { - p.lock.Lock() err := p.db.WeaponStats.Create().SetStat(nMatchPLayer).SetDmg(eqDmg.Dmg).SetVictim(eqDmg.To).SetHitGroup(eqDmg.HitGroup).SetEqType(eqDmg.Eq).Exec(context.Background()) - p.lock.Unlock() if err != nil { log.Errorf("[DP] Unable to create WeaponStat: %v", err) } } for _, eco := range ecoMap[tMatchPlayer.PlayerStats] { - p.lock.Lock() err := p.db.RoundStats.Create().SetStat(nMatchPLayer).SetRound(uint(eco.Round)).SetBank(uint(eco.Bank)).SetEquipment(uint(eco.EqV)).SetSpent(uint(eco.Spent)).Exec(context.Background()) - p.lock.Unlock() if err != nil { log.Errorf("[DP] Unable to create RoundStat: %v", err) } diff --git a/go.sum b/go.sum index 1d35361..a67e548 100644 --- a/go.sum +++ b/go.sum @@ -120,6 +120,7 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-openapi/inflect v0.19.0 h1:9jCH9scKIbHeV9m12SmPilScz6krDxKRasNNSNPXu/4= github.com/go-openapi/inflect v0.19.0/go.mod h1:lHpZVlpIQqLyKwJ4N+YSc9hchQy/i12fJykb83CRBH4= github.com/go-redis/cache/v8 v8.4.3 h1:+RZ0pQM+zOd6h/oWCsOl3+nsCgii9rn26oCYmU87kN8= github.com/go-redis/cache/v8 v8.4.3/go.mod h1:5lQPQ63uyBt4aZuRmdvUJOJRRjPxfLtJtlcJ/z8o1jA= @@ -331,6 +332,7 @@ github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-sqlite3 v1.14.8/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/mattn/go-sqlite3 v1.14.9 h1:10HX2Td0ocZpYEjhilsuo6WWtUqttj2Kb0KtD86/KYA= @@ -365,6 +367,7 @@ github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtb github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= +github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -451,10 +454,12 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasO github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/cobra v1.1.3 h1:xghbfqPkxzxP3C/f3n5DdpAbdKLj4ZE4BWQI362l53M= github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= @@ -565,6 +570,7 @@ golang.org/x/mod v0.1.1-0.20191209134235-331c550502dd/go.mod h1:s0Qsj1ACt9ePp/hM golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.5.1-0.20210830214625-1b1db11ec8f4 h1:7Qds88gNaRx0Dz/1wOwXlR7asekh1B1u26wEwN6FcEI= golang.org/x/mod v0.5.1-0.20210830214625-1b1db11ec8f4/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -691,6 +697,7 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.5 h1:ouewzE6p+/VEB31YYnTbEJdi8pFqKp4P4n85vwo3DHA= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/main.go b/main.go index 16b0238..8a793a2 100644 --- a/main.go +++ b/main.go @@ -29,7 +29,6 @@ import ( "os" "os/signal" "strconv" - "sync" "syscall" "time" ) @@ -38,7 +37,7 @@ var ( conf = utils.Conf{} demoLoader = &csgo.DemoMatchLoader{} router *mux.Router - db *utils.DBWithLock + db *ent.Client rdb *redis.Client rdc *cache.Cache firstHK = true @@ -57,11 +56,9 @@ func housekeeping() { firstHK = false // update players from steam - db.Lock.RLock() - tPlayerNeedSteamUpdate, err := db.Client.Player.Query().Where( + tPlayerNeedSteamUpdate, err := db.Player.Query().Where( player.SteamUpdatedLTE(time.Now().UTC().AddDate(0, 0, -1)), ).All(context.Background()) - db.Lock.RUnlock() if err != nil { log.Errorf("[HK] Can't query players: %v", err) continue @@ -69,7 +66,7 @@ func housekeeping() { if len(tPlayerNeedSteamUpdate) > 0 { log.Infof("[HK] Refreshing %d profiles from steam", len(tPlayerNeedSteamUpdate)) - _, err = utils.UpdatePlayerFromSteam(tPlayerNeedSteamUpdate, db.Client, conf.Steam.APIKey, db.Lock, rL) + _, err = utils.UpdatePlayerFromSteam(tPlayerNeedSteamUpdate, db, conf.Steam.APIKey, rL) if err != nil { log.Warningf("[HK] Unable to update profiles from steam: %v", err) } @@ -81,8 +78,7 @@ func housekeeping() { continue } - db.Lock.RLock() - tPlayerNeedShareCodeUpdate, err := db.Client.Player.Query().Where( + tPlayerNeedShareCodeUpdate, err := db.Player.Query().Where( player.And( player.Or( player.SharecodeUpdatedLTE(time.Now().UTC().Add(time.Duration(-30)*time.Minute)), @@ -90,21 +86,18 @@ func housekeeping() { ), player.Not(player.AuthCodeIsNil()), )).All(context.Background()) - db.Lock.RUnlock() if err != nil { log.Errorf("[HK] Can't query players: %v", err) continue } for _, tPlayer := range tPlayerNeedShareCodeUpdate { - shareCodes, err := utils.GetNewShareCodesForPlayer(tPlayer, db.Lock, conf.Steam.APIKey, rL) + shareCodes, err := utils.GetNewShareCodesForPlayer(tPlayer, conf.Steam.APIKey, rL) if err != nil { switch err.(type) { case utils.AuthcodeUnauthorizedError: log.Infof("[HK] authCode for player %d is no longer valid", tPlayer.ID) - db.Lock.Lock() err = tPlayer.Update().ClearAuthCode().ClearSharecodeUpdated().Exec(context.Background()) - db.Lock.Unlock() if err != nil { log.Errorf("[HK] Unable to clear authcode for player %d: %v", tPlayer.ID, err) } @@ -129,9 +122,7 @@ func housekeeping() { } // try parsing demos not parsed - db.Lock.RLock() - tMatches, err := db.Client.Match.Query().Where(match.And(match.DateGT(time.Now().UTC().AddDate(0, 0, -30)), match.DemoParsed(false))).All(context.Background()) - db.Lock.RUnlock() + tMatches, err := db.Match.Query().Where(match.And(match.DateGT(time.Now().UTC().AddDate(0, 0, -30)), match.DemoParsed(false))).All(context.Background()) if err != nil { log.Warningf("[HK] Failure getting matches to retry parsing: %v", err) continue @@ -180,7 +171,7 @@ func getPlayerMeta(w http.ResponseWriter, r *http.Request) { metaStats := new(utils.MetaStatsResponse) err = rdc.Get(context.Background(), fmt.Sprintf(utils.SideMetaCacheKey, tPlayer.ID), &metaStats) if err != nil { - metaStats, err = utils.GetMetaStats(tPlayer, db.Lock) + metaStats, err = utils.GetMetaStats(tPlayer) if err != nil { log.Infof("[GPM] Unable to get MetaStats: %v", err) w.WriteHeader(http.StatusInternalServerError) @@ -221,7 +212,7 @@ func getPlayerMeta(w http.ResponseWriter, r *http.Request) { for _, p := range append(metaStats.BestMates, metaStats.MostMates...) { if p.Player.Name == "" { - tP, err := db.Client.Player.Get(context.Background(), p.Player.SteamID64) + tP, err := db.Player.Get(context.Background(), p.Player.SteamID64) if err != nil { log.Warningf("[GPM] Failure getting player: %v", err) w.WriteHeader(http.StatusInternalServerError) @@ -289,13 +280,9 @@ func getPlayer(w http.ResponseWriter, r *http.Request) { var tMatches []*ent.Match if !offsetTime.IsZero() { - db.Lock.RLock() tMatches, err = tPlayer.QueryMatches().Where(match.DateLT(offsetTime)).Order(ent.Desc(match.FieldDate)).Limit(10).All(context.Background()) - db.Lock.RUnlock() } else { - db.Lock.RLock() tMatches, err = tPlayer.QueryMatches().Order(ent.Desc(match.FieldDate)).Limit(10).All(context.Background()) - db.Lock.RUnlock() } if err != nil || len(tMatches) == 0 { log.Debugf("[GP] No matches found for player %s", id) @@ -311,7 +298,7 @@ func getPlayer(w http.ResponseWriter, r *http.Request) { metaStats := new(utils.MatchStats) err = rdc.Get(context.Background(), fmt.Sprintf(utils.MatchMetaCacheKey, tPlayer.ID), &metaStats) if err != nil { - wins, ties, losses, err := utils.GetMatchStats(tPlayer, db.Lock) + wins, ties, losses, err := utils.GetMatchStats(tPlayer) if err != nil { log.Errorf("[GP] Error retrieving match-stats for player %s: %v", id, err) w.WriteHeader(http.StatusInternalServerError) @@ -358,14 +345,12 @@ func getPlayer(w http.ResponseWriter, r *http.Request) { Parsed: iMatch.DemoParsed, } - db.Lock.RLock() tStats, err := iMatch.QueryStats().Modify(func(s *sql.Selector) { s.Select(stats.FieldTeamID, stats.FieldKills, stats.FieldDeaths, stats.FieldAssists, stats.FieldHeadshot, stats.FieldMvp, stats.FieldScore, stats.FieldMk2, stats.FieldMk3, stats.FieldMk4, stats.FieldMk5, stats.FieldRankOld, stats.FieldRankNew, stats.FieldDmgTeam, stats.FieldDmgEnemy) s.Where(sql.EQ(s.C(stats.PlayersColumn), tPlayer.ID)) }).Only(context.Background()) - db.Lock.RUnlock() if err != nil { response.Matches = append(response.Matches, mResponse) continue @@ -433,7 +418,7 @@ func deletePlayerTrack(w http.ResponseWriter, r *http.Request) { return } - _, err = utils.IsAuthCodeValid(tPlayer, db.Lock, conf.Steam.APIKey, "", authCode, nil) + _, err = utils.IsAuthCodeValid(tPlayer, conf.Steam.APIKey, "", authCode, nil) if err != nil { switch e := err.(type) { case utils.AuthcodeUnauthorizedError: @@ -447,9 +432,7 @@ func deletePlayerTrack(w http.ResponseWriter, r *http.Request) { } } - db.Lock.Lock() err = tPlayer.Update().ClearAuthCode().ClearSharecodeUpdated().Exec(context.Background()) - db.Lock.Unlock() if err != nil { log.Warningf("[PPT] update player failed: %+v", err) w.WriteHeader(http.StatusInternalServerError) @@ -484,7 +467,7 @@ func postPlayerTrack(w http.ResponseWriter, r *http.Request) { return } - _, err = utils.IsAuthCodeValid(tPlayer, db.Lock, conf.Steam.APIKey, shareCode, authCode, rL) + _, err = utils.IsAuthCodeValid(tPlayer, conf.Steam.APIKey, shareCode, authCode, rL) if err != nil { switch e := err.(type) { case utils.AuthcodeUnauthorizedError: @@ -502,9 +485,7 @@ func postPlayerTrack(w http.ResponseWriter, r *http.Request) { } } - db.Lock.Lock() err = tPlayer.Update().SetAuthCode(authCode).Exec(context.Background()) - db.Lock.Unlock() if err != nil { log.Warningf("[PPT] update player failed: %+v", err) w.WriteHeader(http.StatusInternalServerError) @@ -559,9 +540,7 @@ func getMatchRounds(w http.ResponseWriter, r *http.Request) { return } - db.Lock.RLock() - tStats, err := db.Client.Stats.Query().Where(stats.HasMatchesWith(match.ID(matchId))).All(context.Background()) - db.Lock.RUnlock() + tStats, err := db.Stats.Query().Where(stats.HasMatchesWith(match.ID(matchId))).All(context.Background()) if err != nil { log.Infof("[GMR] match %d not found: %+v", matchId, err) w.WriteHeader(http.StatusNotFound) @@ -571,9 +550,7 @@ func getMatchRounds(w http.ResponseWriter, r *http.Request) { resp := map[uint]map[string][]uint{} for _, stat := range tStats { - db.Lock.RLock() tRoundStats, err := stat.QueryRoundStats().All(context.Background()) - db.Lock.RUnlock() if err != nil { log.Warningf("[GMR] Unable to get RoundStats for player %d: %v", stat.PlayerStats, err) continue @@ -610,9 +587,7 @@ func getMatchWeapons(w http.ResponseWriter, r *http.Request) { return } - db.Lock.RLock() - tStats, err := db.Client.Stats.Query().Where(stats.HasMatchesWith(match.ID(matchId))).All(context.Background()) - db.Lock.RUnlock() + tStats, err := db.Stats.Query().Where(stats.HasMatchesWith(match.ID(matchId))).All(context.Background()) if err != nil { log.Infof("[GMW] match %d not found: %+v", matchId, err) w.WriteHeader(http.StatusNotFound) @@ -628,9 +603,7 @@ func getMatchWeapons(w http.ResponseWriter, r *http.Request) { } for _, stat := range tStats { - db.Lock.RLock() mWs, err := stat.QueryWeaponStats().All(context.Background()) - db.Lock.RUnlock() if err != nil { log.Warningf("[GMW] Unable to get WeaponStats for player %d: %v", stat.PlayerStats, err) continue @@ -676,9 +649,7 @@ func getMatch(w http.ResponseWriter, r *http.Request) { return } - db.Lock.RLock() - tMatch, err := db.Client.Match.Query().Where(match.ID(matchId)).Only(context.Background()) - db.Lock.RUnlock() + tMatch, err := db.Match.Query().Where(match.ID(matchId)).Only(context.Background()) if err != nil { log.Infof("[GM] match %d not found: %v", matchId, err) w.WriteHeader(http.StatusNotFound) @@ -702,9 +673,7 @@ func getMatch(w http.ResponseWriter, r *http.Request) { mResponse.ReplayURL = tMatch.ReplayURL } - db.Lock.RLock() tStats, err := tMatch.QueryStats().WithPlayers().All(context.Background()) - db.Lock.RUnlock() if err != nil { log.Errorf("[GM] Unable to find stats for match %d: %v", tMatch.ID, err) w.WriteHeader(http.StatusInternalServerError) @@ -810,10 +779,6 @@ func main() { journalhook.Enable() } - db = &utils.DBWithLock{ - Lock: new(sync.RWMutex), - } - if conf.Db.Driver == "pgx" { pdb, err := sql.Open("pgx", conf.Db.ConnectTo) if err != nil { @@ -821,27 +786,27 @@ func main() { } drv := sql.OpenDB(dialect.Postgres, pdb.DB()) - db.Client = ent.NewClient(ent.Driver(drv)) + db = ent.NewClient(ent.Driver(drv)) } else { - db.Client, err = ent.Open(conf.Db.Driver, conf.Db.ConnectTo) + db, err = ent.Open(conf.Db.Driver, conf.Db.ConnectTo) if err != nil { log.Panicf("Failed to open database %s: %v", conf.Db.ConnectTo, err) } defer func(Client *ent.Client) { _ = Client.Close() - }(db.Client) + }(db) } if *sqlDebugFlag { - db.Client = db.Client.Debug() + db = db.Debug() } - if err := db.Client.Schema.Create( + if err := db.Schema.Create( context.Background(), migrate.WithDropIndex(true), migrate.WithDropColumn(true), ); err != nil { - log.Panicf("Automigrate failed: %v", err) + log.Fatalf("Automigrate failed: %v", err) } rdb = redis.NewClient(&redis.Options{ @@ -865,8 +830,7 @@ func main() { Sentry: conf.Steam.Sentry, LoginKey: conf.Steam.LoginKey, ServerList: conf.Steam.ServerList, - Db: db.Client, - Lock: db.Lock, + Db: db, Worker: conf.Parser.Worker, ApiKey: conf.Steam.APIKey, RateLimit: rL, diff --git a/utils/utils.go b/utils/utils.go index 7375cef..e7a4f62 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -22,7 +22,6 @@ import ( "sort" "strconv" "strings" - "sync" "time" ) @@ -60,11 +59,6 @@ type Conf struct { } } -type DBWithLock struct { - Client *ent.Client - Lock *sync.RWMutex -} - type CommunityXML struct { SteamID64 uint64 `xml:"steamID64"` AvatarURL string `xml:"avatarFull"` @@ -221,8 +215,8 @@ func SendJSON(data interface{}, w http.ResponseWriter) error { return nil } -func GetMatchStats(dbPlayer *ent.Player, lock *sync.RWMutex) (int, int, int, error) { - wins, loss, ties, err := getWinLossTieFromPlayer(dbPlayer, lock) +func GetMatchStats(dbPlayer *ent.Player) (int, int, int, error) { + wins, loss, ties, err := getWinLossTieFromPlayer(dbPlayer) if err != nil { return 0, 0, 0, err } @@ -230,20 +224,16 @@ func GetMatchStats(dbPlayer *ent.Player, lock *sync.RWMutex) (int, int, int, err return wins, ties, loss, nil } -func GetMetaStats(dbPlayer *ent.Player, lock *sync.RWMutex) (*MetaStatsResponse, error) { +func GetMetaStats(dbPlayer *ent.Player) (*MetaStatsResponse, error) { mResponse := new(MetaStatsResponse) mResponse.Player = &PlayerResponse{SteamID64: dbPlayer.ID} - lock.RLock() tPlayers, err := dbPlayer.QueryMatches().QueryPlayers().Select(player.FieldID).All(context.Background()) - lock.RUnlock() if err != nil { return nil, err } - lock.RLock() matchIDs, err := dbPlayer.QueryMatches().IDs(context.Background()) - lock.RUnlock() if err != nil { return nil, err } @@ -266,14 +256,12 @@ func GetMetaStats(dbPlayer *ent.Player, lock *sync.RWMutex) (*MetaStatsResponse, SteamID64: s.ID, } - lock.RLock() pMatches, err := s.QueryMatches(). Select(match.FieldID, match.FieldMatchResult, match.FieldMap). Where(match.IDIn(matchIDs...)). WithStats(). Where(match.HasStatsWith(stats.Or(stats.PlayerStats(dbPlayer.ID), stats.PlayerStats(s.ID)))). All(context.Background()) - lock.RUnlock() if err != nil { return nil, err } @@ -307,10 +295,8 @@ func GetMetaStats(dbPlayer *ent.Player, lock *sync.RWMutex) (*MetaStatsResponse, } } - lock.RLock() wSs, err := subjectStats.QueryWeaponStats(). Select(weaponstats.FieldEqType, weaponstats.FieldDmg).All(context.Background()) - lock.RUnlock() if err != nil { return nil, err } @@ -396,13 +382,12 @@ func GetMetaStats(dbPlayer *ent.Player, lock *sync.RWMutex) (*MetaStatsResponse, return mResponse, nil } -func getWinLossTieFromPlayer(dbPlayer *ent.Player, lock *sync.RWMutex) (int, int, int, error) { +func getWinLossTieFromPlayer(dbPlayer *ent.Player) (int, int, int, error) { var res []struct { MatchResult int `json:"match_result"` Count int `json:"count"` } - lock.RLock() err := dbPlayer.QueryMatches().GroupBy(match.FieldMatchResult).Aggregate(func(s *sql.Selector) string { sT := sql.Table(stats.Table) @@ -416,16 +401,13 @@ func getWinLossTieFromPlayer(dbPlayer *ent.Player, lock *sync.RWMutex) (int, int )) return sql.Count("*") }).Scan(context.Background(), &res) - lock.RUnlock() if err != nil { return 0, 0, 0, err } - lock.RLock() total, err := dbPlayer.QueryMatches().Modify(func(s *sql.Selector) { s.Select("COUNT(*)") }).Int(context.Background()) - lock.RUnlock() if err != nil { return 0, 0, 0, err } @@ -447,13 +429,11 @@ func getWinLossTieFromPlayer(dbPlayer *ent.Player, lock *sync.RWMutex) (int, int return wins, total - wins - ties, ties, nil } -func IsAuthCodeValid(player *ent.Player, lock *sync.RWMutex, apiKey string, shareCode string, authCode string, rl ratelimit.Limiter) (bool, error) { +func IsAuthCodeValid(player *ent.Player, apiKey string, shareCode string, authCode string, rl ratelimit.Limiter) (bool, error) { var tMatch *ent.Match var err error if shareCode == "" { - lock.RLock() tMatch, err = player.QueryMatches().Order(ent.Asc(match.FieldDate)).First(context.Background()) - lock.RUnlock() if err != nil { return false, err } @@ -472,17 +452,13 @@ func IsAuthCodeValid(player *ent.Player, lock *sync.RWMutex, apiKey string, shar } } -func GetNewShareCodesForPlayer(player *ent.Player, lock *sync.RWMutex, apiKey string, rl ratelimit.Limiter) ([]string, error) { - lock.RLock() +func GetNewShareCodesForPlayer(player *ent.Player, apiKey string, rl ratelimit.Limiter) ([]string, error) { latestMatch, err := player.QueryMatches().Order(ent.Desc(match.FieldDate)).First(context.Background()) - lock.RUnlock() if err != nil { return nil, err } - lock.RLock() oldestMatch, err := player.QueryMatches().Order(ent.Asc(match.FieldDate)).First(context.Background()) - lock.RUnlock() if err != nil { return nil, err } @@ -506,9 +482,7 @@ func GetNewShareCodesForPlayer(player *ent.Player, lock *sync.RWMutex, apiKey st } } - lock.Lock() err = player.Update().SetSharecodeUpdated(time.Now().UTC()).SetOldestSharecodeSeen(oldestMatch.ShareCode).Exec(context.Background()) - lock.Unlock() if err != nil { return nil, err } @@ -562,7 +536,7 @@ func getNextShareCode(lastCode string, apiKey string, authCode string, steamId u return rJson.Result.Code, nil } -func GetPlayer(db *DBWithLock, id interface{}, apiKey string, rl ratelimit.Limiter) (*ent.Player, error) { +func GetPlayer(db *ent.Client, id interface{}, apiKey string, rl ratelimit.Limiter) (*ent.Player, error) { switch e := id.(type) { case uint64: return GetPlayerFromSteamID64(db, e, apiKey, rl) @@ -582,14 +556,12 @@ func GetPlayer(db *DBWithLock, id interface{}, apiKey string, rl ratelimit.Limit } } -func GetPlayerFromVanityURL(db *DBWithLock, id string, apiKey string, rl ratelimit.Limiter) (*ent.Player, error) { +func GetPlayerFromVanityURL(db *ent.Client, id string, apiKey string, rl ratelimit.Limiter) (*ent.Player, error) { if id == "" { return nil, fmt.Errorf("invalid arguments") } - db.Lock.RLock() - tPlayer, err := db.Client.Player.Query().Where(player.VanityURL(strings.ToLower(id))).Only(context.Background()) - db.Lock.RUnlock() + tPlayer, err := db.Player.Query().Where(player.VanityURL(strings.ToLower(id))).Only(context.Background()) if err == nil { return tPlayer, nil } else { @@ -614,21 +586,17 @@ func GetPlayerFromVanityURL(db *DBWithLock, id string, apiKey string, rl ratelim } } -func GetPlayerFromSteamID64(db *DBWithLock, steamID uint64, apiKey string, rl ratelimit.Limiter) (*ent.Player, error) { - db.Lock.RLock() - tPlayer, err := db.Client.Player.Get(context.Background(), steamID) - db.Lock.RUnlock() +func GetPlayerFromSteamID64(db *ent.Client, steamID uint64, apiKey string, rl ratelimit.Limiter) (*ent.Player, error) { + tPlayer, err := db.Player.Get(context.Background(), steamID) if err == nil { return tPlayer, nil } else { - db.Lock.Lock() - nPlayer, err := db.Client.Player.Create().SetID(steamID).Save(context.Background()) - db.Lock.Unlock() + nPlayer, err := db.Player.Create().SetID(steamID).Save(context.Background()) if err != nil { return nil, err } - uPlayer, err := UpdatePlayerFromSteam([]*ent.Player{nPlayer}, db.Client, apiKey, db.Lock, rl) + uPlayer, err := UpdatePlayerFromSteam([]*ent.Player{nPlayer}, db, apiKey, rl) if err != nil { return nil, err } @@ -641,7 +609,7 @@ func GetPlayerFromSteamID64(db *DBWithLock, steamID uint64, apiKey string, rl ra } } -func UpdatePlayerFromSteam(players []*ent.Player, db *ent.Client, apiKey string, lock *sync.RWMutex, rl ratelimit.Limiter) ([]*ent.Player, error) { +func UpdatePlayerFromSteam(players []*ent.Player, db *ent.Client, apiKey string, rl ratelimit.Limiter) ([]*ent.Player, error) { var idsToUpdate []uint64 for _, updatePlayer := range players { @@ -667,7 +635,6 @@ func UpdatePlayerFromSteam(players []*ent.Player, db *ent.Client, apiKey string, pS.ProfileURL = path.Base(pS.ProfileURL) } - lock.Lock() tPlayer, err := db.Player.UpdateOneID(pS.SteamID). SetName(pS.PersonaName). SetAvatar(pS.AvatarHash). @@ -677,7 +644,6 @@ func UpdatePlayerFromSteam(players []*ent.Player, db *ent.Client, apiKey string, SetSteamUpdated(time.Now().UTC()). SetProfileCreated(time.Unix(pS.TimeCreated, 0).UTC()). Save(context.Background()) - lock.Unlock() if err != nil { return nil, err } @@ -696,9 +662,7 @@ func UpdatePlayerFromSteam(players []*ent.Player, db *ent.Client, apiKey string, if ban.NumberOfVACBans > 0 { banDate := time.Now().UTC().AddDate(0, 0, -1*int(ban.DaysSinceLastBan)) - lock.Lock() err := db.Player.UpdateOneID(ban.SteamID).SetVacCount(int(ban.NumberOfVACBans)).SetVacDate(banDate).Exec(context.Background()) - lock.Unlock() if err != nil { return nil, err } @@ -706,9 +670,7 @@ func UpdatePlayerFromSteam(players []*ent.Player, db *ent.Client, apiKey string, if ban.NumberOfGameBans > 0 { banDate := time.Now().UTC().AddDate(0, 0, -1*int(ban.DaysSinceLastBan)) - lock.Lock() err := db.Player.UpdateOneID(ban.SteamID).SetGameBanCount(int(ban.NumberOfGameBans)).SetGameBanDate(banDate).Exec(context.Background()) - lock.Unlock() if err != nil { return nil, err }