removed locks, switching to WAL for SQLite default

This commit is contained in:
2021-10-25 02:12:12 +02:00
parent c30da099ed
commit efd0f9a6af
6 changed files with 46 additions and 143 deletions

View File

@@ -3,7 +3,7 @@ logging:
db:
driver: sqlite3
connect_to: "file:csgowtfd.db?_fk=1&cache=shared"
connect_to: "file:csgowtfd.db?_journal_mode=WAL&_fk=1&cache=shared&_sync=NORMAL"
parser:
worker: 6

View File

@@ -18,7 +18,6 @@ import (
"io/ioutil"
"math/rand"
"os"
"sync"
"time"
)
@@ -34,7 +33,6 @@ type DemoMatchLoaderConfig struct {
LoginKey string
ServerList string
Db *ent.Client
Lock *sync.RWMutex
Worker int
ApiKey string
RateLimit ratelimit.Limiter
@@ -51,7 +49,6 @@ type DemoMatchLoader struct {
loginKey string
serverList string
db *ent.Client
lock *sync.RWMutex
dp *DemoParser
parseDemo chan *Demo
parseMap map[string]bool
@@ -171,12 +168,11 @@ func (d *DemoMatchLoader) Setup(config *DemoMatchLoaderConfig) error {
d.loginKey = config.LoginKey
d.sentryFile = config.Sentry
d.serverList = config.ServerList
d.lock = config.Lock
d.db = config.Db
d.dp = &DemoParser{}
d.parseMap = map[string]bool{}
d.cache = config.Cache
err := d.dp.Setup(config.Db, config.Lock, config.Worker)
err := d.dp.Setup(config.Db, config.Worker)
if err != nil {
return err
}
@@ -357,9 +353,7 @@ func (d *DemoMatchLoader) gcWorker(apiKey string, rl ratelimit.Limiter) {
continue
}
d.lock.RLock()
iMatch, err := d.db.Match.Get(context.Background(), matchId)
d.lock.RUnlock()
if err != nil {
switch e := err.(type) {
case *ent.NotFoundError:
@@ -399,10 +393,7 @@ func (d *DemoMatchLoader) gcWorker(apiKey string, rl ratelimit.Limiter) {
var players []*ent.Player
for _, accountId := range lastRound.GetReservation().GetAccountIds() {
tPlayer, err := utils.GetPlayer(&utils.DBWithLock{
Client: d.db,
Lock: d.lock,
}, AccountId2SteamId(accountId), apiKey, rl)
tPlayer, err := utils.GetPlayer(d.db, AccountId2SteamId(accountId), apiKey, rl)
if err != nil {
log.Warningf("[DL] Unable to get player for steamid %d: %v", AccountId2SteamId(accountId), err)
continue
@@ -413,7 +404,6 @@ func (d *DemoMatchLoader) gcWorker(apiKey string, rl ratelimit.Limiter) {
demo.Url = lastRound.GetMap()
demo.MatchId = matchZero.GetMatchid()
d.lock.Lock()
tMatch, err := d.db.Match.Create().
SetID(matchZero.GetMatchid()).
AddPlayers(players...).
@@ -426,7 +416,6 @@ func (d *DemoMatchLoader) gcWorker(apiKey string, rl ratelimit.Limiter) {
SetScoreTeamB(int(lastRound.GetTeamScores()[1])).
SetMatchResult(int(lastRound.GetMatchResult())).
Save(context.Background())
d.lock.Unlock()
if err != nil {
log.Warningf("[DL] Unable to create match %d: %v", matchZero.GetMatchid(), err)
delete(d.parseMap, demo.ShareCode)
@@ -468,7 +457,6 @@ func (d *DemoMatchLoader) gcWorker(apiKey string, rl ratelimit.Limiter) {
}
kills, deaths, assists, hs, score, mvp := playerStatsFromRound(lastRound, mPlayer)
d.lock.Lock()
err := d.db.Stats.Create().
SetMatches(tMatch).
SetPlayers(mPlayer).
@@ -484,7 +472,6 @@ func (d *DemoMatchLoader) gcWorker(apiKey string, rl ratelimit.Limiter) {
SetMk4(mk4).
SetMk5(mk5).
Exec(context.Background())
d.lock.Unlock()
if err != nil {
log.Warningf("[DL] Unable to create stats for player %d in match %d: %v", mPlayer.ID, tMatch.ID, err)
}

View File

@@ -14,7 +14,6 @@ import (
log "github.com/sirupsen/logrus"
"io"
"net/http"
"sync"
"time"
)
@@ -28,17 +27,15 @@ type DemoParser struct {
demoQueue chan *Demo
tempDir string
db *ent.Client
lock *sync.RWMutex
}
type DemoNotFoundError struct {
error
}
func (p *DemoParser) Setup(db *ent.Client, lock *sync.RWMutex, worker int) error {
func (p *DemoParser) Setup(db *ent.Client, worker int) error {
p.demoQueue = make(chan *Demo, 1000)
p.db = db
p.lock = lock
for i := 0; i < worker; i++ {
go p.parseWorker()
}
@@ -70,13 +67,11 @@ func (p *DemoParser) downloadReplay(demo *Demo) (io.Reader, error) {
}
func (p *DemoParser) getDBPlayer(demo *Demo, demoPlayer *common.Player) (*ent.Stats, error) {
p.lock.RLock()
tMatchPlayer, err := p.db.Stats.Query().WithMatches(func(q *ent.MatchQuery) {
q.Where(match.ID(demo.MatchId))
}).WithPlayers(func(q *ent.PlayerQuery) {
q.Where(player.ID(demoPlayer.SteamID64))
}).Only(context.Background())
p.lock.RUnlock()
if err != nil {
return nil, err
}
@@ -108,9 +103,7 @@ func (p *DemoParser) parseWorker() {
continue
}
p.lock.RLock()
tMatch, err := p.db.Match.Get(context.Background(), demo.MatchId)
p.lock.RUnlock()
if err != nil {
log.Errorf("[DP] Unable to get match %d: %v", demo.MatchId, err)
continue
@@ -139,9 +132,7 @@ func (p *DemoParser) parseWorker() {
}
downloadTime := time.Now().Sub(startTime)
p.lock.RLock()
tStats, err := tMatch.QueryStats().WithPlayers().All(context.Background())
p.lock.RUnlock()
if err != nil {
log.Errorf("[DP] Failed to find players for match %d: %v", demo.MatchId, err)
continue
@@ -294,9 +285,7 @@ func (p *DemoParser) parseWorker() {
continue
}
p.lock.Lock()
err = tMatch.Update().SetMap(demoParser.Header().MapName).SetDemoParsed(true).Exec(context.Background())
p.lock.Unlock()
if err != nil {
log.Errorf("[DP] Unable to update match %d in database: %v", demo.MatchId, err)
continue
@@ -307,7 +296,6 @@ func (p *DemoParser) parseWorker() {
tMatchPlayer.Color = stats.ColorGrey
}
p.lock.Lock()
nMatchPLayer, err := tMatchPlayer.Update().
SetDmgTeam(tMatchPlayer.DmgTeam).
SetDmgEnemy(tMatchPlayer.DmgEnemy).
@@ -328,25 +316,20 @@ func (p *DemoParser) parseWorker() {
SetDmgTeam(tMatchPlayer.DmgTeam).
SetDmgEnemy(tMatchPlayer.DmgEnemy).
Save(context.Background())
p.lock.Unlock()
if err != nil {
log.Errorf("[DP] Unable to update stats %d in database: %v", tMatchPlayer.PlayerStats, err)
continue
}
for _, eqDmg := range eqMap[tMatchPlayer.PlayerStats] {
p.lock.Lock()
err := p.db.WeaponStats.Create().SetStat(nMatchPLayer).SetDmg(eqDmg.Dmg).SetVictim(eqDmg.To).SetHitGroup(eqDmg.HitGroup).SetEqType(eqDmg.Eq).Exec(context.Background())
p.lock.Unlock()
if err != nil {
log.Errorf("[DP] Unable to create WeaponStat: %v", err)
}
}
for _, eco := range ecoMap[tMatchPlayer.PlayerStats] {
p.lock.Lock()
err := p.db.RoundStats.Create().SetStat(nMatchPLayer).SetRound(uint(eco.Round)).SetBank(uint(eco.Bank)).SetEquipment(uint(eco.EqV)).SetSpent(uint(eco.Spent)).Exec(context.Background())
p.lock.Unlock()
if err != nil {
log.Errorf("[DP] Unable to create RoundStat: %v", err)
}

7
go.sum
View File

@@ -120,6 +120,7 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
github.com/go-openapi/inflect v0.19.0 h1:9jCH9scKIbHeV9m12SmPilScz6krDxKRasNNSNPXu/4=
github.com/go-openapi/inflect v0.19.0/go.mod h1:lHpZVlpIQqLyKwJ4N+YSc9hchQy/i12fJykb83CRBH4=
github.com/go-redis/cache/v8 v8.4.3 h1:+RZ0pQM+zOd6h/oWCsOl3+nsCgii9rn26oCYmU87kN8=
github.com/go-redis/cache/v8 v8.4.3/go.mod h1:5lQPQ63uyBt4aZuRmdvUJOJRRjPxfLtJtlcJ/z8o1jA=
@@ -331,6 +332,7 @@ github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd
github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0=
github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
github.com/mattn/go-sqlite3 v1.14.8/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
github.com/mattn/go-sqlite3 v1.14.9 h1:10HX2Td0ocZpYEjhilsuo6WWtUqttj2Kb0KtD86/KYA=
@@ -365,6 +367,7 @@ github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtb
github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA=
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
@@ -451,10 +454,12 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasO
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
github.com/spf13/cobra v1.1.3 h1:xghbfqPkxzxP3C/f3n5DdpAbdKLj4ZE4BWQI362l53M=
github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo=
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
@@ -565,6 +570,7 @@ golang.org/x/mod v0.1.1-0.20191209134235-331c550502dd/go.mod h1:s0Qsj1ACt9ePp/hM
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.5.1-0.20210830214625-1b1db11ec8f4 h1:7Qds88gNaRx0Dz/1wOwXlR7asekh1B1u26wEwN6FcEI=
golang.org/x/mod v0.5.1-0.20210830214625-1b1db11ec8f4/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -691,6 +697,7 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY
golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
golang.org/x/tools v0.1.5 h1:ouewzE6p+/VEB31YYnTbEJdi8pFqKp4P4n85vwo3DHA=
golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

78
main.go
View File

@@ -29,7 +29,6 @@ import (
"os"
"os/signal"
"strconv"
"sync"
"syscall"
"time"
)
@@ -38,7 +37,7 @@ var (
conf = utils.Conf{}
demoLoader = &csgo.DemoMatchLoader{}
router *mux.Router
db *utils.DBWithLock
db *ent.Client
rdb *redis.Client
rdc *cache.Cache
firstHK = true
@@ -57,11 +56,9 @@ func housekeeping() {
firstHK = false
// update players from steam
db.Lock.RLock()
tPlayerNeedSteamUpdate, err := db.Client.Player.Query().Where(
tPlayerNeedSteamUpdate, err := db.Player.Query().Where(
player.SteamUpdatedLTE(time.Now().UTC().AddDate(0, 0, -1)),
).All(context.Background())
db.Lock.RUnlock()
if err != nil {
log.Errorf("[HK] Can't query players: %v", err)
continue
@@ -69,7 +66,7 @@ func housekeeping() {
if len(tPlayerNeedSteamUpdate) > 0 {
log.Infof("[HK] Refreshing %d profiles from steam", len(tPlayerNeedSteamUpdate))
_, err = utils.UpdatePlayerFromSteam(tPlayerNeedSteamUpdate, db.Client, conf.Steam.APIKey, db.Lock, rL)
_, err = utils.UpdatePlayerFromSteam(tPlayerNeedSteamUpdate, db, conf.Steam.APIKey, rL)
if err != nil {
log.Warningf("[HK] Unable to update profiles from steam: %v", err)
}
@@ -81,8 +78,7 @@ func housekeeping() {
continue
}
db.Lock.RLock()
tPlayerNeedShareCodeUpdate, err := db.Client.Player.Query().Where(
tPlayerNeedShareCodeUpdate, err := db.Player.Query().Where(
player.And(
player.Or(
player.SharecodeUpdatedLTE(time.Now().UTC().Add(time.Duration(-30)*time.Minute)),
@@ -90,21 +86,18 @@ func housekeeping() {
),
player.Not(player.AuthCodeIsNil()),
)).All(context.Background())
db.Lock.RUnlock()
if err != nil {
log.Errorf("[HK] Can't query players: %v", err)
continue
}
for _, tPlayer := range tPlayerNeedShareCodeUpdate {
shareCodes, err := utils.GetNewShareCodesForPlayer(tPlayer, db.Lock, conf.Steam.APIKey, rL)
shareCodes, err := utils.GetNewShareCodesForPlayer(tPlayer, conf.Steam.APIKey, rL)
if err != nil {
switch err.(type) {
case utils.AuthcodeUnauthorizedError:
log.Infof("[HK] authCode for player %d is no longer valid", tPlayer.ID)
db.Lock.Lock()
err = tPlayer.Update().ClearAuthCode().ClearSharecodeUpdated().Exec(context.Background())
db.Lock.Unlock()
if err != nil {
log.Errorf("[HK] Unable to clear authcode for player %d: %v", tPlayer.ID, err)
}
@@ -129,9 +122,7 @@ func housekeeping() {
}
// try parsing demos not parsed
db.Lock.RLock()
tMatches, err := db.Client.Match.Query().Where(match.And(match.DateGT(time.Now().UTC().AddDate(0, 0, -30)), match.DemoParsed(false))).All(context.Background())
db.Lock.RUnlock()
tMatches, err := db.Match.Query().Where(match.And(match.DateGT(time.Now().UTC().AddDate(0, 0, -30)), match.DemoParsed(false))).All(context.Background())
if err != nil {
log.Warningf("[HK] Failure getting matches to retry parsing: %v", err)
continue
@@ -180,7 +171,7 @@ func getPlayerMeta(w http.ResponseWriter, r *http.Request) {
metaStats := new(utils.MetaStatsResponse)
err = rdc.Get(context.Background(), fmt.Sprintf(utils.SideMetaCacheKey, tPlayer.ID), &metaStats)
if err != nil {
metaStats, err = utils.GetMetaStats(tPlayer, db.Lock)
metaStats, err = utils.GetMetaStats(tPlayer)
if err != nil {
log.Infof("[GPM] Unable to get MetaStats: %v", err)
w.WriteHeader(http.StatusInternalServerError)
@@ -221,7 +212,7 @@ func getPlayerMeta(w http.ResponseWriter, r *http.Request) {
for _, p := range append(metaStats.BestMates, metaStats.MostMates...) {
if p.Player.Name == "" {
tP, err := db.Client.Player.Get(context.Background(), p.Player.SteamID64)
tP, err := db.Player.Get(context.Background(), p.Player.SteamID64)
if err != nil {
log.Warningf("[GPM] Failure getting player: %v", err)
w.WriteHeader(http.StatusInternalServerError)
@@ -289,13 +280,9 @@ func getPlayer(w http.ResponseWriter, r *http.Request) {
var tMatches []*ent.Match
if !offsetTime.IsZero() {
db.Lock.RLock()
tMatches, err = tPlayer.QueryMatches().Where(match.DateLT(offsetTime)).Order(ent.Desc(match.FieldDate)).Limit(10).All(context.Background())
db.Lock.RUnlock()
} else {
db.Lock.RLock()
tMatches, err = tPlayer.QueryMatches().Order(ent.Desc(match.FieldDate)).Limit(10).All(context.Background())
db.Lock.RUnlock()
}
if err != nil || len(tMatches) == 0 {
log.Debugf("[GP] No matches found for player %s", id)
@@ -311,7 +298,7 @@ func getPlayer(w http.ResponseWriter, r *http.Request) {
metaStats := new(utils.MatchStats)
err = rdc.Get(context.Background(), fmt.Sprintf(utils.MatchMetaCacheKey, tPlayer.ID), &metaStats)
if err != nil {
wins, ties, losses, err := utils.GetMatchStats(tPlayer, db.Lock)
wins, ties, losses, err := utils.GetMatchStats(tPlayer)
if err != nil {
log.Errorf("[GP] Error retrieving match-stats for player %s: %v", id, err)
w.WriteHeader(http.StatusInternalServerError)
@@ -358,14 +345,12 @@ func getPlayer(w http.ResponseWriter, r *http.Request) {
Parsed: iMatch.DemoParsed,
}
db.Lock.RLock()
tStats, err := iMatch.QueryStats().Modify(func(s *sql.Selector) {
s.Select(stats.FieldTeamID, stats.FieldKills, stats.FieldDeaths, stats.FieldAssists, stats.FieldHeadshot,
stats.FieldMvp, stats.FieldScore, stats.FieldMk2, stats.FieldMk3, stats.FieldMk4, stats.FieldMk5,
stats.FieldRankOld, stats.FieldRankNew, stats.FieldDmgTeam, stats.FieldDmgEnemy)
s.Where(sql.EQ(s.C(stats.PlayersColumn), tPlayer.ID))
}).Only(context.Background())
db.Lock.RUnlock()
if err != nil {
response.Matches = append(response.Matches, mResponse)
continue
@@ -433,7 +418,7 @@ func deletePlayerTrack(w http.ResponseWriter, r *http.Request) {
return
}
_, err = utils.IsAuthCodeValid(tPlayer, db.Lock, conf.Steam.APIKey, "", authCode, nil)
_, err = utils.IsAuthCodeValid(tPlayer, conf.Steam.APIKey, "", authCode, nil)
if err != nil {
switch e := err.(type) {
case utils.AuthcodeUnauthorizedError:
@@ -447,9 +432,7 @@ func deletePlayerTrack(w http.ResponseWriter, r *http.Request) {
}
}
db.Lock.Lock()
err = tPlayer.Update().ClearAuthCode().ClearSharecodeUpdated().Exec(context.Background())
db.Lock.Unlock()
if err != nil {
log.Warningf("[PPT] update player failed: %+v", err)
w.WriteHeader(http.StatusInternalServerError)
@@ -484,7 +467,7 @@ func postPlayerTrack(w http.ResponseWriter, r *http.Request) {
return
}
_, err = utils.IsAuthCodeValid(tPlayer, db.Lock, conf.Steam.APIKey, shareCode, authCode, rL)
_, err = utils.IsAuthCodeValid(tPlayer, conf.Steam.APIKey, shareCode, authCode, rL)
if err != nil {
switch e := err.(type) {
case utils.AuthcodeUnauthorizedError:
@@ -502,9 +485,7 @@ func postPlayerTrack(w http.ResponseWriter, r *http.Request) {
}
}
db.Lock.Lock()
err = tPlayer.Update().SetAuthCode(authCode).Exec(context.Background())
db.Lock.Unlock()
if err != nil {
log.Warningf("[PPT] update player failed: %+v", err)
w.WriteHeader(http.StatusInternalServerError)
@@ -559,9 +540,7 @@ func getMatchRounds(w http.ResponseWriter, r *http.Request) {
return
}
db.Lock.RLock()
tStats, err := db.Client.Stats.Query().Where(stats.HasMatchesWith(match.ID(matchId))).All(context.Background())
db.Lock.RUnlock()
tStats, err := db.Stats.Query().Where(stats.HasMatchesWith(match.ID(matchId))).All(context.Background())
if err != nil {
log.Infof("[GMR] match %d not found: %+v", matchId, err)
w.WriteHeader(http.StatusNotFound)
@@ -571,9 +550,7 @@ func getMatchRounds(w http.ResponseWriter, r *http.Request) {
resp := map[uint]map[string][]uint{}
for _, stat := range tStats {
db.Lock.RLock()
tRoundStats, err := stat.QueryRoundStats().All(context.Background())
db.Lock.RUnlock()
if err != nil {
log.Warningf("[GMR] Unable to get RoundStats for player %d: %v", stat.PlayerStats, err)
continue
@@ -610,9 +587,7 @@ func getMatchWeapons(w http.ResponseWriter, r *http.Request) {
return
}
db.Lock.RLock()
tStats, err := db.Client.Stats.Query().Where(stats.HasMatchesWith(match.ID(matchId))).All(context.Background())
db.Lock.RUnlock()
tStats, err := db.Stats.Query().Where(stats.HasMatchesWith(match.ID(matchId))).All(context.Background())
if err != nil {
log.Infof("[GMW] match %d not found: %+v", matchId, err)
w.WriteHeader(http.StatusNotFound)
@@ -628,9 +603,7 @@ func getMatchWeapons(w http.ResponseWriter, r *http.Request) {
}
for _, stat := range tStats {
db.Lock.RLock()
mWs, err := stat.QueryWeaponStats().All(context.Background())
db.Lock.RUnlock()
if err != nil {
log.Warningf("[GMW] Unable to get WeaponStats for player %d: %v", stat.PlayerStats, err)
continue
@@ -676,9 +649,7 @@ func getMatch(w http.ResponseWriter, r *http.Request) {
return
}
db.Lock.RLock()
tMatch, err := db.Client.Match.Query().Where(match.ID(matchId)).Only(context.Background())
db.Lock.RUnlock()
tMatch, err := db.Match.Query().Where(match.ID(matchId)).Only(context.Background())
if err != nil {
log.Infof("[GM] match %d not found: %v", matchId, err)
w.WriteHeader(http.StatusNotFound)
@@ -702,9 +673,7 @@ func getMatch(w http.ResponseWriter, r *http.Request) {
mResponse.ReplayURL = tMatch.ReplayURL
}
db.Lock.RLock()
tStats, err := tMatch.QueryStats().WithPlayers().All(context.Background())
db.Lock.RUnlock()
if err != nil {
log.Errorf("[GM] Unable to find stats for match %d: %v", tMatch.ID, err)
w.WriteHeader(http.StatusInternalServerError)
@@ -810,10 +779,6 @@ func main() {
journalhook.Enable()
}
db = &utils.DBWithLock{
Lock: new(sync.RWMutex),
}
if conf.Db.Driver == "pgx" {
pdb, err := sql.Open("pgx", conf.Db.ConnectTo)
if err != nil {
@@ -821,27 +786,27 @@ func main() {
}
drv := sql.OpenDB(dialect.Postgres, pdb.DB())
db.Client = ent.NewClient(ent.Driver(drv))
db = ent.NewClient(ent.Driver(drv))
} else {
db.Client, err = ent.Open(conf.Db.Driver, conf.Db.ConnectTo)
db, err = ent.Open(conf.Db.Driver, conf.Db.ConnectTo)
if err != nil {
log.Panicf("Failed to open database %s: %v", conf.Db.ConnectTo, err)
}
defer func(Client *ent.Client) {
_ = Client.Close()
}(db.Client)
}(db)
}
if *sqlDebugFlag {
db.Client = db.Client.Debug()
db = db.Debug()
}
if err := db.Client.Schema.Create(
if err := db.Schema.Create(
context.Background(),
migrate.WithDropIndex(true),
migrate.WithDropColumn(true),
); err != nil {
log.Panicf("Automigrate failed: %v", err)
log.Fatalf("Automigrate failed: %v", err)
}
rdb = redis.NewClient(&redis.Options{
@@ -865,8 +830,7 @@ func main() {
Sentry: conf.Steam.Sentry,
LoginKey: conf.Steam.LoginKey,
ServerList: conf.Steam.ServerList,
Db: db.Client,
Lock: db.Lock,
Db: db,
Worker: conf.Parser.Worker,
ApiKey: conf.Steam.APIKey,
RateLimit: rL,

View File

@@ -22,7 +22,6 @@ import (
"sort"
"strconv"
"strings"
"sync"
"time"
)
@@ -60,11 +59,6 @@ type Conf struct {
}
}
type DBWithLock struct {
Client *ent.Client
Lock *sync.RWMutex
}
type CommunityXML struct {
SteamID64 uint64 `xml:"steamID64"`
AvatarURL string `xml:"avatarFull"`
@@ -221,8 +215,8 @@ func SendJSON(data interface{}, w http.ResponseWriter) error {
return nil
}
func GetMatchStats(dbPlayer *ent.Player, lock *sync.RWMutex) (int, int, int, error) {
wins, loss, ties, err := getWinLossTieFromPlayer(dbPlayer, lock)
func GetMatchStats(dbPlayer *ent.Player) (int, int, int, error) {
wins, loss, ties, err := getWinLossTieFromPlayer(dbPlayer)
if err != nil {
return 0, 0, 0, err
}
@@ -230,20 +224,16 @@ func GetMatchStats(dbPlayer *ent.Player, lock *sync.RWMutex) (int, int, int, err
return wins, ties, loss, nil
}
func GetMetaStats(dbPlayer *ent.Player, lock *sync.RWMutex) (*MetaStatsResponse, error) {
func GetMetaStats(dbPlayer *ent.Player) (*MetaStatsResponse, error) {
mResponse := new(MetaStatsResponse)
mResponse.Player = &PlayerResponse{SteamID64: dbPlayer.ID}
lock.RLock()
tPlayers, err := dbPlayer.QueryMatches().QueryPlayers().Select(player.FieldID).All(context.Background())
lock.RUnlock()
if err != nil {
return nil, err
}
lock.RLock()
matchIDs, err := dbPlayer.QueryMatches().IDs(context.Background())
lock.RUnlock()
if err != nil {
return nil, err
}
@@ -266,14 +256,12 @@ func GetMetaStats(dbPlayer *ent.Player, lock *sync.RWMutex) (*MetaStatsResponse,
SteamID64: s.ID,
}
lock.RLock()
pMatches, err := s.QueryMatches().
Select(match.FieldID, match.FieldMatchResult, match.FieldMap).
Where(match.IDIn(matchIDs...)).
WithStats().
Where(match.HasStatsWith(stats.Or(stats.PlayerStats(dbPlayer.ID), stats.PlayerStats(s.ID)))).
All(context.Background())
lock.RUnlock()
if err != nil {
return nil, err
}
@@ -307,10 +295,8 @@ func GetMetaStats(dbPlayer *ent.Player, lock *sync.RWMutex) (*MetaStatsResponse,
}
}
lock.RLock()
wSs, err := subjectStats.QueryWeaponStats().
Select(weaponstats.FieldEqType, weaponstats.FieldDmg).All(context.Background())
lock.RUnlock()
if err != nil {
return nil, err
}
@@ -396,13 +382,12 @@ func GetMetaStats(dbPlayer *ent.Player, lock *sync.RWMutex) (*MetaStatsResponse,
return mResponse, nil
}
func getWinLossTieFromPlayer(dbPlayer *ent.Player, lock *sync.RWMutex) (int, int, int, error) {
func getWinLossTieFromPlayer(dbPlayer *ent.Player) (int, int, int, error) {
var res []struct {
MatchResult int `json:"match_result"`
Count int `json:"count"`
}
lock.RLock()
err := dbPlayer.QueryMatches().GroupBy(match.FieldMatchResult).Aggregate(func(s *sql.Selector) string {
sT := sql.Table(stats.Table)
@@ -416,16 +401,13 @@ func getWinLossTieFromPlayer(dbPlayer *ent.Player, lock *sync.RWMutex) (int, int
))
return sql.Count("*")
}).Scan(context.Background(), &res)
lock.RUnlock()
if err != nil {
return 0, 0, 0, err
}
lock.RLock()
total, err := dbPlayer.QueryMatches().Modify(func(s *sql.Selector) {
s.Select("COUNT(*)")
}).Int(context.Background())
lock.RUnlock()
if err != nil {
return 0, 0, 0, err
}
@@ -447,13 +429,11 @@ func getWinLossTieFromPlayer(dbPlayer *ent.Player, lock *sync.RWMutex) (int, int
return wins, total - wins - ties, ties, nil
}
func IsAuthCodeValid(player *ent.Player, lock *sync.RWMutex, apiKey string, shareCode string, authCode string, rl ratelimit.Limiter) (bool, error) {
func IsAuthCodeValid(player *ent.Player, apiKey string, shareCode string, authCode string, rl ratelimit.Limiter) (bool, error) {
var tMatch *ent.Match
var err error
if shareCode == "" {
lock.RLock()
tMatch, err = player.QueryMatches().Order(ent.Asc(match.FieldDate)).First(context.Background())
lock.RUnlock()
if err != nil {
return false, err
}
@@ -472,17 +452,13 @@ func IsAuthCodeValid(player *ent.Player, lock *sync.RWMutex, apiKey string, shar
}
}
func GetNewShareCodesForPlayer(player *ent.Player, lock *sync.RWMutex, apiKey string, rl ratelimit.Limiter) ([]string, error) {
lock.RLock()
func GetNewShareCodesForPlayer(player *ent.Player, apiKey string, rl ratelimit.Limiter) ([]string, error) {
latestMatch, err := player.QueryMatches().Order(ent.Desc(match.FieldDate)).First(context.Background())
lock.RUnlock()
if err != nil {
return nil, err
}
lock.RLock()
oldestMatch, err := player.QueryMatches().Order(ent.Asc(match.FieldDate)).First(context.Background())
lock.RUnlock()
if err != nil {
return nil, err
}
@@ -506,9 +482,7 @@ func GetNewShareCodesForPlayer(player *ent.Player, lock *sync.RWMutex, apiKey st
}
}
lock.Lock()
err = player.Update().SetSharecodeUpdated(time.Now().UTC()).SetOldestSharecodeSeen(oldestMatch.ShareCode).Exec(context.Background())
lock.Unlock()
if err != nil {
return nil, err
}
@@ -562,7 +536,7 @@ func getNextShareCode(lastCode string, apiKey string, authCode string, steamId u
return rJson.Result.Code, nil
}
func GetPlayer(db *DBWithLock, id interface{}, apiKey string, rl ratelimit.Limiter) (*ent.Player, error) {
func GetPlayer(db *ent.Client, id interface{}, apiKey string, rl ratelimit.Limiter) (*ent.Player, error) {
switch e := id.(type) {
case uint64:
return GetPlayerFromSteamID64(db, e, apiKey, rl)
@@ -582,14 +556,12 @@ func GetPlayer(db *DBWithLock, id interface{}, apiKey string, rl ratelimit.Limit
}
}
func GetPlayerFromVanityURL(db *DBWithLock, id string, apiKey string, rl ratelimit.Limiter) (*ent.Player, error) {
func GetPlayerFromVanityURL(db *ent.Client, id string, apiKey string, rl ratelimit.Limiter) (*ent.Player, error) {
if id == "" {
return nil, fmt.Errorf("invalid arguments")
}
db.Lock.RLock()
tPlayer, err := db.Client.Player.Query().Where(player.VanityURL(strings.ToLower(id))).Only(context.Background())
db.Lock.RUnlock()
tPlayer, err := db.Player.Query().Where(player.VanityURL(strings.ToLower(id))).Only(context.Background())
if err == nil {
return tPlayer, nil
} else {
@@ -614,21 +586,17 @@ func GetPlayerFromVanityURL(db *DBWithLock, id string, apiKey string, rl ratelim
}
}
func GetPlayerFromSteamID64(db *DBWithLock, steamID uint64, apiKey string, rl ratelimit.Limiter) (*ent.Player, error) {
db.Lock.RLock()
tPlayer, err := db.Client.Player.Get(context.Background(), steamID)
db.Lock.RUnlock()
func GetPlayerFromSteamID64(db *ent.Client, steamID uint64, apiKey string, rl ratelimit.Limiter) (*ent.Player, error) {
tPlayer, err := db.Player.Get(context.Background(), steamID)
if err == nil {
return tPlayer, nil
} else {
db.Lock.Lock()
nPlayer, err := db.Client.Player.Create().SetID(steamID).Save(context.Background())
db.Lock.Unlock()
nPlayer, err := db.Player.Create().SetID(steamID).Save(context.Background())
if err != nil {
return nil, err
}
uPlayer, err := UpdatePlayerFromSteam([]*ent.Player{nPlayer}, db.Client, apiKey, db.Lock, rl)
uPlayer, err := UpdatePlayerFromSteam([]*ent.Player{nPlayer}, db, apiKey, rl)
if err != nil {
return nil, err
}
@@ -641,7 +609,7 @@ func GetPlayerFromSteamID64(db *DBWithLock, steamID uint64, apiKey string, rl ra
}
}
func UpdatePlayerFromSteam(players []*ent.Player, db *ent.Client, apiKey string, lock *sync.RWMutex, rl ratelimit.Limiter) ([]*ent.Player, error) {
func UpdatePlayerFromSteam(players []*ent.Player, db *ent.Client, apiKey string, rl ratelimit.Limiter) ([]*ent.Player, error) {
var idsToUpdate []uint64
for _, updatePlayer := range players {
@@ -667,7 +635,6 @@ func UpdatePlayerFromSteam(players []*ent.Player, db *ent.Client, apiKey string,
pS.ProfileURL = path.Base(pS.ProfileURL)
}
lock.Lock()
tPlayer, err := db.Player.UpdateOneID(pS.SteamID).
SetName(pS.PersonaName).
SetAvatar(pS.AvatarHash).
@@ -677,7 +644,6 @@ func UpdatePlayerFromSteam(players []*ent.Player, db *ent.Client, apiKey string,
SetSteamUpdated(time.Now().UTC()).
SetProfileCreated(time.Unix(pS.TimeCreated, 0).UTC()).
Save(context.Background())
lock.Unlock()
if err != nil {
return nil, err
}
@@ -696,9 +662,7 @@ func UpdatePlayerFromSteam(players []*ent.Player, db *ent.Client, apiKey string,
if ban.NumberOfVACBans > 0 {
banDate := time.Now().UTC().AddDate(0, 0, -1*int(ban.DaysSinceLastBan))
lock.Lock()
err := db.Player.UpdateOneID(ban.SteamID).SetVacCount(int(ban.NumberOfVACBans)).SetVacDate(banDate).Exec(context.Background())
lock.Unlock()
if err != nil {
return nil, err
}
@@ -706,9 +670,7 @@ func UpdatePlayerFromSteam(players []*ent.Player, db *ent.Client, apiKey string,
if ban.NumberOfGameBans > 0 {
banDate := time.Now().UTC().AddDate(0, 0, -1*int(ban.DaysSinceLastBan))
lock.Lock()
err := db.Player.UpdateOneID(ban.SteamID).SetGameBanCount(int(ban.NumberOfGameBans)).SetGameBanDate(banDate).Exec(context.Background())
lock.Unlock()
if err != nil {
return nil, err
}