CSGOWTF/csgowtfd
3
0
Fork 0
Code Issues Pull Requests Releases 1 Wiki Activity

switched to transactional queries for demo loading & parsing

Browse Source
This commit is contained in:
Giovanni Harting
2022-11-03 03:18:24 +01:00
parent ff3bbe0037
commit abd8b74c08
3 changed files with 109 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

79
csgo/demo_loader.go
View File

@@ -11,11 +11,11 @@ import (
"github.com/an0nfunc/go-steam/v3/protocol/gamecoordinator"
"github.com/an0nfunc/go-steam/v3/protocol/steamlang"
"github.com/go-redis/cache/v8"
"github.com/pkg/errors"
"github.com/sethvargo/go-retry"
log "github.com/sirupsen/logrus"
"golang.org/x/time/rate"
"google.golang.org/protobuf/proto"
"io/ioutil"
"os"
"strings"
"sync"
@@ -44,22 +44,23 @@ type DemoMatchLoaderConfig struct {
}
type DemoMatchLoader struct {
client *steam.Client
GCReady bool
steamLogin *steam.LogOnDetails
matchRecv chan *protobuf.CMsgGCCStrike15V2_MatchList
cmList []*netutil.PortAddr
sentryFile string
loginKey string
db *ent.Client
dp *DemoParser
parseDemo chan *Demo
parseMap map[string]bool
parseMapL *sync.RWMutex
cache *cache.Cache
connectionWait retry.Backoff
connectFeedback chan int
LoggedIn bool
client *steam.Client
GCReady bool
steamLogin *steam.LogOnDetails
matchRecv chan *protobuf.CMsgGCCStrike15V2_MatchList
cmList []*netutil.PortAddr
sentryFile string
loginKey string
db *ent.Client
dp *DemoParser
parseDemo chan *Demo
parseMap map[string]bool
parseMapL *sync.RWMutex
cache *cache.Cache
connectionWait retry.Backoff
connectionWaitTmpl retry.Backoff
connectFeedback chan int
LoggedIn bool
}
func AccountId2SteamId(accId uint32) uint64 {
@@ -115,7 +116,7 @@ func (dml *DemoMatchLoader) HandleGCPacket(pkg *gamecoordinator.GCPacket) {
msg := &protobuf.CMsgConnectionStatus{}
err := proto.Unmarshal(pkg.Body, msg)
if err != nil {
log.Errorf("[DL] Unable to unmarshal event %v: %v", pkg.MsgType, err)
log.Errorf("[GC] Unable to unmarshal event %v: %v", pkg.MsgType, err)
}
log.Debugf("[GC] Status: %+v", msg)
@@ -127,7 +128,7 @@ func (dml *DemoMatchLoader) HandleGCPacket(pkg *gamecoordinator.GCPacket) {
msg := &protobuf.GlobalStatistics{}
err := proto.Unmarshal(pkg.Body, msg)
if err != nil {
log.Errorf("[DL] Unable to unmarshal event %v: %v", pkg.MsgType, err)
log.Errorf("[GC] Unable to unmarshal event %v: %v", pkg.MsgType, err)
}
log.Debugf("[GC] Stats: %+v", msg)
dml.GCReady = true
@@ -135,11 +136,11 @@ func (dml *DemoMatchLoader) HandleGCPacket(pkg *gamecoordinator.GCPacket) {
msg := &protobuf.CMsgGCCStrike15V2_MatchList{}
err := proto.Unmarshal(pkg.Body, msg)
if err != nil {
log.Errorf("[DL] Unable to unmarshal event %v: %v", pkg.MsgType, err)
log.Errorf("[GC] Unable to unmarshal event %v: %v", pkg.MsgType, err)
}
dml.matchRecv <- msg
default:
log.Debugf("[GC] Unhandled GC message: %+v", pkg)
log.Debugf("[GC] Unhandled message: %+v", pkg)
}
}
@@ -190,8 +191,7 @@ func (dml *DemoMatchLoader) Setup(config *DemoMatchLoaderConfig) error {
dml.parseMapL = new(sync.RWMutex)
dml.cache = config.Cache
dml.connectFeedback = make(chan int, 10)
dml.connectionWait = retry.NewExponential(time.Second)
dml.connectionWait = retry.WithCappedDuration(time.Minute*time.Duration(config.RetryTimeout), dml.connectionWait)
dml.connectionWait = retry.WithCappedDuration(time.Minute*time.Duration(config.RetryTimeout), retry.NewExponential(time.Minute))
err := dml.dp.Setup(config.Db, config.Worker, config.SprayTimeout)
if err != nil {
return err
@@ -204,7 +204,7 @@ func (dml *DemoMatchLoader) Setup(config *DemoMatchLoaderConfig) error {
dml.steamLogin.ShouldRememberPassword = true
if _, err := os.Stat(dml.sentryFile); err == nil {
hash, err := ioutil.ReadFile(dml.sentryFile)
hash, err := os.ReadFile(dml.sentryFile)
if err != nil {
return err
}
@@ -212,7 +212,7 @@ func (dml *DemoMatchLoader) Setup(config *DemoMatchLoaderConfig) error {
}
if _, err := os.Stat(dml.loginKey); err == nil {
hash, err := ioutil.ReadFile(dml.loginKey)
hash, err := os.ReadFile(dml.loginKey)
if err != nil {
return err
}
@@ -264,6 +264,8 @@ func (dml *DemoMatchLoader) connectLoop() {
case LoginFailed:
if sleep, ok := dml.connectionWait.Next(); ok {
time.Sleep(sleep)
} else {
panic("retry should never be stop")
}
if !dml.LoggedIn {
log.Infof("[DL] Connecting to steam...")
@@ -274,8 +276,8 @@ func (dml *DemoMatchLoader) connectLoop() {
}
}
case LoginSuccess:
log.Infof("[DL] Steam login successfully restored after %d minutes", dml.connectionWait)
dml.connectionWait = retry.NewExponential(time.Minute)
log.Info("[DL] Steam login successfully restored")
dml.connectionWait = dml.connectionWaitTmpl
}
}
}
@@ -289,7 +291,7 @@ func (dml *DemoMatchLoader) steamEventHandler() {
dml.client.Auth.LogOn(dml.steamLogin)
case *steam.MachineAuthUpdateEvent:
log.Debug("[DL] Got sentry!")
err := ioutil.WriteFile(dml.sentryFile, e.Hash, os.ModePerm)
err := os.WriteFile(dml.sentryFile, e.Hash, os.ModePerm)
if err != nil {
log.Errorf("[DL] Unable write sentry file: %v", err)
}
@@ -320,7 +322,7 @@ func (dml *DemoMatchLoader) steamEventHandler() {
dml.connectFeedback <- LoginFailed
case *steam.LoginKeyEvent:
log.Debug("Got login_key!")
err := ioutil.WriteFile(dml.loginKey, []byte(e.LoginKey), os.ModePerm)
err := os.WriteFile(dml.loginKey, []byte(e.LoginKey), os.ModePerm)
if err != nil {
log.Errorf("[DL] Unable write login_key: %v", err)
}
@@ -414,13 +416,20 @@ func (dml *DemoMatchLoader) handleDemo(demo *Demo, apiKey string, rl *rate.Limit
log.Infof("[DL] Recieved matchdetails for match %d (%s)", matchId, time.Since(t))
// init tx
tx, err := dml.db.Tx(context.Background())
if err != nil {
return errors.Wrap(err, "error creating transaction")
}
matchZero := matchDetails.GetMatches()[0]
lastRound := matchZero.GetRoundstatsall()[len(matchZero.Roundstatsall)-1]
var players []*ent.Player
for _, accountId := range lastRound.GetReservation().GetAccountIds() {
tPlayer, err := utils.Player(dml.db, AccountId2SteamId(accountId), apiKey, rl)
tPlayer, err := utils.Player(tx.Client(), AccountId2SteamId(accountId), apiKey, rl)
if err != nil {
err = utils.Rollback(tx, err)
return fmt.Errorf("error getting player for steamid %d: %w", AccountId2SteamId(accountId), err)
}
players = append(players, tPlayer)
@@ -430,7 +439,7 @@ func (dml *DemoMatchLoader) handleDemo(demo *Demo, apiKey string, rl *rate.Limit
demo.MatchId = matchZero.GetMatchid()
demo.DecryptionKey = []byte(strings.ToUpper(fmt.Sprintf("%016x", matchZero.GetWatchablematchinfo().GetClDecryptdataKeyPub())))
tMatch, err := dml.db.Match.Create().
tMatch, err := tx.Match.Create().
SetID(matchZero.GetMatchid()).
AddPlayers(players...).
SetDate(time.Unix(int64(matchZero.GetMatchtime()), 0).UTC()).
@@ -444,6 +453,7 @@ func (dml *DemoMatchLoader) handleDemo(demo *Demo, apiKey string, rl *rate.Limit
SetDecryptionKey(demo.DecryptionKey).
Save(context.Background())
if err != nil {
err = utils.Rollback(tx, err)
return fmt.Errorf("error creating match %d: %w", matchZero.GetMatchid(), err)
}
@@ -463,9 +473,7 @@ func (dml *DemoMatchLoader) handleDemo(demo *Demo, apiKey string, rl *rate.Limit
for _, round := range matchZero.GetRoundstatsall() {
kills, _, _, _, _, _ := playerStatsFromRound(round, mPlayer)
killDiff := kills - oldKills
switch killDiff {
switch kills - oldKills {
case 2:
mk2++
case 3:
@@ -479,7 +487,7 @@ func (dml *DemoMatchLoader) handleDemo(demo *Demo, apiKey string, rl *rate.Limit
}
kills, deaths, assists, hs, score, mvp := playerStatsFromRound(lastRound, mPlayer)
err := dml.db.MatchPlayer.Create().
err = tx.MatchPlayer.Create().
SetMatches(tMatch).
SetPlayers(mPlayer).
SetTeamID(teamId).
@@ -495,6 +503,7 @@ func (dml *DemoMatchLoader) handleDemo(demo *Demo, apiKey string, rl *rate.Limit
SetMk5(mk5).
Exec(context.Background())
if err != nil {
err = utils.Rollback(tx, err)
return fmt.Errorf("error creating stats for player %d in match %d: %w", mPlayer.ID, tMatch.ID, err)
}
}