CSGOWTF/csgowtfd
3
0
Fork 0
Code Issues Pull Requests Releases 1 Wiki Activity

switched to transactional queries for demo loading & parsing

Browse Source
This commit is contained in:
Giovanni Harting
2022-11-03 03:18:24 +01:00
parent ff3bbe0037
commit abd8b74c08
3 changed files with 109 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

71
csgo/demo_parser.go
View File

@@ -5,11 +5,13 @@ import (
"compress/bzip2"
"context"
"encoding/gob"
"errors"
"fmt"
"git.harting.dev/csgowtf/csgowtfd/ent"
"git.harting.dev/csgowtf/csgowtfd/ent/match"
"git.harting.dev/csgowtf/csgowtfd/ent/matchplayer"
"git.harting.dev/csgowtf/csgowtfd/ent/player"
"git.harting.dev/csgowtf/csgowtfd/utils"
"github.com/golang/geo/r2"
"github.com/markus-wa/demoinfocs-golang/v3/pkg/demoinfocs"
"github.com/markus-wa/demoinfocs-golang/v3/pkg/demoinfocs/common"
@@ -173,7 +175,8 @@ func setMatchPlayerColor(matchPlayer *ent.MatchPlayer, demoPlayer *common.Player
}
matchPlayer.Crosshair = demoPlayer.CrosshairCode()
switch demoPlayer.Color() {
color, _ := demoPlayer.ColorOrErr()
switch color {
case common.Yellow:
matchPlayer.Color = matchplayer.ColorYellow
break
@@ -189,10 +192,13 @@ func setMatchPlayerColor(matchPlayer *ent.MatchPlayer, demoPlayer *common.Player
case common.Orange:
matchPlayer.Color = matchplayer.ColorOrange
break
case common.Grey:
matchPlayer.Color = matchplayer.ColorGrey
}
}
func (dp *DemoParser) parseWorker() {
workloop:
for demo := range dp.demoQueue {
if demo.MatchId == 0 {
log.Warningf("[DP] can't parse match %s: no matchid found", demo.ShareCode)
@@ -200,14 +206,24 @@ func (dp *DemoParser) parseWorker() {
continue
}
tMatch, err := dp.db.Match.Get(context.Background(), demo.MatchId)
// init tx
tx, err := dp.db.Tx(context.Background())
if err != nil {
log.Errorf("[DP] error creating transaction: %v", err)
dp.Done <- demo
return
}
tMatch, err := tx.Match.Get(context.Background(), demo.MatchId)
if err != nil {
err = utils.Rollback(tx, err)
log.Errorf("[DP] Unable to get match %d: %v", demo.MatchId, err)
dp.Done <- demo
continue
}
if tMatch.DemoParsed {
err = utils.Rollback(tx, err)
log.Infof("[DP] skipped already parsed %d", demo.MatchId)
dp.Done <- demo
continue
@@ -216,15 +232,17 @@ func (dp *DemoParser) parseWorker() {
startTime := time.Now()
fDemo, err := demo.download()
if err != nil {
if _, ok := err.(DemoNotFoundError); ok {
if errors.Is(err, DemoNotFoundError{}) {
err = utils.Rollback(tx, err)
if tMatch.Date.Before(time.Now().UTC().AddDate(0, 0, -30)) {
log.Infof("[DP] demo expired for match %d", tMatch.ID)
} else {
log.Infof("[DP] demo 404 not found for match %d. Trying again later.", demo.MatchId)
log.Infof("[DP] demo 404 not found for match %d. Retrying later.", demo.MatchId)
}
dp.Done <- demo
continue
} else {
err = utils.Rollback(tx, err)
log.Errorf("[DP] Unable to download demo for %d: %v", demo.MatchId, err)
dp.Done <- demo
continue
@@ -234,6 +252,7 @@ func (dp *DemoParser) parseWorker() {
tStats, err := tMatch.QueryStats().WithPlayers().All(context.Background())
if err != nil {
err = utils.Rollback(tx, err)
log.Errorf("[DP] Failed to find players for match %d: %v", demo.MatchId, err)
dp.Done <- demo
continue
@@ -263,7 +282,6 @@ func (dp *DemoParser) parseWorker() {
// onChatMessage
demoParser.RegisterEventHandler(func(e events.ChatMessage) {
if e.Sender != nil {
gs := demoParser.GameState()
tAttacker, err := dp.MatchPlayerBySteamID(tStats, e.Sender.SteamID64)
if err != nil {
@@ -453,6 +471,7 @@ func (dp *DemoParser) parseWorker() {
err = demoParser.ParseToEnd()
if err != nil {
err = utils.Rollback(tx, err)
log.Errorf("[DP] Error parsing replay: %v", err)
dp.Done <- demo
continue
@@ -464,6 +483,7 @@ func (dp *DemoParser) parseWorker() {
SetTickRate(demoParser.TickRate()).
Exec(context.Background())
if err != nil {
err = utils.Rollback(tx, err)
log.Errorf("[DP] Unable to update match %d in database: %v", demo.MatchId, err)
dp.Done <- demo
continue
@@ -495,21 +515,29 @@ func (dp *DemoParser) parseWorker() {
SetDmgEnemy(tMatchPlayer.DmgEnemy).
Save(context.Background())
if err != nil {
err = utils.Rollback(tx, err)
log.Errorf("[DP] Unable to update stats %d in database: %v", tMatchPlayer.PlayerStats, err)
continue
dp.Done <- demo
continue workloop
}
for _, eqDmg := range eqMap[tMatchPlayer.PlayerStats] {
err := dp.db.Weapon.Create().SetStat(nMatchPLayer).SetDmg(eqDmg.Dmg).SetVictim(eqDmg.To).SetHitGroup(eqDmg.HitGroup).SetEqType(eqDmg.Eq).Exec(context.Background())
err = tx.Weapon.Create().SetStat(nMatchPLayer).SetDmg(eqDmg.Dmg).SetVictim(eqDmg.To).SetHitGroup(eqDmg.HitGroup).SetEqType(eqDmg.Eq).Exec(context.Background())
if err != nil {
err = utils.Rollback(tx, err)
log.Errorf("[DP] Unable to create WeaponStat: %v", err)
dp.Done <- demo
continue workloop
}
}
for _, eco := range ecoMap[tMatchPlayer.PlayerStats] {
err := dp.db.RoundStats.Create().SetMatchPlayer(nMatchPLayer).SetRound(uint(eco.Round)).SetBank(uint(eco.Bank)).SetEquipment(uint(eco.EqV)).SetSpent(uint(eco.Spent)).Exec(context.Background())
err := tx.RoundStats.Create().SetMatchPlayer(nMatchPLayer).SetRound(uint(eco.Round)).SetBank(uint(eco.Bank)).SetEquipment(uint(eco.EqV)).SetSpent(uint(eco.Spent)).Exec(context.Background())
if err != nil {
err = utils.Rollback(tx, err)
log.Errorf("[DP] Unable to create RoundStat: %v", err)
dp.Done <- demo
continue workloop
}
}
@@ -520,30 +548,45 @@ func (dp *DemoParser) parseWorker() {
enc := gob.NewEncoder(sprayBuf)
err = enc.Encode(sprayAvg)
if err != nil {
err = utils.Rollback(tx, err)
log.Warningf("[DP] Failure to encode spray %v as bytes: %v", spray, err)
continue
dp.Done <- demo
continue workloop
}
err = dp.db.Spray.Create().SetMatchPlayers(nMatchPLayer).SetWeapon(spray.Weapon).SetSpray(sprayBuf.Bytes()).Exec(context.Background())
err = tx.Spray.Create().SetMatchPlayers(nMatchPLayer).SetWeapon(spray.Weapon).SetSpray(sprayBuf.Bytes()).Exec(context.Background())
if err != nil {
err = utils.Rollback(tx, err)
log.Warningf("[DP] Failure adding spray to database: %v", err)
dp.Done <- demo
continue workloop
}
}
}
}
bulk := make([]*ent.MessagesCreate, 0)
var bulk []*ent.MessagesCreate
for _, msg := range tMatchPlayer.Edges.Messages {
bulk = append(bulk, dp.db.Messages.Create().SetMessage(msg.Message).SetAllChat(msg.AllChat).SetTick(msg.Tick).SetMatchPlayer(tMatchPlayer))
bulk = append(bulk, tx.Messages.Create().SetMessage(msg.Message).SetAllChat(msg.AllChat).SetTick(msg.Tick).SetMatchPlayer(tMatchPlayer))
}
if len(bulk) > 0 {
err = dp.db.Messages.CreateBulk(bulk...).Exec(context.Background())
err = tx.Messages.CreateBulk(bulk...).Exec(context.Background())
if err != nil {
log.Warningf("[DP] Failure adding messages to database: %v", err)
err = utils.Rollback(tx, err)
log.Warningf("[DP] Failure adding messages to db: %v", err)
dp.Done <- demo
continue workloop
}
}
}
err = tx.Commit()
if err != nil {
log.Errorf("[DP] error commting to db: %v", err)
dp.Done <- demo
continue
}
log.Infof("[DP] parsed match %d (took %s/%s)", demo.MatchId, downloadTime, time.Since(startTime))
err = demoParser.Close()