package csgo import ( "bytes" "compress/bzip2" "context" "encoding/gob" "errors" "fmt" "git.harting.dev/csgowtf/csgowtfd/ent" "git.harting.dev/csgowtf/csgowtfd/ent/match" "git.harting.dev/csgowtf/csgowtfd/ent/matchplayer" "git.harting.dev/csgowtf/csgowtfd/ent/player" "git.harting.dev/csgowtf/csgowtfd/utils" "github.com/golang/geo/r2" "github.com/markus-wa/demoinfocs-golang/v3/pkg/demoinfocs" "github.com/markus-wa/demoinfocs-golang/v3/pkg/demoinfocs/common" "github.com/markus-wa/demoinfocs-golang/v3/pkg/demoinfocs/events" log "github.com/sirupsen/logrus" "io" "net/http" "time" ) type Demo struct { ShareCode string MatchId uint64 Url string DecryptionKey []byte } type DemoParser struct { demoQueue chan *Demo tempDir string db *ent.Client sprayTimeout int Done chan *Demo } type Encounter struct { Spotted uint64 TimeToReact float32 FirstFrag bool Spray []*Spray CrosshairMovement r2.Point Time time.Duration } type Sprays struct { Sprayer uint64 Sprays []*Spray Weapon int } type Spray struct { Time time.Duration Spray [][]float32 } type DemoNotFoundError struct { error } func (s *Sprays) Add(currentTime time.Duration, sprayPoint []float32, timeout int) { sprayFound := false for _, sp := range s.Sprays { if currentTime.Milliseconds()-sp.Time.Milliseconds() <= int64(timeout) { sprayFound = true sp.Spray = append(sp.Spray, []float32{ sprayPoint[0] - sp.Spray[0][0], sprayPoint[1] - sp.Spray[0][1], }) } } if !sprayFound { s.Sprays = append(s.Sprays, &Spray{ Time: currentTime, Spray: [][]float32{sprayPoint}, }) } } func (s *Sprays) Avg() (avg [][]float32) { var ( total int ) for _, sp := range s.Sprays { for i, r2p := range sp.Spray { if i == 0 { continue } total++ if len(avg) <= i-1 { avg = append(avg, r2p) } else { avg[i-1][0] += r2p[0] avg[i-1][1] += r2p[1] } } } for i, r2p := range avg { avg[i][0] = r2p[0] / float32(total) avg[i][1] = r2p[1] / float32(total) } return } func (dp *DemoParser) Setup(db *ent.Client, worker int, sprayTimeout int) error { dp.demoQueue = make(chan *Demo, 1000) dp.db = db dp.sprayTimeout = sprayTimeout dp.Done = make(chan *Demo, worker) for i := 0; i < worker; i++ { go dp.parseWorker() } return nil } func (dp *DemoParser) ParseDemo(demo *Demo) error { select { case dp.demoQueue <- demo: return nil default: return fmt.Errorf("queue full") } } func (d *Demo) download() (io.Reader, error) { log.Debugf("[DP] Downloading replay for %d", d.MatchId) r, err := http.Get(d.Url) if err != nil { return nil, err } if r.StatusCode != http.StatusOK { return nil, DemoNotFoundError{fmt.Errorf("demo not found")} } return bzip2.NewReader(r.Body), nil } func (dp *DemoParser) getDBPlayer(demo *Demo, demoPlayer *common.Player) (*ent.MatchPlayer, error) { tMatchPlayer, err := dp.db.MatchPlayer.Query().WithMatches(func(q *ent.MatchQuery) { q.Where(match.ID(demo.MatchId)) }).WithPlayers(func(q *ent.PlayerQuery) { q.Where(player.ID(demoPlayer.SteamID64)) }).Only(context.Background()) if err != nil { return nil, err } return tMatchPlayer, nil } func (dp *DemoParser) MatchPlayerBySteamID(stats []*ent.MatchPlayer, steamId uint64) (*ent.MatchPlayer, error) { for _, tStats := range stats { tPLayer, err := tStats.Edges.PlayersOrErr() if err != nil { return nil, fmt.Errorf("unable to get stats from statList: %w", err) } if tPLayer.ID == steamId { return tStats, nil } } return nil, fmt.Errorf("player not found") } func setMatchPlayerColor(matchPlayer *ent.MatchPlayer, demoPlayer *common.Player) { if matchPlayer == nil || demoPlayer == nil { return } matchPlayer.Crosshair = demoPlayer.CrosshairCode() color, _ := demoPlayer.ColorOrErr() switch color { case common.Yellow: matchPlayer.Color = matchplayer.ColorYellow break case common.Green: matchPlayer.Color = matchplayer.ColorGreen break case common.Purple: matchPlayer.Color = matchplayer.ColorPurple break case common.Blue: matchPlayer.Color = matchplayer.ColorBlue break case common.Orange: matchPlayer.Color = matchplayer.ColorOrange break case common.Grey: matchPlayer.Color = matchplayer.ColorGrey } } func (dp *DemoParser) parseWorker() { workloop: for demo := range dp.demoQueue { if demo.MatchId == 0 { log.Warningf("[DP] can't parse match %s: no matchid found", demo.ShareCode) dp.Done <- demo continue } // init tx tx, err := dp.db.Tx(context.Background()) if err != nil { log.Errorf("[DP] error creating transaction: %v", err) dp.Done <- demo return } tMatch, err := tx.Match.Get(context.Background(), demo.MatchId) if err != nil { err = utils.Rollback(tx, err) log.Errorf("[DP] Unable to get match %d: %v", demo.MatchId, err) dp.Done <- demo continue } if tMatch.DemoParsed { err = utils.Rollback(tx, err) log.Infof("[DP] skipped already parsed %d", demo.MatchId) dp.Done <- demo continue } startTime := time.Now() fDemo, err := demo.download() if err != nil { if errors.Is(err, DemoNotFoundError{}) { err = utils.Rollback(tx, err) if tMatch.Date.Before(time.Now().UTC().AddDate(0, 0, -30)) { log.Infof("[DP] demo expired for match %d", tMatch.ID) } else { log.Infof("[DP] demo 404 not found for match %d. Retrying later.", demo.MatchId) } dp.Done <- demo continue } else { err = utils.Rollback(tx, err) log.Errorf("[DP] Unable to download demo for %d: %v", demo.MatchId, err) dp.Done <- demo continue } } downloadTime := time.Since(startTime) tStats, err := tMatch.QueryStats().WithPlayers().All(context.Background()) if err != nil { err = utils.Rollback(tx, err) log.Errorf("[DP] Failed to find players for match %d: %v", demo.MatchId, err) dp.Done <- demo continue } eqMap := make(map[uint64][]*struct { Eq int HitGroup int Dmg uint To uint64 }) ecoMap := make(map[uint64][]*struct { Round int EqV int Bank int Spent int }, 0) encounters := make([]*Encounter, 0) spays := make([]*Sprays, 0) cfg := demoinfocs.DefaultParserConfig if len(demo.DecryptionKey) == 16 { cfg.NetMessageDecryptionKey = demo.DecryptionKey } demoParser := demoinfocs.NewParserWithConfig(fDemo, cfg) // onChatMessage demoParser.RegisterEventHandler(func(e events.ChatMessage) { if e.Sender != nil { gs := demoParser.GameState() tAttacker, err := dp.MatchPlayerBySteamID(tStats, e.Sender.SteamID64) if err != nil { log.Warningf("[DP] Unable to get player for id %d: %v", e.Sender.SteamID64, err) return } tAttacker.Edges.Messages = append(tAttacker.Edges.Messages, &ent.Messages{ Message: e.Text, AllChat: e.IsChatAll, Tick: gs.IngameTick(), }) } }) // onPlayerSpotted demoParser.RegisterEventHandler(func(e events.PlayerSpottersChanged) { gs := demoParser.GameState() if !gs.IsMatchStarted() { return } encounters = append(encounters, &Encounter{ Spotted: e.Spotted.SteamID64, Time: demoParser.CurrentTime(), }) }) // onWeaponFire demoParser.RegisterEventHandler(func(e events.WeaponFire) { gs := demoParser.GameState() if !gs.IsMatchStarted() { return } playerWeaponFound := false for _, spray := range spays { if e.Shooter.SteamID64 == spray.Sprayer && int(e.Weapon.Type) == spray.Weapon { playerWeaponFound = true spray.Add(demoParser.CurrentTime(), []float32{e.Shooter.ViewDirectionX(), e.Shooter.ViewDirectionY()}, dp.sprayTimeout) } } if !playerWeaponFound { spays = append(spays, &Sprays{ Sprayer: e.Shooter.SteamID64, Sprays: []*Spray{{demoParser.CurrentTime(), [][]float32{{e.Shooter.ViewDirectionX(), e.Shooter.ViewDirectionY()}}}}, Weapon: int(e.Weapon.Type), }) } }) // onPlayerHurt demoParser.RegisterEventHandler(func(e events.PlayerHurt) { if e.Attacker == nil || e.Player == nil || e.Weapon == nil || !demoParser.GameState().IsMatchStarted() { return } tAttacker, err := dp.MatchPlayerBySteamID(tStats, e.Attacker.SteamID64) if err != nil { log.Warningf("[DP] Unable to get player for id %d: %v", e.Attacker.SteamID64, err) return } if e.Attacker.Team == e.Player.Team { tAttacker.DmgTeam += uint(e.HealthDamageTaken) } else { tAttacker.DmgEnemy += uint(e.HealthDamageTaken) } if _, ok := eqMap[e.Attacker.SteamID64]; !ok { eqMap[e.Attacker.SteamID64] = make([]*struct { Eq int HitGroup int Dmg uint To uint64 }, 0) } found := false for _, di := range eqMap[e.Attacker.SteamID64] { if di.Eq == int(e.Weapon.Type) && di.HitGroup == int(e.HitGroup) { di.Dmg += uint(e.HealthDamageTaken) found = true } } if !found { eqMap[e.Attacker.SteamID64] = append(eqMap[e.Attacker.SteamID64], &struct { Eq int HitGroup int Dmg uint To uint64 }{Eq: int(e.Weapon.Type), HitGroup: int(e.HitGroup), Dmg: uint(e.HealthDamageTaken), To: e.Player.SteamID64}) } }) // onRoundEnd demoParser.RegisterEventHandler(func(e events.RoundEnd) { gs := demoParser.GameState() if !gs.IsMatchStarted() { return } // track eco for _, p := range gs.Participants().Playing() { ecoMap[p.SteamID64] = append(ecoMap[p.SteamID64], &struct { Round int EqV int Bank int Spent int }{Round: gs.TotalRoundsPlayed(), EqV: p.EquipmentValueCurrent(), Bank: p.Money(), Spent: p.MoneySpentThisRound()}) } }) // onPlayerFlashed demoParser.RegisterEventHandler(func(e events.PlayerFlashed) { if e.Attacker == nil || e.Player == nil || !demoParser.GameState().IsMatchStarted() { return } tAttacker, err := dp.MatchPlayerBySteamID(tStats, e.Attacker.SteamID64) if err != nil { log.Warningf("[DP] Unable to get player for id %d: %v", e.Attacker.SteamID64, err) return } // team flash if e.Attacker.Team == e.Player.Team && e.Attacker.SteamID64 != e.Player.SteamID64 { tAttacker.FlashTotalTeam++ tAttacker.FlashDurationTeam += e.Player.FlashDuration } // own flash if e.Attacker.SteamID64 == e.Player.SteamID64 { tAttacker.FlashTotalSelf++ tAttacker.FlashDurationSelf += e.Player.FlashDuration } // enemy flash if e.Attacker.Team != e.Player.Team { tAttacker.FlashTotalEnemy++ tAttacker.FlashDurationEnemy += e.Player.FlashDuration } }) // onPlayerConnected demoParser.RegisterEventHandler(func(e events.PlayerTeamChange) { if e.Player != nil && e.Player.SteamID64 != 0 { tMatchPlayer, err := dp.MatchPlayerBySteamID(tStats, e.Player.SteamID64) if err != nil { log.Warningf("[DP] Unable to get player for id %d: %v", e.Player.SteamID64, err) return } setMatchPlayerColor(tMatchPlayer, e.Player) } }) // onMatchStart demoParser.RegisterEventHandler(func(e events.MatchStart) { gs := demoParser.GameState() for _, demoPlayer := range gs.Participants().Playing() { if demoPlayer != nil && demoPlayer.SteamID64 != 0 { tMatchPlayer, err := dp.MatchPlayerBySteamID(tStats, demoPlayer.SteamID64) if err != nil { log.Warningf("[DP] Unable to get player for id %d: %v", demoPlayer.SteamID64, err) return } setMatchPlayerColor(tMatchPlayer, demoPlayer) } } }) // onRankUpdate demoParser.RegisterEventHandler(func(e events.RankUpdate) { if e.SteamID64() != 0 { tMatchPlayer, err := dp.MatchPlayerBySteamID(tStats, e.SteamID64()) if err != nil { log.Warningf("[DP] Unable to get player for id %d: %v", e.SteamID64(), err) return } tMatchPlayer.RankOld = e.RankOld tMatchPlayer.RankNew = e.RankNew } }) err = demoParser.ParseToEnd() if err != nil { err = utils.Rollback(tx, err) log.Errorf("[DP] Error parsing replay: %v", err) dp.Done <- demo continue } err = tMatch.Update(). SetMap(demoParser.Header().MapName). SetDemoParsed(true). SetTickRate(demoParser.TickRate()). Exec(context.Background()) if err != nil { err = utils.Rollback(tx, err) log.Errorf("[DP] Unable to update match %d in database: %v", demo.MatchId, err) dp.Done <- demo continue } for _, tMatchPlayer := range tStats { if tMatchPlayer.Color == "" { tMatchPlayer.Color = matchplayer.ColorGrey } nMatchPLayer, err := tMatchPlayer.Update(). SetDmgTeam(tMatchPlayer.DmgTeam). SetDmgEnemy(tMatchPlayer.DmgEnemy). SetUdHe(tMatchPlayer.UdHe). SetUdFlash(tMatchPlayer.UdFlash). SetUdSmoke(tMatchPlayer.UdSmoke). SetUdFlames(tMatchPlayer.UdFlames). SetRankOld(tMatchPlayer.RankOld). SetRankNew(tMatchPlayer.RankNew). SetColor(tMatchPlayer.Color). SetCrosshair(tMatchPlayer.Crosshair). SetFlashDurationTeam(tMatchPlayer.FlashDurationTeam). SetFlashDurationSelf(tMatchPlayer.FlashDurationSelf). SetFlashDurationEnemy(tMatchPlayer.FlashDurationEnemy). SetFlashTotalEnemy(tMatchPlayer.FlashTotalEnemy). SetFlashTotalSelf(tMatchPlayer.FlashTotalSelf). SetFlashTotalTeam(tMatchPlayer.FlashTotalTeam). SetDmgTeam(tMatchPlayer.DmgTeam). SetDmgEnemy(tMatchPlayer.DmgEnemy). Save(context.Background()) if err != nil { err = utils.Rollback(tx, err) log.Errorf("[DP] Unable to update stats %d in database: %v", tMatchPlayer.PlayerStats, err) dp.Done <- demo continue workloop } for _, eqDmg := range eqMap[tMatchPlayer.PlayerStats] { err = tx.Weapon.Create().SetStat(nMatchPLayer).SetDmg(eqDmg.Dmg).SetVictim(eqDmg.To).SetHitGroup(eqDmg.HitGroup).SetEqType(eqDmg.Eq).Exec(context.Background()) if err != nil { err = utils.Rollback(tx, err) log.Errorf("[DP] Unable to create WeaponStat: %v", err) dp.Done <- demo continue workloop } } for _, eco := range ecoMap[tMatchPlayer.PlayerStats] { err := tx.RoundStats.Create().SetMatchPlayer(nMatchPLayer).SetRound(uint(eco.Round)).SetBank(uint(eco.Bank)).SetEquipment(uint(eco.EqV)).SetSpent(uint(eco.Spent)).Exec(context.Background()) if err != nil { err = utils.Rollback(tx, err) log.Errorf("[DP] Unable to create RoundStat: %v", err) dp.Done <- demo continue workloop } } for _, spray := range spays { if spray.Sprayer == nMatchPLayer.PlayerStats { if sprayAvg := spray.Avg(); len(sprayAvg) >= 5 { sprayBuf := new(bytes.Buffer) enc := gob.NewEncoder(sprayBuf) err = enc.Encode(sprayAvg) if err != nil { err = utils.Rollback(tx, err) log.Warningf("[DP] Failure to encode spray %v as bytes: %v", spray, err) dp.Done <- demo continue workloop } err = tx.Spray.Create().SetMatchPlayers(nMatchPLayer).SetWeapon(spray.Weapon).SetSpray(sprayBuf.Bytes()).Exec(context.Background()) if err != nil { err = utils.Rollback(tx, err) log.Warningf("[DP] Failure adding spray to database: %v", err) dp.Done <- demo continue workloop } } } } var bulk []*ent.MessagesCreate for _, msg := range tMatchPlayer.Edges.Messages { bulk = append(bulk, tx.Messages.Create().SetMessage(msg.Message).SetAllChat(msg.AllChat).SetTick(msg.Tick).SetMatchPlayer(tMatchPlayer)) } if len(bulk) > 0 { err = tx.Messages.CreateBulk(bulk...).Exec(context.Background()) if err != nil { err = utils.Rollback(tx, err) log.Warningf("[DP] Failure adding messages to db: %v", err) dp.Done <- demo continue workloop } } } err = tx.Commit() if err != nil { log.Errorf("[DP] error commting to db: %v", err) dp.Done <- demo continue } log.Infof("[DP] parsed match %d (took %s/%s)", demo.MatchId, downloadTime, time.Since(startTime)) err = demoParser.Close() if err != nil { log.Errorf("[DP] Unable close demo file for match %d: %v", demo.MatchId, err) } dp.Done <- demo } }