CSGOWTF/csgowtfd
3
0
Fork 0
Code Issues Pull Requests Releases 1 Wiki Activity

reworked preventing parsing one match multiple times

Browse Source
This commit is contained in:
Giovanni Harting
2022-02-04 05:43:29 +01:00
parent 6a4e2f3ee3
commit 466a000c86
3 changed files with 161 additions and 144 deletions
Download Patch File Download Diff File Expand all files Collapse all files

61
csgo/demo_parser.go
View File

@@ -32,6 +32,7 @@ type DemoParser struct {
tempDir string
db *ent.Client
sprayTimeout int
Done chan *Demo
}
type Encounter struct {
@@ -105,19 +106,20 @@ func (s *Sprays) Avg() (avg [][]float32) {
return
}
func (p *DemoParser) Setup(db *ent.Client, worker int, sprayTimeout int) error {
p.demoQueue = make(chan *Demo, 1000)
p.db = db
p.sprayTimeout = sprayTimeout
func (dp *DemoParser) Setup(db *ent.Client, worker int, sprayTimeout int) error {
dp.demoQueue = make(chan *Demo, 1000)
dp.db = db
dp.sprayTimeout = sprayTimeout
dp.Done = make(chan *Demo, worker)
for i := 0; i < worker; i++ {
go p.parseWorker()
go dp.parseWorker()
}
return nil
}
func (p *DemoParser) ParseDemo(demo *Demo) error {
func (dp *DemoParser) ParseDemo(demo *Demo) error {
select {
case p.demoQueue <- demo:
case dp.demoQueue <- demo:
return nil
default:
return fmt.Errorf("queue full")
@@ -138,8 +140,8 @@ func (d *Demo) download() (io.Reader, error) {
return bzip2.NewReader(r.Body), nil
}
func (p *DemoParser) getDBPlayer(demo *Demo, demoPlayer *common.Player) (*ent.MatchPlayer, error) {
tMatchPlayer, err := p.db.MatchPlayer.Query().WithMatches(func(q *ent.MatchQuery) {
func (dp *DemoParser) getDBPlayer(demo *Demo, demoPlayer *common.Player) (*ent.MatchPlayer, error) {
tMatchPlayer, err := dp.db.MatchPlayer.Query().WithMatches(func(q *ent.MatchQuery) {
q.Where(match.ID(demo.MatchId))
}).WithPlayers(func(q *ent.PlayerQuery) {
q.Where(player.ID(demoPlayer.SteamID64))
@@ -151,7 +153,7 @@ func (p *DemoParser) getDBPlayer(demo *Demo, demoPlayer *common.Player) (*ent.Ma
return tMatchPlayer, nil
}
func (p *DemoParser) MatchPlayerBySteamID(stats []*ent.MatchPlayer, steamId uint64) (*ent.MatchPlayer, error) {
func (dp *DemoParser) MatchPlayerBySteamID(stats []*ent.MatchPlayer, steamId uint64) (*ent.MatchPlayer, error) {
for _, tStats := range stats {
tPLayer, err := tStats.Edges.PlayersOrErr()
if err != nil {
@@ -190,21 +192,24 @@ func setMatchPlayerColor(matchPlayer *ent.MatchPlayer, demoPlayer *common.Player
}
}
func (p *DemoParser) parseWorker() {
for demo := range p.demoQueue {
func (dp *DemoParser) parseWorker() {
for demo := range dp.demoQueue {
if demo.MatchId == 0 {
log.Warningf("[DP] can't parse match %s: no matchid found", demo.ShareCode)
dp.Done <- demo
continue
}
tMatch, err := p.db.Match.Get(context.Background(), demo.MatchId)
tMatch, err := dp.db.Match.Get(context.Background(), demo.MatchId)
if err != nil {
log.Errorf("[DP] Unable to get match %d: %v", demo.MatchId, err)
dp.Done <- demo
continue
}
if tMatch.DemoParsed {
log.Infof("[DP] skipped already parsed %d", demo.MatchId)
dp.Done <- demo
continue
}
@@ -217,9 +222,11 @@ func (p *DemoParser) parseWorker() {
} else {
log.Infof("[DP] demo 404 not found for match %d. Trying again later.", demo.MatchId)
}
dp.Done <- demo
continue
} else {
log.Errorf("[DP] Unable to download demo for %d: %v", demo.MatchId, err)
dp.Done <- demo
continue
}
}
@@ -228,6 +235,7 @@ func (p *DemoParser) parseWorker() {
tStats, err := tMatch.QueryStats().WithPlayers().All(context.Background())
if err != nil {
log.Errorf("[DP] Failed to find players for match %d: %v", demo.MatchId, err)
dp.Done <- demo
continue
}
@@ -255,7 +263,7 @@ func (p *DemoParser) parseWorker() {
// onChatMessage
demoParser.RegisterEventHandler(func(e events.ChatMessage) {
gs := demoParser.GameState()
tAttacker, err := p.MatchPlayerBySteamID(tStats, e.Sender.SteamID64)
tAttacker, err := dp.MatchPlayerBySteamID(tStats, e.Sender.SteamID64)
if err != nil {
log.Warningf("[DP] Unable to get player for id %d: %v", e.Sender.SteamID64, err)
return
@@ -292,7 +300,7 @@ func (p *DemoParser) parseWorker() {
for _, spray := range spays {
if e.Shooter.SteamID64 == spray.Sprayer && int(e.Weapon.Type) == spray.Weapon {
playerWeaponFound = true
spray.Add(demoParser.CurrentTime(), []float32{e.Shooter.ViewDirectionX(), e.Shooter.ViewDirectionY()}, p.sprayTimeout)
spray.Add(demoParser.CurrentTime(), []float32{e.Shooter.ViewDirectionX(), e.Shooter.ViewDirectionY()}, dp.sprayTimeout)
}
}
@@ -311,7 +319,7 @@ func (p *DemoParser) parseWorker() {
return
}
tAttacker, err := p.MatchPlayerBySteamID(tStats, e.Attacker.SteamID64)
tAttacker, err := dp.MatchPlayerBySteamID(tStats, e.Attacker.SteamID64)
if err != nil {
log.Warningf("[DP] Unable to get player for id %d: %v", e.Attacker.SteamID64, err)
return
@@ -373,7 +381,7 @@ func (p *DemoParser) parseWorker() {
return
}
tAttacker, err := p.MatchPlayerBySteamID(tStats, e.Attacker.SteamID64)
tAttacker, err := dp.MatchPlayerBySteamID(tStats, e.Attacker.SteamID64)
if err != nil {
log.Warningf("[DP] Unable to get player for id %d: %v", e.Attacker.SteamID64, err)
return
@@ -401,7 +409,7 @@ func (p *DemoParser) parseWorker() {
// onPlayerConnected
demoParser.RegisterEventHandler(func(e events.PlayerTeamChange) {
if e.Player != nil && e.Player.SteamID64 != 0 {
tMatchPlayer, err := p.MatchPlayerBySteamID(tStats, e.Player.SteamID64)
tMatchPlayer, err := dp.MatchPlayerBySteamID(tStats, e.Player.SteamID64)
if err != nil {
log.Warningf("[DP] Unable to get player for id %d: %v", e.Player.SteamID64, err)
return
@@ -416,7 +424,7 @@ func (p *DemoParser) parseWorker() {
for _, demoPlayer := range gs.Participants().Playing() {
if demoPlayer != nil && demoPlayer.SteamID64 != 0 {
tMatchPlayer, err := p.MatchPlayerBySteamID(tStats, demoPlayer.SteamID64)
tMatchPlayer, err := dp.MatchPlayerBySteamID(tStats, demoPlayer.SteamID64)
if err != nil {
log.Warningf("[DP] Unable to get player for id %d: %v", demoPlayer.SteamID64, err)
return
@@ -429,7 +437,7 @@ func (p *DemoParser) parseWorker() {
// onRankUpdate
demoParser.RegisterEventHandler(func(e events.RankUpdate) {
if e.SteamID64() != 0 {
tMatchPlayer, err := p.MatchPlayerBySteamID(tStats, e.SteamID64())
tMatchPlayer, err := dp.MatchPlayerBySteamID(tStats, e.SteamID64())
if err != nil {
log.Warningf("[DP] Unable to get player for id %d: %v", e.SteamID64(), err)
return
@@ -443,6 +451,7 @@ func (p *DemoParser) parseWorker() {
err = demoParser.ParseToEnd()
if err != nil {
log.Errorf("[DP] Error parsing replay: %v", err)
dp.Done <- demo
continue
}
@@ -453,6 +462,7 @@ func (p *DemoParser) parseWorker() {
Exec(context.Background())
if err != nil {
log.Errorf("[DP] Unable to update match %d in database: %v", demo.MatchId, err)
dp.Done <- demo
continue
}
@@ -487,14 +497,14 @@ func (p *DemoParser) parseWorker() {
}
for _, eqDmg := range eqMap[tMatchPlayer.PlayerStats] {
err := p.db.Weapon.Create().SetStat(nMatchPLayer).SetDmg(eqDmg.Dmg).SetVictim(eqDmg.To).SetHitGroup(eqDmg.HitGroup).SetEqType(eqDmg.Eq).Exec(context.Background())
err := dp.db.Weapon.Create().SetStat(nMatchPLayer).SetDmg(eqDmg.Dmg).SetVictim(eqDmg.To).SetHitGroup(eqDmg.HitGroup).SetEqType(eqDmg.Eq).Exec(context.Background())
if err != nil {
log.Errorf("[DP] Unable to create WeaponStat: %v", err)
}
}
for _, eco := range ecoMap[tMatchPlayer.PlayerStats] {
err := p.db.RoundStats.Create().SetMatchPlayer(nMatchPLayer).SetRound(uint(eco.Round)).SetBank(uint(eco.Bank)).SetEquipment(uint(eco.EqV)).SetSpent(uint(eco.Spent)).Exec(context.Background())
err := dp.db.RoundStats.Create().SetMatchPlayer(nMatchPLayer).SetRound(uint(eco.Round)).SetBank(uint(eco.Bank)).SetEquipment(uint(eco.EqV)).SetSpent(uint(eco.Spent)).Exec(context.Background())
if err != nil {
log.Errorf("[DP] Unable to create RoundStat: %v", err)
}
@@ -511,7 +521,7 @@ func (p *DemoParser) parseWorker() {
continue
}
err = p.db.Spray.Create().SetMatchPlayers(nMatchPLayer).SetWeapon(spray.Weapon).SetSpray(sprayBuf.Bytes()).Exec(context.Background())
err = dp.db.Spray.Create().SetMatchPlayers(nMatchPLayer).SetWeapon(spray.Weapon).SetSpray(sprayBuf.Bytes()).Exec(context.Background())
if err != nil {
log.Warningf("[DP] Failure adding spray to database: %v", err)
}
@@ -521,10 +531,10 @@ func (p *DemoParser) parseWorker() {
bulk := make([]*ent.MessagesCreate, 0)
for _, msg := range tMatchPlayer.Edges.Messages {
bulk = append(bulk, p.db.Messages.Create().SetMessage(msg.Message).SetAllChat(msg.AllChat).SetTick(msg.Tick).SetMatchPlayer(tMatchPlayer))
bulk = append(bulk, dp.db.Messages.Create().SetMessage(msg.Message).SetAllChat(msg.AllChat).SetTick(msg.Tick).SetMatchPlayer(tMatchPlayer))
}
if len(bulk) > 0 {
err = p.db.Messages.CreateBulk(bulk...).Exec(context.Background())
err = dp.db.Messages.CreateBulk(bulk...).Exec(context.Background())
if err != nil {
log.Warningf("[DP] Failure adding messages to database: %v", err)
}
@@ -537,5 +547,6 @@ func (p *DemoParser) parseWorker() {
if err != nil {
log.Errorf("[DP] Unable close demo file for match %d: %v", demo.MatchId, err)
}
dp.Done <- demo
}
}