CSGOWTF/csgowtfd
3
0
Fork 0
Code Issues Pull Requests Releases 1 Wiki Activity

some refactoring

Browse Source
This commit is contained in:
Giovanni Harting
2021-10-30 08:22:05 +02:00
parent 37233be336
commit 268793f0e5
2 changed files with 388 additions and 390 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
csgo/demo_loader.go
View File

@@ -16,6 +16,7 @@ import (
"google.golang.org/protobuf/proto" "google.golang.org/protobuf/proto"
"io/ioutil" "io/ioutil"
"os" "os"
"sync"
"time" "time"
) )
@@ -48,6 +49,7 @@ type DemoMatchLoader struct {
dp *DemoParser dp *DemoParser
parseDemo chan *Demo parseDemo chan *Demo
parseMap map[string]bool parseMap map[string]bool
parseMapL *sync.Mutex
cache *cache.Cache cache *cache.Cache
connecting bool connecting bool
} }
@@ -156,6 +158,7 @@ func (d *DemoMatchLoader) Setup(config *DemoMatchLoaderConfig) error {
d.db = config.Db d.db = config.Db
d.dp = &DemoParser{} d.dp = &DemoParser{}
d.parseMap = map[string]bool{} d.parseMap = map[string]bool{}
d.parseMapL = new(sync.Mutex)
d.cache = config.Cache d.cache = config.Cache
err := d.dp.Setup(config.Db, config.Worker) err := d.dp.Setup(config.Db, config.Worker)
if err != nil { if err != nil {
@@ -298,15 +301,16 @@ func (d *DemoMatchLoader) requestDemoInfo(matchId uint64, conclusionId uint64, t
} }
func (d *DemoMatchLoader) gcWorker(apiKey string, rl ratelimit.Limiter) { func (d *DemoMatchLoader) gcWorker(apiKey string, rl ratelimit.Limiter) {
for { for demo := range d.parseDemo {
select { d.parseMapL.Lock()
case demo := <-d.parseDemo:
if _, ok := d.parseMap[demo.ShareCode]; ok { if _, ok := d.parseMap[demo.ShareCode]; ok {
log.Infof("[DL] Skipping %s: parsing in progress", demo.ShareCode) log.Infof("[DL] Skipping %s: parsing in progress", demo.ShareCode)
d.parseMapL.Unlock()
continue continue
} else { } else {
d.parseMap[demo.ShareCode] = true d.parseMap[demo.ShareCode] = true
} }
d.parseMapL.Unlock()
if !d.GCReady { if !d.GCReady {
log.Infof("[DL] Postponing match %d (%s): GC not ready", demo.MatchId, demo.ShareCode) log.Infof("[DL] Postponing match %d (%s): GC not ready", demo.MatchId, demo.ShareCode)
@@ -466,4 +470,3 @@ func (d *DemoMatchLoader) gcWorker(apiKey string, rl ratelimit.Limiter) {
delete(d.parseMap, demo.ShareCode) delete(d.parseMap, demo.ShareCode)
} }
} }
}

17
csgo/demo_parser.go
View File

@@ -51,7 +51,7 @@ func (p *DemoParser) ParseDemo(demo *Demo) error {
} }
} }
func (p *DemoParser) downloadReplay(demo *Demo) (io.Reader, error) { func (p *DemoParser) downloadDecompressReplay(demo *Demo) (io.Reader, error) {
log.Debugf("[DP] Downloading replay for %d", demo.MatchId) log.Debugf("[DP] Downloading replay for %d", demo.MatchId)
r, err := http.Get(demo.Url) r, err := http.Get(demo.Url)
@@ -63,7 +63,6 @@ func (p *DemoParser) downloadReplay(demo *Demo) (io.Reader, error) {
return nil, DemoNotFoundError{fmt.Errorf("demo not found")} return nil, DemoNotFoundError{fmt.Errorf("demo not found")}
} }
return bzip2.NewReader(r.Body), nil return bzip2.NewReader(r.Body), nil
} }
func (p *DemoParser) getDBPlayer(demo *Demo, demoPlayer *common.Player) (*ent.Stats, error) { func (p *DemoParser) getDBPlayer(demo *Demo, demoPlayer *common.Player) (*ent.Stats, error) {
@@ -95,9 +94,7 @@ func (p *DemoParser) getMatchPlayerBySteamID(stats []*ent.Stats, steamId uint64)
} }
func (p *DemoParser) parseWorker() { func (p *DemoParser) parseWorker() {
for { for demo := range p.demoQueue {
select {
case demo := <-p.demoQueue:
if demo.MatchId == 0 { if demo.MatchId == 0 {
log.Warningf("[DP] can't parse match %s: no matchid found", demo.ShareCode) log.Warningf("[DP] can't parse match %s: no matchid found", demo.ShareCode)
continue continue
@@ -115,18 +112,17 @@ func (p *DemoParser) parseWorker() {
} }
startTime := time.Now() startTime := time.Now()
fDemo, err := p.downloadReplay(demo) fDemo, err := p.downloadDecompressReplay(demo)
if err != nil { if err != nil {
switch e := err.(type) { if _, ok := err.(DemoNotFoundError); ok {
case DemoNotFoundError:
if tMatch.Date.Before(time.Now().UTC().AddDate(0, 0, -30)) { if tMatch.Date.Before(time.Now().UTC().AddDate(0, 0, -30)) {
log.Infof("[DP] demo expired for match %d", tMatch.ID) log.Infof("[DP] demo expired for match %d", tMatch.ID)
} else { } else {
log.Infof("[DP] demo 404 not found for match %d. Trying again later.", demo.MatchId) log.Infof("[DP] demo 404 not found for match %d. Trying again later.", demo.MatchId)
} }
continue continue
default: } else {
log.Errorf("[DP] Unable to download demo for %d: %v", demo.MatchId, e) log.Errorf("[DP] Unable to download demo for %d: %v", demo.MatchId, err)
continue continue
} }
} }
@@ -344,4 +340,3 @@ func (p *DemoParser) parseWorker() {
} }
} }
} }
}