prevent HK from trying to reparse while parsing is in progress

csgo/demo_loader.go

@@ -51,8 +51,8 @@ type DemoMatchLoader struct {
 	db         *ent.Client
 	dp         *DemoParser
 	parseDemo  chan *Demo
-	parseMap   map[string]bool
+	ParseMap   map[string]bool
-	parseMapL  *sync.Mutex
+	ParseMapL  *sync.Mutex
 	cache      *cache.Cache
 	connecting bool
 }
@@ -160,8 +160,8 @@ func (d *DemoMatchLoader) Setup(config *DemoMatchLoaderConfig) error {
 	d.sentryFile = config.Sentry
 	d.db = config.Db
 	d.dp = &DemoParser{}
-	d.parseMap = map[string]bool{}
+	d.ParseMap = map[string]bool{}
-	d.parseMapL = new(sync.Mutex)
+	d.ParseMapL = new(sync.Mutex)
 	d.cache = config.Cache
 	err := d.dp.Setup(config.Db, config.Worker, config.SprayTimeout)
 	if err != nil {
@@ -306,22 +306,22 @@ func (d *DemoMatchLoader) requestDemoInfo(matchId uint64, conclusionId uint64, t
 
 func (d *DemoMatchLoader) gcWorker(apiKey string, rl ratelimit.Limiter) {
 	for demo := range d.parseDemo {
-		d.parseMapL.Lock()
+		d.ParseMapL.Lock()
-		if _, ok := d.parseMap[demo.ShareCode]; ok {
+		if _, ok := d.ParseMap[demo.ShareCode]; ok {
 			log.Infof("[DL] Skipping %s: parsing in progress", demo.ShareCode)
-			d.parseMapL.Unlock()
+			d.ParseMapL.Unlock()
 			continue
 		} else {
-			d.parseMap[demo.ShareCode] = true
+			d.ParseMap[demo.ShareCode] = true
 		}
-		d.parseMapL.Unlock()
+		d.ParseMapL.Unlock()
 
 		if !d.GCReady {
 			log.Infof("[DL] Postponing match %d (%s): GC not ready", demo.MatchId, demo.ShareCode)
 			time.Sleep(5 * time.Second)
-			d.parseMapL.Lock()
+			d.ParseMapL.Lock()
-			delete(d.parseMap, demo.ShareCode)
+			delete(d.ParseMap, demo.ShareCode)
-			d.parseMapL.Unlock()
+			d.ParseMapL.Unlock()
 			d.parseDemo <- demo
 			continue
 		}
@@ -329,9 +329,9 @@ func (d *DemoMatchLoader) gcWorker(apiKey string, rl ratelimit.Limiter) {
 		matchId, _, _, err := DecodeSharecode(demo.ShareCode)
 		if err != nil || matchId == 0 {
 			log.Warningf("[DL] Can't parse match with sharecode %s: %v", demo.ShareCode, err)
-			d.parseMapL.Lock()
+			d.ParseMapL.Lock()
-			delete(d.parseMap, demo.ShareCode)
+			delete(d.ParseMap, demo.ShareCode)
-			d.parseMapL.Unlock()
+			d.ParseMapL.Unlock()
 			continue
 		}
 
@@ -342,9 +342,9 @@ func (d *DemoMatchLoader) gcWorker(apiKey string, rl ratelimit.Limiter) {
 				break
 			default:
 				log.Errorf("[DL] Failure trying to lookup match %d in db: %v", matchId, e)
-				d.parseMapL.Lock()
+				d.ParseMapL.Lock()
-				delete(d.parseMap, demo.ShareCode)
+				delete(d.ParseMap, demo.ShareCode)
-				d.parseMapL.Unlock()
+				d.ParseMapL.Unlock()
 				continue
 			}
 		} else {
@@ -357,16 +357,16 @@ func (d *DemoMatchLoader) gcWorker(apiKey string, rl ratelimit.Limiter) {
 				if err != nil {
 					log.Warningf("[DL] Parsing demo from match %d failed: %v", demo.MatchId, err)
 				}
-				d.parseMapL.Lock()
+				d.ParseMapL.Lock()
-				delete(d.parseMap, demo.ShareCode)
+				delete(d.ParseMap, demo.ShareCode)
-				d.parseMapL.Unlock()
+				d.ParseMapL.Unlock()
 				continue
 			}
 
 			log.Infof("[DL] Skipped match %d: already loaded", matchId)
-			d.parseMapL.Lock()
+			d.ParseMapL.Lock()
-			delete(d.parseMap, demo.ShareCode)
+			delete(d.ParseMap, demo.ShareCode)
-			d.parseMapL.Unlock()
+			d.ParseMapL.Unlock()
 			continue
 		}
 
@@ -376,9 +376,9 @@ func (d *DemoMatchLoader) gcWorker(apiKey string, rl ratelimit.Limiter) {
 		matchDetails, err := d.getMatchDetails(demo.ShareCode)
 		if err != nil {
 			log.Warningf("[DL] Failure to get match-details for %d from GC: %v", demo.MatchId, err)
-			d.parseMapL.Lock()
+			d.ParseMapL.Lock()
-			delete(d.parseMap, demo.ShareCode)
+			delete(d.ParseMap, demo.ShareCode)
-			d.parseMapL.Unlock()
+			d.ParseMapL.Unlock()
 			continue
 		}
 
@@ -416,9 +416,9 @@ func (d *DemoMatchLoader) gcWorker(apiKey string, rl ratelimit.Limiter) {
 			Save(context.Background())
 		if err != nil {
 			log.Warningf("[DL] Unable to create match %d: %v", matchZero.GetMatchid(), err)
-			d.parseMapL.Lock()
+			d.ParseMapL.Lock()
-			delete(d.parseMap, demo.ShareCode)
+			delete(d.ParseMap, demo.ShareCode)
-			d.parseMapL.Unlock()
+			d.ParseMapL.Unlock()
 			continue
 		}
 
@@ -499,8 +499,8 @@ func (d *DemoMatchLoader) gcWorker(apiKey string, rl ratelimit.Limiter) {
 		if err != nil {
 			log.Warningf("[DL] Can't queue demo %d for parsing: %v", demo.MatchId, err)
 		}
-		d.parseMapL.Lock()
+		d.ParseMapL.Lock()
-		delete(d.parseMap, demo.ShareCode)
+		delete(d.ParseMap, demo.ShareCode)
-		d.parseMapL.Unlock()
+		d.ParseMapL.Unlock()
 	}
 }

main.go

@@ -174,11 +174,18 @@ func housekeeping() {
 		}
 
 		for _, m := range tMatches {
+			demoLoader.ParseMapL.Lock()
+			if _, ok := demoLoader.ParseMap[m.ShareCode]; ok {
+				log.Infof("[HK] Skipping %s: parsing in progress", m.ShareCode)
+				demoLoader.ParseMapL.Unlock()
+				continue
+			}
 			log.Infof("[HK] Try reparsing match %d, played on %s", m.ID, m.Date)
 			err := demoLoader.LoadDemo(&csgo.Demo{MatchId: m.ID, ShareCode: m.ShareCode})
 			if err != nil {
 				log.Warningf("[HK] Failure trying to parse match %d: %v", m.ID, err)
 			}
+			demoLoader.ParseMapL.Unlock()
 		}
 
 		// check for inconsistent matches