CSGOWTF/csgowtfd
3
0
Fork 0
Code Issues Pull Requests Releases 1 Wiki Activity

reworked preventing parsing one match multiple times

Browse Source
This commit is contained in:
Giovanni Harting
2022-02-04 05:43:29 +01:00
parent 6a4e2f3ee3
commit 466a000c86
3 changed files with 161 additions and 144 deletions
Download Patch File Download Diff File Expand all files Collapse all files

236
csgo/demo_loader.go
View File

@@ -51,8 +51,8 @@ type DemoMatchLoader struct {
db *ent.Client
dp *DemoParser
parseDemo chan *Demo
ParseMap map[string]bool
ParseMapL *sync.Mutex
parseMap map[string]bool
parseMapL *sync.RWMutex
cache *cache.Cache
connecting bool
}
@@ -75,7 +75,28 @@ func playerStatsFromRound(round *protobuf.CMsgGCCStrike15V2_MatchmakingServerRou
return 0, 0, 0, 0, 0, 0
}
func (d *DemoMatchLoader) HandleGCPacket(pkg *gamecoordinator.GCPacket) {
func (dml *DemoMatchLoader) IsLoading(demo *Demo) bool {
dml.parseMapL.RLock()
defer dml.parseMapL.RUnlock()
if _, ok := dml.parseMap[demo.ShareCode]; ok {
return true
}
return false
}
func (dml *DemoMatchLoader) unlockDemo(demo *Demo) {
dml.parseMapL.Lock()
defer dml.parseMapL.Unlock()
delete(dml.parseMap, demo.ShareCode)
}
func (dml *DemoMatchLoader) lockDemo(demo *Demo) {
dml.parseMapL.Lock()
defer dml.parseMapL.Unlock()
dml.parseMap[demo.ShareCode] = true
}
func (dml *DemoMatchLoader) HandleGCPacket(pkg *gamecoordinator.GCPacket) {
switch pkg.MsgType {
case uint32(protobuf.EGCBaseClientMsg_k_EMsgGCClientWelcome):
msg := &protobuf.CMsgClientWelcome{}
@@ -84,7 +105,7 @@ func (d *DemoMatchLoader) HandleGCPacket(pkg *gamecoordinator.GCPacket) {
log.Errorf("[DL] Unable to unmarshal event %v: %v", pkg.MsgType, err)
}
log.Debugf("[GC] Welcome: %+v", msg)
d.GCReady = true
dml.GCReady = true
case uint32(protobuf.EGCBaseClientMsg_k_EMsgGCClientConnectionStatus):
msg := &protobuf.CMsgConnectionStatus{}
err := proto.Unmarshal(pkg.Body, msg)
@@ -94,8 +115,8 @@ func (d *DemoMatchLoader) HandleGCPacket(pkg *gamecoordinator.GCPacket) {
log.Debugf("[GC] Status: %+v", msg)
if msg.GetStatus() != protobuf.GCConnectionStatus_GCConnectionStatus_HAVE_SESSION {
d.GCReady = false
go d.greetGC()
dml.GCReady = false
go dml.greetGC()
}
case uint32(protobuf.ECsgoGCMsg_k_EMsgGCCStrike15_v2_GC2ClientGlobalStats):
msg := &protobuf.GlobalStatistics{}
@@ -104,21 +125,21 @@ func (d *DemoMatchLoader) HandleGCPacket(pkg *gamecoordinator.GCPacket) {
log.Errorf("[DL] Unable to unmarshal event %v: %v", pkg.MsgType, err)
}
log.Debugf("[GC] Stats: %+v", msg)
d.GCReady = true
dml.GCReady = true
case uint32(protobuf.ECsgoGCMsg_k_EMsgGCCStrike15_v2_MatchList):
msg := &protobuf.CMsgGCCStrike15V2_MatchList{}
err := proto.Unmarshal(pkg.Body, msg)
if err != nil {
log.Errorf("[DL] Unable to unmarshal event %v: %v", pkg.MsgType, err)
}
d.matchRecv <- msg
dml.matchRecv <- msg
default:
log.Debugf("[GC] Unhandled GC message: %+v", pkg)
}
}
func (d *DemoMatchLoader) getMatchDetails(sharecode string) (*protobuf.CMsgGCCStrike15V2_MatchList, error) {
if !d.GCReady {
func (dml *DemoMatchLoader) getMatchDetails(sharecode string) (*protobuf.CMsgGCCStrike15V2_MatchList, error) {
if !dml.GCReady {
return nil, fmt.Errorf("gc not ready")
}
@@ -126,142 +147,143 @@ func (d *DemoMatchLoader) getMatchDetails(sharecode string) (*protobuf.CMsgGCCSt
if err != nil {
return nil, err
}
err = d.requestDemoInfo(matchId, outcomeId, uint32(tokenId))
err = dml.requestDemoInfo(matchId, outcomeId, uint32(tokenId))
if err != nil {
return nil, err
}
for {
select {
case matchDetails := <-d.matchRecv:
case matchDetails := <-dml.matchRecv:
if *matchDetails.Matches[0].Matchid == matchId {
return matchDetails, nil
} else {
d.matchRecv <- matchDetails
dml.matchRecv <- matchDetails
}
}
}
}
func (d *DemoMatchLoader) connectToSteam() error {
if d.client.Connected() {
func (dml *DemoMatchLoader) connectToSteam() error {
if dml.client.Connected() {
return nil
}
_, err := d.client.Connect()
_, err := dml.client.Connect()
if err != nil {
return err
}
return nil
}
func (d *DemoMatchLoader) Setup(config *DemoMatchLoaderConfig) error {
d.loginKey = config.LoginKey
d.sentryFile = config.Sentry
d.db = config.Db
d.dp = &DemoParser{}
d.ParseMap = map[string]bool{}
d.ParseMapL = new(sync.Mutex)
d.cache = config.Cache
err := d.dp.Setup(config.Db, config.Worker, config.SprayTimeout)
func (dml *DemoMatchLoader) Setup(config *DemoMatchLoaderConfig) error {
dml.loginKey = config.LoginKey
dml.sentryFile = config.Sentry
dml.db = config.Db
dml.dp = &DemoParser{}
dml.parseMap = map[string]bool{}
dml.parseMapL = new(sync.RWMutex)
dml.cache = config.Cache
err := dml.dp.Setup(config.Db, config.Worker, config.SprayTimeout)
if err != nil {
return err
}
d.steamLogin = new(steam.LogOnDetails)
d.steamLogin.Username = config.Username
d.steamLogin.Password = config.Password
d.steamLogin.AuthCode = config.AuthCode
d.steamLogin.ShouldRememberPassword = true
dml.steamLogin = new(steam.LogOnDetails)
dml.steamLogin.Username = config.Username
dml.steamLogin.Password = config.Password
dml.steamLogin.AuthCode = config.AuthCode
dml.steamLogin.ShouldRememberPassword = true
if _, err := os.Stat(d.sentryFile); err == nil {
hash, err := ioutil.ReadFile(d.sentryFile)
if _, err := os.Stat(dml.sentryFile); err == nil {
hash, err := ioutil.ReadFile(dml.sentryFile)
if err != nil {
return err
}
d.steamLogin.SentryFileHash = hash
dml.steamLogin.SentryFileHash = hash
}
if _, err := os.Stat(d.loginKey); err == nil {
hash, err := ioutil.ReadFile(d.loginKey)
if _, err := os.Stat(dml.loginKey); err == nil {
hash, err := ioutil.ReadFile(dml.loginKey)
if err != nil {
return err
}
d.steamLogin.LoginKey = string(hash)
dml.steamLogin.LoginKey = string(hash)
}
d.client = steam.NewClient()
dml.client = steam.NewClient()
err = steam.InitializeSteamDirectory()
if err != nil {
return err
}
d.matchRecv = make(chan *protobuf.CMsgGCCStrike15V2_MatchList, 1000)
d.parseDemo = make(chan *Demo, 1000)
dml.matchRecv = make(chan *protobuf.CMsgGCCStrike15V2_MatchList, 1000)
dml.parseDemo = make(chan *Demo, 1000)
go d.connectLoop()
go d.steamEventHandler()
go dml.connectLoop()
go dml.steamEventHandler()
go dml.demoWorker()
for i := 0; i < config.Worker; i++ {
go d.gcWorker(config.ApiKey, config.RateLimit)
go dml.gcWorker(config.ApiKey, config.RateLimit)
}
return nil
}
func (d DemoMatchLoader) LoadDemo(demo *Demo) error {
func (dml DemoMatchLoader) LoadDemo(demo *Demo) error {
select {
case d.parseDemo <- demo:
case dml.parseDemo <- demo:
return nil
default:
return fmt.Errorf("queue full")
}
}
func (d DemoMatchLoader) connectLoop() {
if !d.connecting {
d.connecting = true
for d.connectToSteam() != nil {
func (dml DemoMatchLoader) connectLoop() {
if !dml.connecting {
dml.connecting = true
for dml.connectToSteam() != nil {
log.Infof("[DL] Retrying connecting to steam...")
time.Sleep(time.Minute * 10)
}
}
}
func (d *DemoMatchLoader) steamEventHandler() {
for event := range d.client.Events() {
func (dml *DemoMatchLoader) steamEventHandler() {
for event := range dml.client.Events() {
switch e := event.(type) {
case *steam.ConnectedEvent:
log.Debug("[DL] Connected!")
d.client.Auth.LogOn(d.steamLogin)
dml.client.Auth.LogOn(dml.steamLogin)
case *steam.MachineAuthUpdateEvent:
log.Debug("[DL] Got sentry!")
err := ioutil.WriteFile(d.sentryFile, e.Hash, os.ModePerm)
err := ioutil.WriteFile(dml.sentryFile, e.Hash, os.ModePerm)
if err != nil {
log.Errorf("[DL] Unable write sentry file: %v", err)
}
case *steam.LoggedOnEvent:
log.Debug("[DL] Login successfully!")
d.client.Social.SetPersonaState(steamlang.EPersonaState_Online)
go d.setPlaying()
dml.client.Social.SetPersonaState(steamlang.EPersonaState_Online)
go dml.setPlaying()
case *steam.LogOnFailedEvent:
log.Warningf("[DL] Steam login denied: %+v", e)
switch e.Result {
case steamlang.EResult_AccountLogonDenied:
log.Fatalf("[DL] Please provide AuthCode in config")
case steamlang.EResult_InvalidPassword:
_ = os.Remove(d.sentryFile)
_ = os.Remove(d.loginKey)
_ = os.Remove(dml.sentryFile)
_ = os.Remove(dml.loginKey)
log.Warningf("[DL] Steam login wrong")
go d.connectLoop()
go dml.connectLoop()
case steamlang.EResult_InvalidLoginAuthCode:
log.Fatalf("[DL] Steam auth code wrong")
}
case *steam.DisconnectedEvent:
log.Warningf("Steam disconnected, trying to reconnect...")
go d.connectLoop()
go dml.connectLoop()
case *steam.LoginKeyEvent:
log.Debug("Got login_key!")
err := ioutil.WriteFile(d.loginKey, []byte(e.LoginKey), os.ModePerm)
err := ioutil.WriteFile(dml.loginKey, []byte(e.LoginKey), os.ModePerm)
if err != nil {
log.Errorf("[DL] Unable write login_key: %v", err)
}
@@ -275,23 +297,23 @@ func (d *DemoMatchLoader) steamEventHandler() {
}
}
func (d *DemoMatchLoader) setPlaying() {
d.client.GC.SetGamesPlayed(APPID)
d.client.GC.RegisterPacketHandler(d)
go d.greetGC()
func (dml *DemoMatchLoader) setPlaying() {
dml.client.GC.SetGamesPlayed(APPID)
dml.client.GC.RegisterPacketHandler(dml)
go dml.greetGC()
}
func (d *DemoMatchLoader) greetGC() {
for !d.GCReady {
func (dml *DemoMatchLoader) greetGC() {
for !dml.GCReady {
log.Debugf("[DL] Sending GC greeting")
msg := protobuf.CMsgClientHello{}
d.client.GC.Write(gamecoordinator.NewGCMsgProtobuf(APPID, uint32(protobuf.EGCBaseClientMsg_k_EMsgGCClientHello), &msg))
dml.client.GC.Write(gamecoordinator.NewGCMsgProtobuf(APPID, uint32(protobuf.EGCBaseClientMsg_k_EMsgGCClientHello), &msg))
time.Sleep(500 * time.Millisecond)
}
}
func (d *DemoMatchLoader) requestDemoInfo(matchId uint64, conclusionId uint64, tokenId uint32) error {
if !d.GCReady {
func (dml *DemoMatchLoader) requestDemoInfo(matchId uint64, conclusionId uint64, tokenId uint32) error {
if !dml.GCReady {
return fmt.Errorf("gc not ready")
}
@@ -299,52 +321,48 @@ func (d *DemoMatchLoader) requestDemoInfo(matchId uint64, conclusionId uint64, t
Outcomeid: &conclusionId,
Token: &tokenId}
d.client.GC.Write(gamecoordinator.NewGCMsgProtobuf(APPID, uint32(protobuf.ECsgoGCMsg_k_EMsgGCCStrike15_v2_MatchListRequestFullGameInfo), &msg))
dml.client.GC.Write(gamecoordinator.NewGCMsgProtobuf(APPID, uint32(protobuf.ECsgoGCMsg_k_EMsgGCCStrike15_v2_MatchListRequestFullGameInfo), &msg))
return nil
}
func (d *DemoMatchLoader) gcWorker(apiKey string, rl ratelimit.Limiter) {
for demo := range d.parseDemo {
d.ParseMapL.Lock()
if _, ok := d.ParseMap[demo.ShareCode]; ok {
log.Infof("[DL] Skipping %s: parsing in progress", demo.ShareCode)
d.ParseMapL.Unlock()
continue
} else {
d.ParseMap[demo.ShareCode] = true
}
d.ParseMapL.Unlock()
func (dml *DemoMatchLoader) demoWorker() {
for demo := range dml.dp.Done {
dml.unlockDemo(demo)
}
}
if !d.GCReady {
func (dml *DemoMatchLoader) gcWorker(apiKey string, rl ratelimit.Limiter) {
for demo := range dml.parseDemo {
if !dml.IsLoading(demo) {
log.Infof("[DL] Skipping %s: parsing in progress", demo.ShareCode)
continue
}
dml.lockDemo(demo)
if !dml.GCReady {
log.Infof("[DL] Postponing match %d (%s): GC not ready", demo.MatchId, demo.ShareCode)
time.Sleep(5 * time.Second)
d.ParseMapL.Lock()
delete(d.ParseMap, demo.ShareCode)
d.ParseMapL.Unlock()
d.parseDemo <- demo
dml.unlockDemo(demo)
dml.parseDemo <- demo
continue
}
matchId, _, _, err := DecodeSharecode(demo.ShareCode)
if err != nil || matchId == 0 {
log.Warningf("[DL] Can't parse match with sharecode %s: %v", demo.ShareCode, err)
d.ParseMapL.Lock()
delete(d.ParseMap, demo.ShareCode)
d.ParseMapL.Unlock()
dml.unlockDemo(demo)
continue
}
iMatch, err := d.db.Match.Get(context.Background(), matchId)
iMatch, err := dml.db.Match.Get(context.Background(), matchId)
if err != nil {
switch e := err.(type) {
case *ent.NotFoundError:
break
default:
log.Errorf("[DL] Failure trying to lookup match %d in db: %v", matchId, e)
d.ParseMapL.Lock()
delete(d.ParseMap, demo.ShareCode)
d.ParseMapL.Unlock()
dml.unlockDemo(demo)
continue
}
} else {
@@ -353,32 +371,26 @@ func (d *DemoMatchLoader) gcWorker(apiKey string, rl ratelimit.Limiter) {
demo.MatchId = matchId
demo.Url = iMatch.ReplayURL
demo.DecryptionKey = iMatch.DecryptionKey
err := d.dp.ParseDemo(demo)
err := dml.dp.ParseDemo(demo)
if err != nil {
log.Warningf("[DL] Parsing demo from match %d failed: %v", demo.MatchId, err)
dml.unlockDemo(demo)
}
d.ParseMapL.Lock()
delete(d.ParseMap, demo.ShareCode)
d.ParseMapL.Unlock()
continue
}
log.Infof("[DL] Skipped match %d: already loaded", matchId)
d.ParseMapL.Lock()
delete(d.ParseMap, demo.ShareCode)
d.ParseMapL.Unlock()
dml.unlockDemo(demo)
continue
}
log.Infof("[DL] Requesting match %d from GC", matchId)
t := time.Now()
matchDetails, err := d.getMatchDetails(demo.ShareCode)
matchDetails, err := dml.getMatchDetails(demo.ShareCode)
if err != nil {
log.Warningf("[DL] Failure to get match-details for %d from GC: %v", demo.MatchId, err)
d.ParseMapL.Lock()
delete(d.ParseMap, demo.ShareCode)
d.ParseMapL.Unlock()
dml.unlockDemo(demo)
continue
}
@@ -389,7 +401,7 @@ func (d *DemoMatchLoader) gcWorker(apiKey string, rl ratelimit.Limiter) {
var players []*ent.Player
for _, accountId := range lastRound.GetReservation().GetAccountIds() {
tPlayer, err := utils.Player(d.db, AccountId2SteamId(accountId), apiKey, rl)
tPlayer, err := utils.Player(dml.db, AccountId2SteamId(accountId), apiKey, rl)
if err != nil {
log.Warningf("[DL] Unable to get player for steamid %d: %v", AccountId2SteamId(accountId), err)
continue
@@ -401,7 +413,7 @@ func (d *DemoMatchLoader) gcWorker(apiKey string, rl ratelimit.Limiter) {
demo.MatchId = matchZero.GetMatchid()
demo.DecryptionKey = []byte(strings.ToUpper(strconv.FormatUint(matchZero.GetWatchablematchinfo().GetClDecryptdataKeyPub(), 16)))
tMatch, err := d.db.Match.Create().
tMatch, err := dml.db.Match.Create().
SetID(matchZero.GetMatchid()).
AddPlayers(players...).
SetDate(time.Unix(int64(matchZero.GetMatchtime()), 0).UTC()).
@@ -416,9 +428,7 @@ func (d *DemoMatchLoader) gcWorker(apiKey string, rl ratelimit.Limiter) {
Save(context.Background())
if err != nil {
log.Warningf("[DL] Unable to create match %d: %v", matchZero.GetMatchid(), err)
d.ParseMapL.Lock()
delete(d.ParseMap, demo.ShareCode)
d.ParseMapL.Unlock()
dml.unlockDemo(demo)
continue
}
@@ -457,7 +467,7 @@ func (d *DemoMatchLoader) gcWorker(apiKey string, rl ratelimit.Limiter) {
}
kills, deaths, assists, hs, score, mvp := playerStatsFromRound(lastRound, mPlayer)
err := d.db.MatchPlayer.Create().
err := dml.db.MatchPlayer.Create().
SetMatches(tMatch).
SetPlayers(mPlayer).
SetTeamID(teamId).
@@ -479,7 +489,7 @@ func (d *DemoMatchLoader) gcWorker(apiKey string, rl ratelimit.Limiter) {
// clear cache or regen values player
for _, p := range players {
err = d.cache.Delete(context.Background(), fmt.Sprintf(utils.SideMetaCacheKey, p.ID))
err = dml.cache.Delete(context.Background(), fmt.Sprintf(utils.SideMetaCacheKey, p.ID))
if err != nil {
log.Warningf("[DL] Unable to delete cache key %s: %v", fmt.Sprintf(utils.SideMetaCacheKey, p.ID), err)
}
@@ -495,12 +505,10 @@ func (d *DemoMatchLoader) gcWorker(apiKey string, rl ratelimit.Limiter) {
}
}
err = d.dp.ParseDemo(demo)
err = dml.dp.ParseDemo(demo)
if err != nil {
log.Warningf("[DL] Can't queue demo %d for parsing: %v", demo.MatchId, err)
dml.unlockDemo(demo)
}
d.ParseMapL.Lock()
delete(d.ParseMap, demo.ShareCode)
d.ParseMapL.Unlock()
}
}