package csgo import ( "context" "csgowtfd/ent" "csgowtfd/utils" "fmt" "github.com/an0nfunc/go-steam/v3" "github.com/an0nfunc/go-steam/v3/csgo/protocol/protobuf" "github.com/an0nfunc/go-steam/v3/netutil" "github.com/an0nfunc/go-steam/v3/protocol/gamecoordinator" "github.com/an0nfunc/go-steam/v3/protocol/steamlang" "github.com/go-redis/cache/v8" log "github.com/sirupsen/logrus" "go.uber.org/ratelimit" "google.golang.org/protobuf/proto" "io/ioutil" "os" "strconv" "strings" "sync" "time" ) const ( APPID = 730 ) type DemoMatchLoaderConfig struct { Username string Password string AuthCode string Sentry string LoginKey string Db *ent.Client Worker int ApiKey string RateLimit ratelimit.Limiter Cache *cache.Cache SprayTimeout int } type DemoMatchLoader struct { client *steam.Client GCReady bool steamLogin *steam.LogOnDetails matchRecv chan *protobuf.CMsgGCCStrike15V2_MatchList cmList []*netutil.PortAddr sentryFile string loginKey string db *ent.Client dp *DemoParser parseDemo chan *Demo parseMap map[string]bool parseMapL *sync.Mutex cache *cache.Cache connecting bool } func AccountId2SteamId(accId uint32) uint64 { return uint64(accId) + 76561197960265728 } func SteamId2AccountId(steamId uint64) uint32 { return uint32(steamId - 76561197960265728) } func playerStatsFromRound(round *protobuf.CMsgGCCStrike15V2_MatchmakingServerRoundStats, p *ent.Player) (kills int32, deaths int32, assists int32, headshots int32, score int32, mvps int32) { for i, acc := range round.GetReservation().GetAccountIds() { if AccountId2SteamId(acc) == p.ID { return round.GetKills()[i], round.GetDeaths()[i], round.GetAssists()[i], round.GetEnemyHeadshots()[i], round.GetScores()[i], round.GetMvps()[i] } } return 0, 0, 0, 0, 0, 0 } func (d *DemoMatchLoader) HandleGCPacket(pkg *gamecoordinator.GCPacket) { switch pkg.MsgType { case uint32(protobuf.EGCBaseClientMsg_k_EMsgGCClientWelcome): msg := &protobuf.CMsgClientWelcome{} err := proto.Unmarshal(pkg.Body, msg) if err != nil { log.Errorf("[DL] Unable to unmarshal event %v: %v", pkg.MsgType, err) } log.Debugf("[GC] Welcome: %+v", msg) d.GCReady = true case uint32(protobuf.EGCBaseClientMsg_k_EMsgGCClientConnectionStatus): msg := &protobuf.CMsgConnectionStatus{} err := proto.Unmarshal(pkg.Body, msg) if err != nil { log.Errorf("[DL] Unable to unmarshal event %v: %v", pkg.MsgType, err) } log.Debugf("[GC] Status: %+v", msg) if msg.GetStatus() != protobuf.GCConnectionStatus_GCConnectionStatus_HAVE_SESSION { d.GCReady = false go d.greetGC() } case uint32(protobuf.ECsgoGCMsg_k_EMsgGCCStrike15_v2_GC2ClientGlobalStats): msg := &protobuf.GlobalStatistics{} err := proto.Unmarshal(pkg.Body, msg) if err != nil { log.Errorf("[DL] Unable to unmarshal event %v: %v", pkg.MsgType, err) } log.Debugf("[GC] Stats: %+v", msg) d.GCReady = true case uint32(protobuf.ECsgoGCMsg_k_EMsgGCCStrike15_v2_MatchList): msg := &protobuf.CMsgGCCStrike15V2_MatchList{} err := proto.Unmarshal(pkg.Body, msg) if err != nil { log.Errorf("[DL] Unable to unmarshal event %v: %v", pkg.MsgType, err) } d.matchRecv <- msg default: log.Debugf("[GC] Unhandled GC message: %+v", pkg) } } func (d *DemoMatchLoader) getMatchDetails(sharecode string) (*protobuf.CMsgGCCStrike15V2_MatchList, error) { if !d.GCReady { return nil, fmt.Errorf("gc not ready") } matchId, outcomeId, tokenId, err := DecodeSharecode(sharecode) if err != nil { return nil, err } err = d.requestDemoInfo(matchId, outcomeId, uint32(tokenId)) if err != nil { return nil, err } for { select { case matchDetails := <-d.matchRecv: if *matchDetails.Matches[0].Matchid == matchId { return matchDetails, nil } else { d.matchRecv <- matchDetails } } } } func (d *DemoMatchLoader) connectToSteam() error { if d.client.Connected() { return nil } _, err := d.client.Connect() if err != nil { return err } return nil } func (d *DemoMatchLoader) Setup(config *DemoMatchLoaderConfig) error { d.loginKey = config.LoginKey d.sentryFile = config.Sentry d.db = config.Db d.dp = &DemoParser{} d.parseMap = map[string]bool{} d.parseMapL = new(sync.Mutex) d.cache = config.Cache err := d.dp.Setup(config.Db, config.Worker, config.SprayTimeout) if err != nil { return err } d.steamLogin = new(steam.LogOnDetails) d.steamLogin.Username = config.Username d.steamLogin.Password = config.Password d.steamLogin.AuthCode = config.AuthCode d.steamLogin.ShouldRememberPassword = true if _, err := os.Stat(d.sentryFile); err == nil { hash, err := ioutil.ReadFile(d.sentryFile) if err != nil { return err } d.steamLogin.SentryFileHash = hash } if _, err := os.Stat(d.loginKey); err == nil { hash, err := ioutil.ReadFile(d.loginKey) if err != nil { return err } d.steamLogin.LoginKey = string(hash) } d.client = steam.NewClient() err = steam.InitializeSteamDirectory() if err != nil { return err } d.matchRecv = make(chan *protobuf.CMsgGCCStrike15V2_MatchList, 1000) d.parseDemo = make(chan *Demo, 1000) go d.connectLoop() go d.steamEventHandler() for i := 0; i < config.Worker; i++ { go d.gcWorker(config.ApiKey, config.RateLimit) } return nil } func (d DemoMatchLoader) LoadDemo(demo *Demo) error { select { case d.parseDemo <- demo: return nil default: return fmt.Errorf("queue full") } } func (d DemoMatchLoader) connectLoop() { if !d.connecting { d.connecting = true for d.connectToSteam() != nil { log.Infof("[DL] Retrying connecting to steam...") time.Sleep(time.Minute * 10) } } } func (d *DemoMatchLoader) steamEventHandler() { for event := range d.client.Events() { switch e := event.(type) { case *steam.ConnectedEvent: log.Debug("[DL] Connected!") d.client.Auth.LogOn(d.steamLogin) case *steam.MachineAuthUpdateEvent: log.Debug("[DL] Got sentry!") err := ioutil.WriteFile(d.sentryFile, e.Hash, os.ModePerm) if err != nil { log.Errorf("[DL] Unable write sentry file: %v", err) } case *steam.LoggedOnEvent: log.Debug("[DL] Login successfully!") d.client.Social.SetPersonaState(steamlang.EPersonaState_Online) go d.setPlaying() case *steam.LogOnFailedEvent: log.Warningf("[DL] Steam login denied: %+v", e) switch e.Result { case steamlang.EResult_AccountLogonDenied: log.Fatalf("[DL] Please provide AuthCode in config") case steamlang.EResult_InvalidPassword: _ = os.Remove(d.sentryFile) _ = os.Remove(d.loginKey) log.Warningf("[DL] Steam login wrong") go d.connectLoop() case steamlang.EResult_InvalidLoginAuthCode: log.Fatalf("[DL] Steam auth code wrong") } case *steam.DisconnectedEvent: log.Warningf("Steam disconnected, trying to reconnect...") go d.connectLoop() case *steam.LoginKeyEvent: log.Debug("Got login_key!") err := ioutil.WriteFile(d.loginKey, []byte(e.LoginKey), os.ModePerm) if err != nil { log.Errorf("[DL] Unable write login_key: %v", err) } case steam.FatalErrorEvent: log.Debugf("[DL] Got FatalError %+v", e) case error: log.Fatalf("[DL] Got error %+v", e) default: log.Debugf("[DL] %T: %v", e, e) } } } func (d *DemoMatchLoader) setPlaying() { d.client.GC.SetGamesPlayed(APPID) d.client.GC.RegisterPacketHandler(d) go d.greetGC() } func (d *DemoMatchLoader) greetGC() { for !d.GCReady { log.Debugf("[DL] Sending GC greeting") msg := protobuf.CMsgClientHello{} d.client.GC.Write(gamecoordinator.NewGCMsgProtobuf(APPID, uint32(protobuf.EGCBaseClientMsg_k_EMsgGCClientHello), &msg)) time.Sleep(500 * time.Millisecond) } } func (d *DemoMatchLoader) requestDemoInfo(matchId uint64, conclusionId uint64, tokenId uint32) error { if !d.GCReady { return fmt.Errorf("gc not ready") } msg := protobuf.CMsgGCCStrike15V2_MatchListRequestFullGameInfo{Matchid: &matchId, Outcomeid: &conclusionId, Token: &tokenId} d.client.GC.Write(gamecoordinator.NewGCMsgProtobuf(APPID, uint32(protobuf.ECsgoGCMsg_k_EMsgGCCStrike15_v2_MatchListRequestFullGameInfo), &msg)) return nil } func (d *DemoMatchLoader) gcWorker(apiKey string, rl ratelimit.Limiter) { for demo := range d.parseDemo { d.parseMapL.Lock() if _, ok := d.parseMap[demo.ShareCode]; ok { log.Infof("[DL] Skipping %s: parsing in progress", demo.ShareCode) d.parseMapL.Unlock() continue } else { d.parseMap[demo.ShareCode] = true } d.parseMapL.Unlock() if !d.GCReady { log.Infof("[DL] Postponing match %d (%s): GC not ready", demo.MatchId, demo.ShareCode) time.Sleep(5 * time.Second) d.parseMapL.Lock() delete(d.parseMap, demo.ShareCode) d.parseMapL.Unlock() d.parseDemo <- demo continue } matchId, _, _, err := DecodeSharecode(demo.ShareCode) if err != nil || matchId == 0 { log.Warningf("[DL] Can't parse match with sharecode %s: %v", demo.ShareCode, err) d.parseMapL.Lock() delete(d.parseMap, demo.ShareCode) d.parseMapL.Unlock() continue } iMatch, err := d.db.Match.Get(context.Background(), matchId) if err != nil { switch e := err.(type) { case *ent.NotFoundError: break default: log.Errorf("[DL] Failure trying to lookup match %d in db: %v", matchId, e) d.parseMapL.Lock() delete(d.parseMap, demo.ShareCode) d.parseMapL.Unlock() continue } } else { if iMatch.DemoParsed == false && iMatch.Date.After(time.Now().UTC().AddDate(0, 0, -30)) { log.Infof("[DL] Match %d is loaded, but not parsed. Try parsing.", iMatch.ID) demo.MatchId = matchId demo.Url = iMatch.ReplayURL demo.DecryptionKey = iMatch.DecryptionKey err := d.dp.ParseDemo(demo) if err != nil { log.Warningf("[DL] Parsing demo from match %d failed: %v", demo.MatchId, err) } d.parseMapL.Lock() delete(d.parseMap, demo.ShareCode) d.parseMapL.Unlock() continue } log.Infof("[DL] Skipped match %d: already loaded", matchId) d.parseMapL.Lock() delete(d.parseMap, demo.ShareCode) d.parseMapL.Unlock() continue } log.Infof("[DL] Requesting match %d from GC", matchId) t := time.Now() matchDetails, err := d.getMatchDetails(demo.ShareCode) if err != nil { log.Warningf("[DL] Failure to get match-details for %d from GC: %v", demo.MatchId, err) d.parseMapL.Lock() delete(d.parseMap, demo.ShareCode) d.parseMapL.Unlock() continue } log.Infof("[DL] Recieved matchdetails for match %d (%s)", matchId, time.Since(t)) matchZero := matchDetails.GetMatches()[0] lastRound := matchZero.GetRoundstatsall()[len(matchZero.Roundstatsall)-1] var players []*ent.Player for _, accountId := range lastRound.GetReservation().GetAccountIds() { tPlayer, err := utils.Player(d.db, AccountId2SteamId(accountId), apiKey, rl) if err != nil { log.Warningf("[DL] Unable to get player for steamid %d: %v", AccountId2SteamId(accountId), err) continue } players = append(players, tPlayer) } demo.Url = lastRound.GetMap() demo.MatchId = matchZero.GetMatchid() demo.DecryptionKey = []byte(strings.ToUpper(strconv.FormatUint(matchZero.GetWatchablematchinfo().GetClDecryptdataKeyPub(), 16))) tMatch, err := d.db.Match.Create(). SetID(matchZero.GetMatchid()). AddPlayers(players...). SetDate(time.Unix(int64(matchZero.GetMatchtime()), 0).UTC()). SetMaxRounds(int(lastRound.GetMaxRounds())). SetDuration(int(lastRound.GetMatchDuration())). SetShareCode(demo.ShareCode). SetReplayURL(lastRound.GetMap()). SetScoreTeamA(int(lastRound.GetTeamScores()[0])). SetScoreTeamB(int(lastRound.GetTeamScores()[1])). SetMatchResult(int(lastRound.GetMatchResult())). SetDecryptionKey(demo.DecryptionKey). Save(context.Background()) if err != nil { log.Warningf("[DL] Unable to create match %d: %v", matchZero.GetMatchid(), err) d.parseMapL.Lock() delete(d.parseMap, demo.ShareCode) d.parseMapL.Unlock() continue } for i, mPlayer := range players { var ( teamId int mk2 uint mk3 uint mk4 uint mk5 uint ) if i > 4 { teamId = 2 } else { teamId = 1 } var oldKills int32 for _, round := range matchZero.GetRoundstatsall() { kills, _, _, _, _, _ := playerStatsFromRound(round, mPlayer) killDiff := kills - oldKills switch killDiff { case 2: mk2++ case 3: mk3++ case 4: mk4++ case 5: mk5++ } oldKills = kills } kills, deaths, assists, hs, score, mvp := playerStatsFromRound(lastRound, mPlayer) err := d.db.MatchPlayer.Create(). SetMatches(tMatch). SetPlayers(mPlayer). SetTeamID(teamId). SetKills(int(kills)). SetDeaths(int(deaths)). SetAssists(int(assists)). SetMvp(uint(mvp)). SetScore(int(score)). SetHeadshot(int(hs)). SetMk2(mk2). SetMk3(mk3). SetMk4(mk4). SetMk5(mk5). Exec(context.Background()) if err != nil { log.Warningf("[DL] Unable to create stats for player %d in match %d: %v", mPlayer.ID, tMatch.ID, err) } } // clear cache or regen values player for _, p := range players { err = d.cache.Delete(context.Background(), fmt.Sprintf(utils.SideMetaCacheKey, p.ID)) if err != nil { log.Warningf("[DL] Unable to delete cache key %s: %v", fmt.Sprintf(utils.SideMetaCacheKey, p.ID), err) } w, l, t, err := utils.GetWinLossTieForPlayer(p) if err != nil { log.Warningf("[DL] Failure to calculate WinLossTie for player %d: %v", p.ID, err) continue } err = p.Update().SetWins(w).SetTies(t).SetLooses(l).Exec(context.Background()) if err != nil { log.Warningf("[DL] Failure to save WinLossTie for player %d: %v", p.ID, err) } } err = d.dp.ParseDemo(demo) if err != nil { log.Warningf("[DL] Can't queue demo %d for parsing: %v", demo.MatchId, err) } d.parseMapL.Lock() delete(d.parseMap, demo.ShareCode) d.parseMapL.Unlock() } }