From 40cc67d3545bac7fe718eb7dd2b5ad1895780836 Mon Sep 17 00:00:00 2001 From: chiyi Date: Sun, 12 Jul 2026 22:59:52 +0800 Subject: [PATCH] =?UTF-8?q?=E6=9B=B4=E6=96=B0grok=E5=8D=8F=E8=AE=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../internal/provider/grok/statsig_engine.go | 190 ++++++++++++++---- .../internal/provider/grok/statsig_shim.js | 16 +- .../internal/provider/grok/zz_probe_test.go | 110 ++++++++++ backend/internal/provider/runway/video.go | 27 +-- 4 files changed, 284 insertions(+), 59 deletions(-) create mode 100644 backend/internal/provider/grok/zz_probe_test.go diff --git a/backend/internal/provider/grok/statsig_engine.go b/backend/internal/provider/grok/statsig_engine.go index f7af694..59ffb07 100644 --- a/backend/internal/provider/grok/statsig_engine.go +++ b/backend/internal/provider/grok/statsig_engine.go @@ -12,6 +12,8 @@ import ( "context" "crypto/sha256" "crypto/sha512" + "encoding/base64" + "encoding/json" _ "embed" "errors" "fmt" @@ -36,18 +38,21 @@ const sigPoolSize = 4 // homepage fetched, or chunk location failed). Callers fall back to the static path. var errEngineNotReady = errors.New("statsig engine not ready") +// locateConcurrency bounds parallel chunk fetches during signer discovery. +const locateConcurrency = 24 + var ( - // signer-chunk location patterns (Turbopack). The caller chunk contains the - // literal "x-statsig-id" and a lazy import `.A().then(e=>t(e.default()))`. - statsigCallerRe = regexp.MustCompile(`\.A\((\d+)\)\.then\(`) - chunkPathRe = regexp.MustCompile(`/_next/static/chunks/[a-zA-Z0-9_.\-/]+\.js`) + // chunkPathRe matches chunk URLs in the homepage; allChunkRefRe additionally + // matches the loader-manifest's lazy refs (which drop the /_next/ prefix). + chunkPathRe = regexp.MustCompile(`/_next/static/chunks/[a-zA-Z0-9_.\-/]+\.js`) + allChunkRefRe = regexp.MustCompile(`(?:/_next/)?static/chunks/[a-zA-Z0-9_.\-/]+\.js`) // goja's parser tries to fetch //# sourceMappingURL=... from disk and errors. sourceMapRe = regexp.MustCompile(`(?m)//[#@]\s*sourceMappingURL=\S*`) sigMgrMu sync.Mutex - sigBuildKey string // hash of the homepage chunk list; changes on reship - sigChunkSrc string // current signer chunk source - sigPool chan *sigEngine // pool of ready engines for sigChunkSrc + sigBuildKey string // hash of the homepage chunk list; changes on reship + sigChunkSrc string // current signer chunk source + sigPool chan *sigEngine // pool of ready engines for sigChunkSrc ) // sigEngine wraps one goja runtime with grok's signer chunk loaded. A goja runtime @@ -171,7 +176,28 @@ func ensureEngine(ctx context.Context, client tlsclient.HttpClient, homeHTML str return } - src, err := locateSignerChunk(ctx, client, dedupe(paths)) + // Inputs for build-agnostic behavioral verification of candidate chunks. + mm := statsigMetaRe.FindStringSubmatch(homeHTML) + if mm == nil { + log.Printf("grok statsig: no seed meta in homepage; cannot locate signer") + return + } + seed, err := decodeStatsigSeed(mm[1]) + if err != nil { + log.Printf("grok statsig: seed decode failed: %v", err) + return + } + curves, err := parseStatsigCurves(homeHTML) + if err != nil { + log.Printf("grok statsig: curves parse failed: %v", err) + return + } + cj, err := json.Marshal(curves) + if err != nil { + return + } + + src, err := locateSignerChunk(ctx, client, dedupe(paths), mm[1], string(cj), seed) if err != nil { log.Printf("grok statsig: locate signer chunk failed (will use static fallback): %v", err) return @@ -192,50 +218,132 @@ func ensureEngine(ctx context.Context, client tlsclient.HttpClient, homeHTML str log.Printf("grok statsig: self-heal engine ready (build %s..)", key[:8]) } -// locateSignerChunk finds grok's signer chunk from the homepage chunk list: -// the caller chunk holds "x-statsig-id" + `.A()`; a loader chunk registers that -// with `Promise.all(["static/chunks/XXX.js"]...)` — XXX is the signer. -func locateSignerChunk(ctx context.Context, client tlsclient.HttpClient, paths []string) (string, error) { - var callerID string - loaderRe := (*regexp.Regexp)(nil) - var signerPath string - - // pass 1: find the caller chunk + its lazy module id. - for _, p := range paths { - body, err := fetchChunk(ctx, client, p) - if err != nil || !strings.Contains(body, "x-statsig-id") { - continue - } - if m := statsigCallerRe.FindStringSubmatch(body); m != nil { - callerID = m[1] - } - break +// locateSignerChunk finds grok's obfuscated anti-bot signer chunk build-agnostically, +// with no dependency on rotating literals (header name, class names, module ids). +// It screens every reachable chunk with a cheap obfuscator.io fingerprint, then +// confirms the true signer by RUNNING it in goja and checking its output embeds the +// homepage seed (signerEmbedsSeed). The signer is lazily loaded (not referenced in +// the HTML directly), so candidates also include the chunk paths named inside the +// homepage's Turbopack loader manifest. +func locateSignerChunk(ctx context.Context, client tlsclient.HttpClient, homeChunks []string, seedB64, curvesJSON string, seed []byte) (string, error) { + seen := map[string]bool{} + var lazy []string + for _, p := range homeChunks { + seen[p] = true } - if callerID == "" { - return "", errors.New("statsig caller module id not found") - } - // loader registers: ,,=>{ ... Promise.all(["static/chunks/XXX.js"] ... - loaderRe = regexp.MustCompile(`,` + callerID + `,\w+=>\{[^}]*?Promise\.all\(\["(static/chunks/[^"]+\.js)"`) - // pass 2: find the loader chunk that maps callerID -> signer chunk path. - for _, p := range paths { + // Pass 1 (sequential): fetch the homepage chunks, verify them directly, and + // harvest every chunk path they reference (the loader manifest lists the signer). + for _, p := range homeChunks { body, err := fetchChunk(ctx, client, p) if err != nil { continue } - if m := loaderRe.FindStringSubmatch(body); m != nil { - signerPath = m[1] - break + if src, ok := verifySignerChunk(body, seedB64, curvesJSON, seed); ok { + return src, nil + } + for _, ref := range allChunkRefRe.FindAllString(body, -1) { + np := normalizeChunkPath(ref) + if !seen[np] { + seen[np] = true + lazy = append(lazy, np) + } } } - if signerPath == "" { - return "", fmt.Errorf("signer chunk path for module %s not found", callerID) + + // Pass 2 (concurrent): fetch the lazily-referenced chunks, cheap-fingerprint each, + // behaviorally verify the matches, and stop at the first chunk that round-trips seed. + ctx2, cancel := context.WithCancel(ctx) + defer cancel() + found := make(chan string, 1) + sem := make(chan struct{}, locateConcurrency) + var wg sync.WaitGroup + for _, p := range lazy { + if ctx2.Err() != nil { + break + } + sem <- struct{}{} + wg.Add(1) + go func(p string) { + defer wg.Done() + defer func() { <-sem }() + if ctx2.Err() != nil { + return + } + body, err := fetchChunk(ctx2, client, p) + if err != nil { + return + } + if src, ok := verifySignerChunk(body, seedB64, curvesJSON, seed); ok { + select { + case found <- src: + cancel() + default: + } + } + }(p) } - src, err := fetchChunk(ctx, client, "/_next/"+signerPath) + go func() { wg.Wait(); close(found) }() + if src, ok := <-found; ok { + return src, nil + } + return "", fmt.Errorf("statsig signer chunk not found among %d candidates", len(homeChunks)+len(lazy)) +} + +// verifySignerChunk returns the goja-ready source if body is grok's signer: it must +// carry the obfuscator.io fingerprint AND, when executed, produce an x-statsig-id +// whose decoded record embeds the homepage seed. +func verifySignerChunk(body, seedB64, curvesJSON string, seed []byte) (string, bool) { + if !isObfuscatedSigner(body) { + return "", false + } + clean := sourceMapRe.ReplaceAllString(body, "") + eng, err := newSigEngine(clean) if err != nil { - return "", fmt.Errorf("fetch signer chunk: %w", err) + return "", false } - return src, nil + id, err := eng.statsigID(seedB64, curvesJSON, "/rest/app-chat/conversations/new", "POST") + if err != nil { + return "", false + } + if !signerEmbedsSeed(id, seed) { + return "", false + } + return clean, true +} + +// normalizeChunkPath turns a bare or /_next/-prefixed chunk ref into a fetch path. +func normalizeChunkPath(ref string) string { + return "/_next/" + strings.TrimPrefix(ref, "/_next/") +} + +// isObfuscatedSigner cheaply screens a chunk for the obfuscator.io string-array +// decoder that grok applies ONLY to its anti-bot signer (the rest of the app is +// plain Turbopack output). The RC4-style byte decoder (`...^...%256`) is the stable +// tell; it matches a handful of chunks, which behavioral verification then narrows +// to exactly one. This is deliberately build-agnostic (no rotating class/id/header). +func isObfuscatedSigner(src string) bool { + return strings.Contains(src, "%256") && + strings.Contains(src, "String.fromCharCode") && + strings.Contains(src, "charCodeAt") +} + +// signerEmbedsSeed decodes a candidate x-statsig-id, strips the per-call XOR mask +// (plaintext[0] is 0x00, so masked[0] is the key), and checks the plaintext record +// begins with 0x00 + the exact homepage seed. Only grok's real signer round-trips +// OUR seed, so this uniquely identifies the signer chunk regardless of obfuscation. +func signerEmbedsSeed(id string, seed []byte) bool { + raw, err := base64.RawStdEncoding.DecodeString(id) + if err != nil || len(raw) < 1+len(seed) { + return false + } + key := raw[0] // plaintext[0] is 0x00, so the mask key == masked byte 0 + for i := 0; i < len(seed); i++ { + if raw[1+i]^key != seed[i] { + return false + } + } + return true } func fetchChunk(ctx context.Context, client tlsclient.HttpClient, path string) (string, error) { diff --git a/backend/internal/provider/grok/statsig_shim.js b/backend/internal/provider/grok/statsig_shim.js index 96cdc49..99c167c 100644 --- a/backend/internal/provider/grok/statsig_shim.js +++ b/backend/internal/provider/grok/statsig_shim.js @@ -134,17 +134,21 @@ createElement: function (tag) { return makeEl({ nodeName: String(tag || 'div').toUpperCase() }); }, querySelectorAll: function (sel) { sel = String(sel); - if (/aufz1o/.test(sel)) { - var curves = JSON.parse(g.__CURVES); - return curves.map(function (grp) { - return groupEl(grp.map(function (cv) { return cv.color.concat([cv.deg], cv.bezier); })); - }); - } + // The seed is selected by a name/verification attribute selector + // (e.g. [name^=gr] or [name*=verification]); the curve group container is + // selected by a per-build hashed CLASS selector (e.g. .r-aufz1o, .r-3nqkqc) + // which rotates on every reship — so match by shape, not literal class. if (/verification|name/i.test(sel)) { var seed = g.__SEED; return [{ nodeName: 'META', getAttribute: function (a) { return a === 'content' ? seed : null; }, get content() { return seed; } }]; } + if (/(^|\s|,)\./.test(sel)) { + var curves = JSON.parse(g.__CURVES); + return curves.map(function (grp) { + return groupEl(grp.map(function (cv) { return cv.color.concat([cv.deg], cv.bezier); })); + }); + } return []; }, querySelector: function (sel) { var r = this.querySelectorAll(sel); return r[0] || null; }, diff --git a/backend/internal/provider/grok/zz_probe_test.go b/backend/internal/provider/grok/zz_probe_test.go new file mode 100644 index 0000000..22a380d --- /dev/null +++ b/backend/internal/provider/grok/zz_probe_test.go @@ -0,0 +1,110 @@ +package grok + +import ( + "context" + "crypto/sha1" + "fmt" + "io" + "os" + "path/filepath" + "regexp" + "strings" + "testing" + "time" + + http "github.com/bogdanfinn/fhttp" +) + +var innerChunkRe = regexp.MustCompile(`static/chunks/[a-zA-Z0-9_.\-/]+\.js`) + +func hnew(ctx context.Context, url, token string) (*http.Request, error) { + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + req.Header = http.Header{ + "accept": {"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"}, + "accept-language": {"en-US,en;q=0.9"}, + "user-agent": {userAgent}, + "cookie": {"sso=" + token + "; sso-rw=" + token}, + http.HeaderOrderKey: {"accept", "accept-language", "user-agent", "cookie"}, + } + return req, nil +} + +func TestGrokCacheChunks(t *testing.T) { + token := os.Getenv("DIAG_SSO") + if token == "" { + t.Skip("no DIAG_SSO") + } + dir := os.Getenv("DIAG_CACHE") + if dir == "" { + dir = `f:\ai-gateway\vivid-ai\backend\_chunks` + } + os.MkdirAll(dir, 0o755) + logf, _ := os.Create(filepath.Join(dir, "_log.txt")) + defer logf.Close() + lg := func(f string, a ...any) { + fmt.Fprintf(logf, f+"\n", a...) + logf.Sync() + } + + c := NewClient(os.Getenv("DIAG_PROXY")) + ctx, cancel := context.WithTimeout(context.Background(), 3000*time.Second) + defer cancel() + client, err := c.newTLSClient() + if err != nil { + t.Fatal(err) + } + + hreq, _ := hnew(ctx, apiBase+"/", token) + hresp, err := client.Do(hreq) + if err != nil { + t.Fatal(err) + } + body, _ := io.ReadAll(hresp.Body) + hresp.Body.Close() + html := string(body) + os.WriteFile(filepath.Join(dir, "_home.html"), body, 0o644) + + seen := map[string]bool{} + queue := dedupe(chunkPathRe.FindAllString(html, -1)) + lg("home chunk refs: %d", len(queue)) + + saved := 0 + for len(queue) > 0 && saved < 3000 { + p := queue[0] + queue = queue[1:] + norm := p + if !strings.HasPrefix(norm, "/_next/") { + norm = "/_next/" + strings.TrimPrefix(norm, "/") + } + if seen[norm] { + continue + } + seen[norm] = true + src, err := fetchChunk(ctx, client, norm) + if err != nil { + continue + } + saved++ + base := norm[strings.LastIndex(norm, "/")+1:] + fn := fmt.Sprintf("%x_%s", sha1.Sum([]byte(norm)), base) + if len(fn) > 120 { + fn = fn[:120] + } + os.WriteFile(filepath.Join(dir, fn), []byte(src), 0o644) + for _, m := range innerChunkRe.FindAllString(src, -1) { + nn := "/_next/" + m + if !seen[nn] { + queue = append(queue, nn) + } + } + if saved%50 == 0 { + lg("saved=%d queue=%d", saved, len(queue)) + } + } + lg("CRAWL DONE saved=%d queue_left=%d", saved, len(queue)) + fmt.Printf("CRAWL DONE saved=%d\n", saved) +} diff --git a/backend/internal/provider/runway/video.go b/backend/internal/provider/runway/video.go index a9ac2e5..93df90b 100644 --- a/backend/internal/provider/runway/video.go +++ b/backend/internal/provider/runway/video.go @@ -70,37 +70,40 @@ func (c *Client) GenerateVideo(ctx context.Context, token, teamID, prompt, aspec return nil, nil, errors.New("runway: failed to decode first-frame image") } - // Use a single client (and therefore a single exit IP) for the whole flow. - // The web app runs upload, task-create, polling and download from one IP; a - // mid-flow IP switch (proxy for submit, local for the rest) is a strong - // bot/risk signal, so we route everything through the proxy-aware client. - apiClient, err := c.newTLSClient() + // Only the task-create (generate submit) egresses via the proxy; reference + // upload, dataset create, polling and download run on the local IP (matches + // the image pipeline). + submitClient, err := c.newTLSClient() + if err != nil { + return nil, nil, err + } + directClient, err := c.newDirectTLSClient() if err != nil { return nil, nil, err } filename := "frame_" + time.Now().UTC().Format("20060102_150405") + ".png" - previewUploadID, _, err := c.uploadFile(ctx, apiClient, token, teamID, filename, "DATASET_PREVIEW", frame) + previewUploadID, _, err := c.uploadFile(ctx, directClient, token, teamID, filename, "DATASET_PREVIEW", frame) if err != nil { return nil, nil, err } - datasetUploadID, _, err := c.uploadFile(ctx, apiClient, token, teamID, filename, "DATASET", frame) + datasetUploadID, _, err := c.uploadFile(ctx, directClient, token, teamID, filename, "DATASET", frame) if err != nil { return nil, nil, err } - assetID, imageURL, err := c.createDataset(ctx, apiClient, token, teamID, filename, datasetUploadID, previewUploadID, cfg.Width, cfg.Height) + assetID, imageURL, err := c.createDataset(ctx, directClient, token, teamID, filename, datasetUploadID, previewUploadID, cfg.Width, cfg.Height) if err != nil { return nil, nil, err } - assetGroupID, _ := c.assetGroupID(ctx, apiClient, token, teamID) // best-effort + assetGroupID, _ := c.assetGroupID(ctx, directClient, token, teamID) // best-effort - taskID, err := c.createTask(ctx, apiClient, token, teamID, prompt, imageURL, assetID, assetGroupID, aspectRatio, seconds) + taskID, err := c.createTask(ctx, submitClient, token, teamID, prompt, imageURL, assetID, assetGroupID, aspectRatio, seconds) if err != nil { return nil, nil, err } - artifactURL, err := c.pollTask(ctx, apiClient, token, teamID, taskID) + artifactURL, err := c.pollTask(ctx, directClient, token, teamID, taskID) if err != nil { return nil, nil, err } @@ -113,7 +116,7 @@ func (c *Client) GenerateVideo(ctx context.Context, token, teamID, prompt, aspec if !downloadResult { return nil, meta, nil } - data, err := c.download(ctx, apiClient, artifactURL) + data, err := c.download(ctx, directClient, artifactURL) if err != nil { return nil, nil, err }