更新grok协议

backend/internal/provider/grok/statsig_engine.go

@@ -12,6 +12,8 @@ import (
 	"context"
 	"crypto/sha256"
 	"crypto/sha512"
+	"encoding/base64"
+	"encoding/json"
 	_ "embed"
 	"errors"
 	"fmt"
@@ -36,11 +38,14 @@ const sigPoolSize = 4
 // homepage fetched, or chunk location failed). Callers fall back to the static path.
 var errEngineNotReady = errors.New("statsig engine not ready")
 
+// locateConcurrency bounds parallel chunk fetches during signer discovery.
+const locateConcurrency = 24
+
 var (
-	// signer-chunk location patterns (Turbopack). The caller chunk contains the
-	// literal "x-statsig-id" and a lazy import `.A(<moduleId>).then(e=>t(e.default()))`.
-	statsigCallerRe = regexp.MustCompile(`\.A\((\d+)\)\.then\(`)
+	// chunkPathRe matches chunk URLs in the homepage; allChunkRefRe additionally
+	// matches the loader-manifest's lazy refs (which drop the /_next/ prefix).
 	chunkPathRe   = regexp.MustCompile(`/_next/static/chunks/[a-zA-Z0-9_.\-/]+\.js`)
+	allChunkRefRe = regexp.MustCompile(`(?:/_next/)?static/chunks/[a-zA-Z0-9_.\-/]+\.js`)
 	// goja's parser tries to fetch //# sourceMappingURL=... from disk and errors.
 	sourceMapRe = regexp.MustCompile(`(?m)//[#@]\s*sourceMappingURL=\S*`)
 
@@ -171,7 +176,28 @@ func ensureEngine(ctx context.Context, client tlsclient.HttpClient, homeHTML str
 		return
 	}
 
-	src, err := locateSignerChunk(ctx, client, dedupe(paths))
+	// Inputs for build-agnostic behavioral verification of candidate chunks.
+	mm := statsigMetaRe.FindStringSubmatch(homeHTML)
+	if mm == nil {
+		log.Printf("grok statsig: no seed meta in homepage; cannot locate signer")
+		return
+	}
+	seed, err := decodeStatsigSeed(mm[1])
+	if err != nil {
+		log.Printf("grok statsig: seed decode failed: %v", err)
+		return
+	}
+	curves, err := parseStatsigCurves(homeHTML)
+	if err != nil {
+		log.Printf("grok statsig: curves parse failed: %v", err)
+		return
+	}
+	cj, err := json.Marshal(curves)
+	if err != nil {
+		return
+	}
+
+	src, err := locateSignerChunk(ctx, client, dedupe(paths), mm[1], string(cj), seed)
 	if err != nil {
 		log.Printf("grok statsig: locate signer chunk failed (will use static fallback): %v", err)
 		return
@@ -192,50 +218,132 @@ func ensureEngine(ctx context.Context, client tlsclient.HttpClient, homeHTML str
 	log.Printf("grok statsig: self-heal engine ready (build %s..)", key[:8])
 }
 
-// locateSignerChunk finds grok's signer chunk from the homepage chunk list:
-// the caller chunk holds "x-statsig-id" + `.A(<id>)`; a loader chunk registers that
-// <id> with `Promise.all(["static/chunks/XXX.js"]...)` — XXX is the signer.
-func locateSignerChunk(ctx context.Context, client tlsclient.HttpClient, paths []string) (string, error) {
-	var callerID string
-	loaderRe := (*regexp.Regexp)(nil)
-	var signerPath string
+// locateSignerChunk finds grok's obfuscated anti-bot signer chunk build-agnostically,
+// with no dependency on rotating literals (header name, class names, module ids).
+// It screens every reachable chunk with a cheap obfuscator.io fingerprint, then
+// confirms the true signer by RUNNING it in goja and checking its output embeds the
+// homepage seed (signerEmbedsSeed). The signer is lazily loaded (not referenced in
+// the HTML directly), so candidates also include the chunk paths named inside the
+// homepage's Turbopack loader manifest.
+func locateSignerChunk(ctx context.Context, client tlsclient.HttpClient, homeChunks []string, seedB64, curvesJSON string, seed []byte) (string, error) {
+	seen := map[string]bool{}
+	var lazy []string
+	for _, p := range homeChunks {
+		seen[p] = true
+	}
 
-	// pass 1: find the caller chunk + its lazy module id.
-	for _, p := range paths {
-		body, err := fetchChunk(ctx, client, p)
-		if err != nil || !strings.Contains(body, "x-statsig-id") {
-			continue
-		}
-		if m := statsigCallerRe.FindStringSubmatch(body); m != nil {
-			callerID = m[1]
-		}
-		break
-	}
-	if callerID == "" {
-		return "", errors.New("statsig caller module id not found")
-	}
-	// loader registers: ,<callerID>,<param>=>{ ... Promise.all(["static/chunks/XXX.js"] ...
-	loaderRe = regexp.MustCompile(`,` + callerID + `,\w+=>\{[^}]*?Promise\.all\(\["(static/chunks/[^"]+\.js)"`)
-
-	// pass 2: find the loader chunk that maps callerID -> signer chunk path.
-	for _, p := range paths {
+	// Pass 1 (sequential): fetch the homepage chunks, verify them directly, and
+	// harvest every chunk path they reference (the loader manifest lists the signer).
+	for _, p := range homeChunks {
 		body, err := fetchChunk(ctx, client, p)
 		if err != nil {
 			continue
 		}
-		if m := loaderRe.FindStringSubmatch(body); m != nil {
-			signerPath = m[1]
-			break
-		}
-	}
-	if signerPath == "" {
-		return "", fmt.Errorf("signer chunk path for module %s not found", callerID)
-	}
-	src, err := fetchChunk(ctx, client, "/_next/"+signerPath)
-	if err != nil {
-		return "", fmt.Errorf("fetch signer chunk: %w", err)
-	}
+		if src, ok := verifySignerChunk(body, seedB64, curvesJSON, seed); ok {
 			return src, nil
+		}
+		for _, ref := range allChunkRefRe.FindAllString(body, -1) {
+			np := normalizeChunkPath(ref)
+			if !seen[np] {
+				seen[np] = true
+				lazy = append(lazy, np)
+			}
+		}
+	}
+
+	// Pass 2 (concurrent): fetch the lazily-referenced chunks, cheap-fingerprint each,
+	// behaviorally verify the matches, and stop at the first chunk that round-trips seed.
+	ctx2, cancel := context.WithCancel(ctx)
+	defer cancel()
+	found := make(chan string, 1)
+	sem := make(chan struct{}, locateConcurrency)
+	var wg sync.WaitGroup
+	for _, p := range lazy {
+		if ctx2.Err() != nil {
+			break
+		}
+		sem <- struct{}{}
+		wg.Add(1)
+		go func(p string) {
+			defer wg.Done()
+			defer func() { <-sem }()
+			if ctx2.Err() != nil {
+				return
+			}
+			body, err := fetchChunk(ctx2, client, p)
+			if err != nil {
+				return
+			}
+			if src, ok := verifySignerChunk(body, seedB64, curvesJSON, seed); ok {
+				select {
+				case found <- src:
+					cancel()
+				default:
+				}
+			}
+		}(p)
+	}
+	go func() { wg.Wait(); close(found) }()
+	if src, ok := <-found; ok {
+		return src, nil
+	}
+	return "", fmt.Errorf("statsig signer chunk not found among %d candidates", len(homeChunks)+len(lazy))
+}
+
+// verifySignerChunk returns the goja-ready source if body is grok's signer: it must
+// carry the obfuscator.io fingerprint AND, when executed, produce an x-statsig-id
+// whose decoded record embeds the homepage seed.
+func verifySignerChunk(body, seedB64, curvesJSON string, seed []byte) (string, bool) {
+	if !isObfuscatedSigner(body) {
+		return "", false
+	}
+	clean := sourceMapRe.ReplaceAllString(body, "")
+	eng, err := newSigEngine(clean)
+	if err != nil {
+		return "", false
+	}
+	id, err := eng.statsigID(seedB64, curvesJSON, "/rest/app-chat/conversations/new", "POST")
+	if err != nil {
+		return "", false
+	}
+	if !signerEmbedsSeed(id, seed) {
+		return "", false
+	}
+	return clean, true
+}
+
+// normalizeChunkPath turns a bare or /_next/-prefixed chunk ref into a fetch path.
+func normalizeChunkPath(ref string) string {
+	return "/_next/" + strings.TrimPrefix(ref, "/_next/")
+}
+
+// isObfuscatedSigner cheaply screens a chunk for the obfuscator.io string-array
+// decoder that grok applies ONLY to its anti-bot signer (the rest of the app is
+// plain Turbopack output). The RC4-style byte decoder (`...^...%256`) is the stable
+// tell; it matches a handful of chunks, which behavioral verification then narrows
+// to exactly one. This is deliberately build-agnostic (no rotating class/id/header).
+func isObfuscatedSigner(src string) bool {
+	return strings.Contains(src, "%256") &&
+		strings.Contains(src, "String.fromCharCode") &&
+		strings.Contains(src, "charCodeAt")
+}
+
+// signerEmbedsSeed decodes a candidate x-statsig-id, strips the per-call XOR mask
+// (plaintext[0] is 0x00, so masked[0] is the key), and checks the plaintext record
+// begins with 0x00 + the exact homepage seed. Only grok's real signer round-trips
+// OUR seed, so this uniquely identifies the signer chunk regardless of obfuscation.
+func signerEmbedsSeed(id string, seed []byte) bool {
+	raw, err := base64.RawStdEncoding.DecodeString(id)
+	if err != nil || len(raw) < 1+len(seed) {
+		return false
+	}
+	key := raw[0] // plaintext[0] is 0x00, so the mask key == masked byte 0
+	for i := 0; i < len(seed); i++ {
+		if raw[1+i]^key != seed[i] {
+			return false
+		}
+	}
+	return true
 }
 
 func fetchChunk(ctx context.Context, client tlsclient.HttpClient, path string) (string, error) {

backend/internal/provider/grok/statsig_shim.js

@@ -134,17 +134,21 @@
     createElement: function (tag) { return makeEl({ nodeName: String(tag || 'div').toUpperCase() }); },
     querySelectorAll: function (sel) {
       sel = String(sel);
-      if (/aufz1o/.test(sel)) {
-        var curves = JSON.parse(g.__CURVES);
-        return curves.map(function (grp) {
-          return groupEl(grp.map(function (cv) { return cv.color.concat([cv.deg], cv.bezier); }));
-        });
-      }
+      // The seed <meta> is selected by a name/verification attribute selector
+      // (e.g. [name^=gr] or [name*=verification]); the curve group container is
+      // selected by a per-build hashed CLASS selector (e.g. .r-aufz1o, .r-3nqkqc)
+      // which rotates on every reship — so match by shape, not literal class.
       if (/verification|name/i.test(sel)) {
         var seed = g.__SEED;
         return [{ nodeName: 'META', getAttribute: function (a) { return a === 'content' ? seed : null; },
                   get content() { return seed; } }];
       }
+      if (/(^|\s|,)\./.test(sel)) {
+        var curves = JSON.parse(g.__CURVES);
+        return curves.map(function (grp) {
+          return groupEl(grp.map(function (cv) { return cv.color.concat([cv.deg], cv.bezier); }));
+        });
+      }
       return [];
     },
     querySelector: function (sel) { var r = this.querySelectorAll(sel); return r[0] || null; },

backend/internal/provider/grok/zz_probe_test.go

@@ -0,0 +1,110 @@
+package grok
+
+import (
+	"context"
+	"crypto/sha1"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"testing"
+	"time"
+
+	http "github.com/bogdanfinn/fhttp"
+)
+
+var innerChunkRe = regexp.MustCompile(`static/chunks/[a-zA-Z0-9_.\-/]+\.js`)
+
+func hnew(ctx context.Context, url, token string) (*http.Request, error) {
+	req, err := http.NewRequest(http.MethodGet, url, nil)
+	if err != nil {
+		return nil, err
+	}
+	req = req.WithContext(ctx)
+	req.Header = http.Header{
+		"accept":          {"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"},
+		"accept-language": {"en-US,en;q=0.9"},
+		"user-agent":      {userAgent},
+		"cookie":          {"sso=" + token + "; sso-rw=" + token},
+		http.HeaderOrderKey: {"accept", "accept-language", "user-agent", "cookie"},
+	}
+	return req, nil
+}
+
+func TestGrokCacheChunks(t *testing.T) {
+	token := os.Getenv("DIAG_SSO")
+	if token == "" {
+		t.Skip("no DIAG_SSO")
+	}
+	dir := os.Getenv("DIAG_CACHE")
+	if dir == "" {
+		dir = `f:\ai-gateway\vivid-ai\backend\_chunks`
+	}
+	os.MkdirAll(dir, 0o755)
+	logf, _ := os.Create(filepath.Join(dir, "_log.txt"))
+	defer logf.Close()
+	lg := func(f string, a ...any) {
+		fmt.Fprintf(logf, f+"\n", a...)
+		logf.Sync()
+	}
+
+	c := NewClient(os.Getenv("DIAG_PROXY"))
+	ctx, cancel := context.WithTimeout(context.Background(), 3000*time.Second)
+	defer cancel()
+	client, err := c.newTLSClient()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	hreq, _ := hnew(ctx, apiBase+"/", token)
+	hresp, err := client.Do(hreq)
+	if err != nil {
+		t.Fatal(err)
+	}
+	body, _ := io.ReadAll(hresp.Body)
+	hresp.Body.Close()
+	html := string(body)
+	os.WriteFile(filepath.Join(dir, "_home.html"), body, 0o644)
+
+	seen := map[string]bool{}
+	queue := dedupe(chunkPathRe.FindAllString(html, -1))
+	lg("home chunk refs: %d", len(queue))
+
+	saved := 0
+	for len(queue) > 0 && saved < 3000 {
+		p := queue[0]
+		queue = queue[1:]
+		norm := p
+		if !strings.HasPrefix(norm, "/_next/") {
+			norm = "/_next/" + strings.TrimPrefix(norm, "/")
+		}
+		if seen[norm] {
+			continue
+		}
+		seen[norm] = true
+		src, err := fetchChunk(ctx, client, norm)
+		if err != nil {
+			continue
+		}
+		saved++
+		base := norm[strings.LastIndex(norm, "/")+1:]
+		fn := fmt.Sprintf("%x_%s", sha1.Sum([]byte(norm)), base)
+		if len(fn) > 120 {
+			fn = fn[:120]
+		}
+		os.WriteFile(filepath.Join(dir, fn), []byte(src), 0o644)
+		for _, m := range innerChunkRe.FindAllString(src, -1) {
+			nn := "/_next/" + m
+			if !seen[nn] {
+				queue = append(queue, nn)
+			}
+		}
+		if saved%50 == 0 {
+			lg("saved=%d queue=%d", saved, len(queue))
+		}
+	}
+	lg("CRAWL DONE saved=%d queue_left=%d", saved, len(queue))
+	fmt.Printf("CRAWL DONE saved=%d\n", saved)
+}

backend/internal/provider/runway/video.go

@@ -70,37 +70,40 @@ func (c *Client) GenerateVideo(ctx context.Context, token, teamID, prompt, aspec
 		return nil, nil, errors.New("runway: failed to decode first-frame image")
 	}
 
-	// Use a single client (and therefore a single exit IP) for the whole flow.
-	// The web app runs upload, task-create, polling and download from one IP; a
-	// mid-flow IP switch (proxy for submit, local for the rest) is a strong
-	// bot/risk signal, so we route everything through the proxy-aware client.
-	apiClient, err := c.newTLSClient()
+	// Only the task-create (generate submit) egresses via the proxy; reference
+	// upload, dataset create, polling and download run on the local IP (matches
+	// the image pipeline).
+	submitClient, err := c.newTLSClient()
+	if err != nil {
+		return nil, nil, err
+	}
+	directClient, err := c.newDirectTLSClient()
 	if err != nil {
 		return nil, nil, err
 	}
 
 	filename := "frame_" + time.Now().UTC().Format("20060102_150405") + ".png"
-	previewUploadID, _, err := c.uploadFile(ctx, apiClient, token, teamID, filename, "DATASET_PREVIEW", frame)
+	previewUploadID, _, err := c.uploadFile(ctx, directClient, token, teamID, filename, "DATASET_PREVIEW", frame)
 	if err != nil {
 		return nil, nil, err
 	}
-	datasetUploadID, _, err := c.uploadFile(ctx, apiClient, token, teamID, filename, "DATASET", frame)
+	datasetUploadID, _, err := c.uploadFile(ctx, directClient, token, teamID, filename, "DATASET", frame)
 	if err != nil {
 		return nil, nil, err
 	}
 
-	assetID, imageURL, err := c.createDataset(ctx, apiClient, token, teamID, filename, datasetUploadID, previewUploadID, cfg.Width, cfg.Height)
+	assetID, imageURL, err := c.createDataset(ctx, directClient, token, teamID, filename, datasetUploadID, previewUploadID, cfg.Width, cfg.Height)
 	if err != nil {
 		return nil, nil, err
 	}
-	assetGroupID, _ := c.assetGroupID(ctx, apiClient, token, teamID) // best-effort
+	assetGroupID, _ := c.assetGroupID(ctx, directClient, token, teamID) // best-effort
 
-	taskID, err := c.createTask(ctx, apiClient, token, teamID, prompt, imageURL, assetID, assetGroupID, aspectRatio, seconds)
+	taskID, err := c.createTask(ctx, submitClient, token, teamID, prompt, imageURL, assetID, assetGroupID, aspectRatio, seconds)
 	if err != nil {
 		return nil, nil, err
 	}
 
-	artifactURL, err := c.pollTask(ctx, apiClient, token, teamID, taskID)
+	artifactURL, err := c.pollTask(ctx, directClient, token, teamID, taskID)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -113,7 +116,7 @@ func (c *Client) GenerateVideo(ctx context.Context, token, teamID, prompt, aspec
 	if !downloadResult {
 		return nil, meta, nil
 	}
-	data, err := c.download(ctx, apiClient, artifactURL)
+	data, err := c.download(ctx, directClient, artifactURL)
 	if err != nil {
 		return nil, nil, err
 	}