优化408

backend/internal/provider/adobe/client.go

@@ -34,6 +34,7 @@ const (
 
 var (
 	ErrAuth              = errors.New("adobe auth failed")
+	ErrAuthPermanent     = errors.New("adobe auth permanently failed")
 	ErrQuotaExhausted    = errors.New("adobe quota exhausted")
 	ErrTemporaryUpstream = errors.New("adobe upstream temporary error")
 	ErrDeadUpstream      = errors.New("adobe upstream fatal error")
@@ -52,6 +53,13 @@ func isContentRejection(status int, body string) bool {
 	return status == 451 && strings.Contains(body, "unsafe")
 }
 
+func isPermanentAuthError(header string, body []byte) bool {
+	return strings.EqualFold(header, "user_not_entitled") ||
+		strings.EqualFold(header, "access_error") ||
+		strings.Contains(string(body), "user_not_entitled") ||
+		strings.Contains(string(body), "access_error")
+}
+
 var profileURLs = []string{
 	"https://ims-na1.adobelogin.com/ims/profile/v1",
 	"https://adobeid-na1.services.adobe.com/ims/profile/v1",
@@ -85,7 +93,7 @@ func (c *Client) ExchangeCookie(ctx context.Context, cookie string) (*CookieExch
 // uploadMaxRetries is how many extra in-place attempts a transient upload
 // failure (transport error / timeout, 429/451/5xx) gets on a fresh connection
 // before the error is surfaced.
-const uploadMaxRetries = 5
+const uploadMaxRetries = 3
 
 // UploadImage stores a reference image and returns its blob id. Transient
 // failures are retried in place (uploadMaxRetries times); the final error keeps
@@ -95,6 +103,13 @@ func (c *Client) UploadImage(ctx context.Context, token string, content []byte,
 	// Reference-image upload runs on the local IP (not the proxy).
 	body, err, retryable := c.uploadImageOnce(ctx, token, content, contentType, engine)
 	for attempt := 0; err != nil && retryable && attempt < uploadMaxRetries && ctx.Err() == nil; attempt++ {
+		if wait := time.Duration(attempt+1) * time.Second; wait > 0 {
+			select {
+			case <-ctx.Done():
+				return "", ctx.Err()
+			case <-time.After(wait):
+			}
+		}
 		body, err, retryable = c.uploadImageOnce(ctx, token, content, contentType, engine)
 	}
 	if err != nil {
@@ -122,7 +137,7 @@ func (c *Client) UploadImage(ctx context.Context, token string, content []byte,
 // body plus whether a failure is retryable (transport error / 429/451/5xx).
 // Auth failures (401/403) and other non-200s are not retryable.
 func (c *Client) uploadImageOnce(ctx context.Context, token string, content []byte, contentType, engine string) ([]byte, error, bool) {
-	sess, err := c.newDirectTLSClient()
+	sess, err := c.newUploadTLSClient()
 	if err != nil {
 		return nil, err, false
 	}
@@ -164,6 +179,9 @@ func (c *Client) uploadImageOnce(ctx context.Context, token string, content []by
 		return nil, err, true
 	}
 	if resp.StatusCode == 401 || resp.StatusCode == 403 {
+		if isPermanentAuthError(resp.Header.Get("x-access-error"), body) {
+			return nil, fmt.Errorf("%w (upload %d %s: %s)", ErrAuthPermanent, resp.StatusCode, resp.Header.Get("x-access-error"), clip(body, 300)), false
+		}
 		return nil, fmt.Errorf("%w (upload %d %s: %s)", ErrAuth, resp.StatusCode, resp.Header.Get("x-access-error"), clip(body, 300)), false
 	}
 	if resp.StatusCode == 429 || resp.StatusCode == 451 || resp.StatusCode >= 500 {
@@ -177,15 +195,19 @@ func (c *Client) uploadImageOnce(ctx context.Context, token string, content []by
 
 func (c *Client) GenerateImage(ctx context.Context, token, modelID, prompt, aspectRatio, resolution string, blobIDs []string, downloadResult bool) ([]byte, map[string]any, error) {
 	// Only the generate submit goes through the proxy; polling + download run on
-	// the local IP.
+	// the local IP. If the proxy connection fails, retry once on the local IP.
 	submitSess, err := c.newTLSClient()
 	if err != nil {
 		return nil, nil, err
 	}
+	directSess, directErr := c.newDirectTLSClient()
 	pollSess, err := c.newDirectTLSClient()
 	if err != nil {
 		return nil, nil, err
 	}
+	if directErr != nil {
+		directSess = pollSess
+	}
 
 	var lastBody []byte
 	var lastErr error
@@ -201,6 +223,9 @@ func (c *Client) GenerateImage(ctx context.Context, token, modelID, prompt, aspe
 	}
 	for _, payload := range candidates {
 		respBody, pollURL, err := c.submitImage(ctx, submitSess, token, prompt, endpoint, payload)
+		if errors.Is(err, ErrTemporaryUpstream) {
+			respBody, pollURL, err = c.submitImage(ctx, directSess, token, prompt, endpoint, payload)
+		}
 		if err == nil {
 			meta, data, pollErr := c.pollImage(ctx, pollSess, token, pollURL, downloadResult)
 			if pollErr != nil {
@@ -210,7 +235,7 @@ func (c *Client) GenerateImage(ctx context.Context, token, modelID, prompt, aspe
 		}
 		lastBody = respBody
 		lastErr = err
-		if errors.Is(err, ErrAuth) || errors.Is(err, ErrQuotaExhausted) || errors.Is(err, ErrContentRejected) {
+		if errors.Is(err, ErrAuth) || errors.Is(err, ErrAuthPermanent) || errors.Is(err, ErrQuotaExhausted) || errors.Is(err, ErrContentRejected) {
 			return nil, nil, err
 		}
 	}
@@ -240,10 +265,14 @@ func (c *Client) GenerateVideo(ctx context.Context, token, engine, prompt, aspec
 	if err != nil {
 		return nil, nil, err
 	}
+	directSess, directErr := c.newDirectTLSClient()
 	pollSess, err := c.newDirectTLSClient()
 	if err != nil {
 		return nil, nil, err
 	}
+	if directErr != nil {
+		directSess = pollSess
+	}
 
 	payload := BuildVideoPayload(engine, prompt, aspectRatio, durationSeconds, resolution, referenceMode, upstreamModel, blobIDs)
 	endpoint := videoSubmitURL
@@ -251,6 +280,9 @@ func (c *Client) GenerateVideo(ctx context.Context, token, engine, prompt, aspec
 		endpoint = fireflyVideoSubmitURL
 	}
 	respBody, pollURL, err := c.submitVideo(ctx, submitSess, token, endpoint, payload)
+	if errors.Is(err, ErrTemporaryUpstream) {
+		respBody, pollURL, err = c.submitVideo(ctx, directSess, token, endpoint, payload)
+	}
 	if err != nil {
 		return nil, nil, err
 	}
@@ -397,6 +429,9 @@ func (c *Client) FetchCreditsBalance(ctx context.Context, token string) (map[str
 		return nil, err
 	}
 	if resp.StatusCode == 401 {
+		if isPermanentAuthError(resp.Header.Get("x-access-error"), body) {
+			return nil, ErrAuthPermanent
+		}
 		return nil, ErrAuth
 	}
 	if resp.StatusCode != 200 {
@@ -496,6 +531,9 @@ func (c *Client) submitImage(ctx context.Context, sess *tlsSession, token, promp
 		if strings.EqualFold(resp.Header.Get("x-access-error"), "taste_exhausted") {
 			return respBody, "", ErrQuotaExhausted
 		}
+		if isPermanentAuthError(resp.Header.Get("x-access-error"), respBody) {
+			return respBody, "", fmt.Errorf("%w (submit %d %s: %s)", ErrAuthPermanent, resp.StatusCode, resp.Header.Get("x-access-error"), clip(respBody, 300))
+		}
 		return respBody, "", fmt.Errorf("%w (submit %d %s: %s)", ErrAuth, resp.StatusCode, resp.Header.Get("x-access-error"), clip(respBody, 300))
 	}
 	// "system under load" / timeout_error = adobe rate-limit/overload (can come on a
@@ -509,6 +547,9 @@ func (c *Client) submitImage(ctx context.Context, sess *tlsSession, token, promp
 	if resp.StatusCode == 429 || resp.StatusCode == 451 || resp.StatusCode >= 500 {
 		return respBody, "", ErrDeadUpstream
 	}
+	if strings.Contains(string(respBody), "access_error") {
+		return respBody, "", fmt.Errorf("%w (submit %d: %s)", ErrAuthPermanent, resp.StatusCode, clip(respBody, 300))
+	}
 	if resp.StatusCode != 200 {
 		return respBody, "", errors.New("submit rejected")
 	}
@@ -679,8 +720,9 @@ func (c *Client) submitVideo(ctx context.Context, sess *tlsSession, token, endpo
 		if strings.EqualFold(resp.Header.Get("x-access-error"), "taste_exhausted") {
 			return respBody, "", ErrQuotaExhausted
 		}
-		// Surface Adobe's response body — "adobe auth failed" alone hides whether
+		if isPermanentAuthError(resp.Header.Get("x-access-error"), respBody) {
-		// it's a bad token, a missing scope, or a WAF/fingerprint block.
+			return respBody, "", fmt.Errorf("%w (%d %s: %s)", ErrAuthPermanent, resp.StatusCode, resp.Header.Get("x-access-error"), clip(respBody, 300))
+		}
 		return respBody, "", fmt.Errorf("%w (%d %s: %s)", ErrAuth, resp.StatusCode, resp.Header.Get("x-access-error"), clip(respBody, 300))
 	}
 	if isContentRejection(resp.StatusCode, string(respBody)) {
@@ -694,6 +736,9 @@ func (c *Client) submitVideo(ctx context.Context, sess *tlsSession, token, endpo
 	if b := string(respBody); strings.Contains(b, "system under load") || strings.Contains(b, "timeout_error") {
 		return respBody, "", ErrTemporaryUpstream
 	}
+	if strings.Contains(string(respBody), "access_error") {
+		return respBody, "", fmt.Errorf("%w (%d: %s)", ErrAuthPermanent, resp.StatusCode, clip(respBody, 300))
+	}
 	if resp.StatusCode != 200 {
 		return respBody, "", fmt.Errorf("video submit rejected: %d %s", resp.StatusCode, clip(respBody, 300))
 	}
@@ -755,6 +800,9 @@ func (c *Client) pollVideo(ctx context.Context, sess *tlsSession, token, pollURL
 			return nil, nil, readErr
 		}
 		if resp.StatusCode == 401 || resp.StatusCode == 403 {
+			if isPermanentAuthError(resp.Header.Get("x-access-error"), body) {
+				return nil, nil, fmt.Errorf("%w (%d %s: %s)", ErrAuthPermanent, resp.StatusCode, resp.Header.Get("x-access-error"), clip(body, 300))
+			}
 			return nil, nil, fmt.Errorf("%w (%d %s: %s)", ErrAuth, resp.StatusCode, resp.Header.Get("x-access-error"), clip(body, 300))
 		}
 		if b := string(body); strings.Contains(b, "system under load") || strings.Contains(b, "timeout_error") {
@@ -923,16 +971,20 @@ type tlsSession struct {
 // image-generation submit goes through the proxy; reference-image upload,
 // polling and result download run on the local IP.
 func (c *Client) newTLSClient() (*tlsSession, error) {
-	return c.newTLSSession(randomFingerprint(), true)
+	return c.newTLSSession(randomFingerprint(), true, 120)
 }
 
 func (c *Client) newDirectTLSClient() (*tlsSession, error) {
-	return c.newTLSSession(randomFingerprint(), false)
+	return c.newTLSSession(randomFingerprint(), false, 60)
 }
 
-func (c *Client) newTLSSession(fp fingerprint, useProxy bool) (*tlsSession, error) {
+func (c *Client) newUploadTLSClient() (*tlsSession, error) {
+	return c.newTLSSession(randomFingerprint(), false, 180)
+}
+
+func (c *Client) newTLSSession(fp fingerprint, useProxy bool, timeout int) (*tlsSession, error) {
 	options := []tlsclient.HttpClientOption{
-		tlsclient.WithTimeoutSeconds(60),
+		tlsclient.WithTimeoutSeconds(timeout),
 		tlsclient.WithClientProfile(fp.profile),
 		tlsclient.WithNotFollowRedirects(),
 		tlsclient.WithRandomTLSExtensionOrder(),

backend/internal/provider/adobe/payloads.go

@@ -112,10 +112,6 @@ func BuildImagePayloadCandidates(modelID, prompt, aspectRatio, outputResolution
 
 func buildGPTImagePayloads(spec modelSpec, prompt, ratio, resolution string, blobIDs []string) []map[string]any {
 	size := getSize(gptImageSize, resolution, ratio, "1:1")
-	// Mirrors the captured working gpt-image request shape: modelSpecificPayload.size,
-	// generationSettings.detailLevel 3, and NO top-level size / outputResolution
-	// (sending those got 403). Keeps the chosen size via modelSpecificPayload.size
-	// ("WxH") rather than "auto".
 	base := map[string]any{
 		"modelId":              spec.UpstreamModelID,
 		"modelVersion":         spec.UpstreamModelVersion,
@@ -124,8 +120,9 @@ func buildGPTImagePayloads(spec modelSpec, prompt, ratio, resolution string, blo
 		"seeds":                []int{int(time.Now().Unix()) % 999999},
 		"output":               map[string]any{"storeInputs": true},
 		"referenceBlobs":       []any{},
+		"size":                 map[string]any{"width": size[0], "height": size[1]},
 		"generationMetadata":   map[string]any{"module": "text2image", "submodule": "ff-image-generate"},
-		"modelSpecificPayload": map[string]any{"size": sizeString(size)},
+		"modelSpecificPayload": map[string]any{},
 		"generationSettings":   map[string]any{"detailLevel": 3},
 	}
 	if len(blobIDs) == 0 {
@@ -211,10 +208,6 @@ func getSize(table map[string]map[string][2]int, resolution, ratio, fallbackRati
 	return size
 }
 
-func sizeString(size [2]int) string {
-	return itoa(size[0]) + "x" + itoa(size[1])
-}
-
 func blobRefs(ids []string, usage string) []any {
 	out := make([]any, 0, len(ids))
 	for _, id := range ids {

backend/internal/service/v1.go

@@ -1550,6 +1550,10 @@ func (s *V1Service) tryAccount(ctx context.Context, eventID, pool string, token
 			})
 			return data, nil, false, false
 		}
+		if errors.Is(err, adobe.ErrAuthPermanent) {
+			s.markTokenDead(ctx, pool, token, kind)
+			return nil, err, true, false
+		}
 		isAuth, isQuota, isTemp, isDead := classify(err)
 		if isQuota {
 			s.markTokenFailure(ctx, pool, token, kind, false, true)