adobe限流不封号
This commit is contained in:
@@ -636,7 +636,7 @@ func (c *Client) submitVideo(ctx context.Context, client tlsclient.HttpClient, t
|
|||||||
return respBody, "", ErrTemporaryUpstream
|
return respBody, "", ErrTemporaryUpstream
|
||||||
}
|
}
|
||||||
// "system under load" / timeout_error = adobe overload — treat as a temporary
|
// "system under load" / timeout_error = adobe overload — treat as a temporary
|
||||||
// error so the tempAsDead policy retires the account (same as the image path).
|
// error so the tempFailover policy moves to the next account (same as the image path).
|
||||||
if b := string(respBody); strings.Contains(b, "system under load") || strings.Contains(b, "timeout_error") {
|
if b := string(respBody); strings.Contains(b, "system under load") || strings.Contains(b, "timeout_error") {
|
||||||
return respBody, "", ErrTemporaryUpstream
|
return respBody, "", ErrTemporaryUpstream
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1262,10 +1262,10 @@ const maxSameAccountAttempts = 3
|
|||||||
// may run (grok tolerates 10, unlike the 1-per-account default elsewhere).
|
// may run (grok tolerates 10, unlike the 1-per-account default elsewhere).
|
||||||
const grokConcurrencyPerAccount = 10
|
const grokConcurrencyPerAccount = 10
|
||||||
|
|
||||||
// maxTempDeadAccounts caps how many accounts the "temporary error = dead account"
|
// maxTempDeadAccounts caps how many accounts the "temporary error = fail over"
|
||||||
// policy (tempAsDead, used by adobe) is allowed to mark dead + fail over before
|
// policy (tempFailover, used by adobe) may burn per request before giving up, so
|
||||||
// giving up, so an upstream-wide blip ("system under load") can't nuke the whole
|
// an upstream-wide blip ("system under load") can't fan a single request out
|
||||||
// pool. After this many accounts fail this way, the request fails.
|
// across the whole pool. After this many accounts fail this way, the request fails.
|
||||||
const maxTempDeadAccounts = 3
|
const maxTempDeadAccounts = 3
|
||||||
|
|
||||||
// runPoolWithFailover drives a generation across a round-robin-ordered account
|
// runPoolWithFailover drives a generation across a round-robin-ordered account
|
||||||
@@ -1277,13 +1277,14 @@ const maxTempDeadAccounts = 3
|
|||||||
// - 认证失效 auth → refresh the token from its cookie and retry ONCE with the
|
// - 认证失效 auth → refresh the token from its cookie and retry ONCE with the
|
||||||
// fresh token; if it still auth-fails (or there's nothing to refresh, e.g.
|
// fresh token; if it still auth-fails (or there's nothing to refresh, e.g.
|
||||||
// chatgpt's JWT IS the credential), mark the account and fail over.
|
// chatgpt's JWT IS the credential), mark the account and fail over.
|
||||||
// - 上游临时 temporary → behavior depends on tempAsDead:
|
// - 上游临时 temporary → behavior depends on tempFailover:
|
||||||
// • tempAsDead=false (default): retry the SAME account up to
|
// • tempFailover=false (default): retry the SAME account up to
|
||||||
// maxSameAccountAttempts times (not counted); if still failing, STOP
|
// maxSameAccountAttempts times (not counted); if still failing, STOP
|
||||||
// (no fan-out — an upstream-wide blip fails identically everywhere).
|
// (no fan-out — an upstream-wide blip fails identically everywhere).
|
||||||
// • tempAsDead=true (adobe): BAN the account (mark dead/disabled) and fail
|
// • tempFailover=true (adobe): fail over to the next account WITHOUT
|
||||||
// over to the next account, capped at maxTempDeadAccounts accounts so a
|
// penalizing this one (rate-limit/overload isn't the account's fault),
|
||||||
// pool-wide blip can't kill everything. Dead accounts don't auto-recover.
|
// capped at maxTempDeadAccounts accounts so a pool-wide blip can't fan
|
||||||
|
// a single request out across everything.
|
||||||
// - 参数错 / request-level (anything else) → return immediately, no retry, no
|
// - 参数错 / request-level (anything else) → return immediately, no retry, no
|
||||||
// account penalty (the account isn't at fault).
|
// account penalty (the account isn't at fault).
|
||||||
//
|
//
|
||||||
@@ -1296,7 +1297,7 @@ func (s *V1Service) runPoolWithFailover(ctx context.Context, eventID, pool strin
|
|||||||
attempt func(token model.TokenAccount) ([]byte, error),
|
attempt func(token model.TokenAccount) ([]byte, error),
|
||||||
classify func(error) (isAuth, isQuota, isTemporary bool),
|
classify func(error) (isAuth, isQuota, isTemporary bool),
|
||||||
refreshOnAuth func(tokenID string) (model.TokenAccount, bool),
|
refreshOnAuth func(tokenID string) (model.TokenAccount, bool),
|
||||||
tempAsDead bool,
|
tempFailover bool,
|
||||||
) ([]byte, error) {
|
) ([]byte, error) {
|
||||||
var lastErr error
|
var lastErr error
|
||||||
busy := 0
|
busy := 0
|
||||||
@@ -1310,16 +1311,16 @@ func (s *V1Service) runPoolWithFailover(ctx context.Context, eventID, pool strin
|
|||||||
// release via defer so a panic in tryAccount can't leak the 1-job slot.
|
// release via defer so a panic in tryAccount can't leak the 1-job slot.
|
||||||
data, err, failover, tempDead := func() ([]byte, error, bool, bool) {
|
data, err, failover, tempDead := func() ([]byte, error, bool, bool) {
|
||||||
defer s.acctRelease(ctx, token.ID, eventID)
|
defer s.acctRelease(ctx, token.ID, eventID)
|
||||||
return s.tryAccount(ctx, eventID, pool, token, kind, attempt, classify, refreshOnAuth, tempAsDead)
|
return s.tryAccount(ctx, eventID, pool, token, kind, attempt, classify, refreshOnAuth, tempFailover)
|
||||||
}()
|
}()
|
||||||
if err == nil {
|
if err == nil {
|
||||||
return data, nil
|
return data, nil
|
||||||
}
|
}
|
||||||
lastErr = err
|
lastErr = err
|
||||||
if tempDead {
|
if tempDead {
|
||||||
// temp-as-dead policy: this account was marked dead for a temporary
|
// temp-failover policy: this account hit a temporary upstream error.
|
||||||
// upstream error. Cap how many accounts that can burn before we stop,
|
// Cap how many accounts one request may burn before we stop, so an
|
||||||
// so an upstream-wide blip doesn't wipe the whole pool.
|
// upstream-wide blip doesn't fan out across the whole pool.
|
||||||
tempDeadCount++
|
tempDeadCount++
|
||||||
if tempDeadCount >= maxTempDeadAccounts {
|
if tempDeadCount >= maxTempDeadAccounts {
|
||||||
return nil, lastErr
|
return nil, lastErr
|
||||||
@@ -1350,7 +1351,7 @@ func (s *V1Service) tryAccount(ctx context.Context, eventID, pool string, token
|
|||||||
attempt func(token model.TokenAccount) ([]byte, error),
|
attempt func(token model.TokenAccount) ([]byte, error),
|
||||||
classify func(error) (isAuth, isQuota, isTemporary bool),
|
classify func(error) (isAuth, isQuota, isTemporary bool),
|
||||||
refreshOnAuth func(tokenID string) (model.TokenAccount, bool),
|
refreshOnAuth func(tokenID string) (model.TokenAccount, bool),
|
||||||
tempAsDead bool,
|
tempFailover bool,
|
||||||
) ([]byte, error, bool, bool) {
|
) ([]byte, error, bool, bool) {
|
||||||
_ = s.events.SetAccount(ctx, eventID, token.ID)
|
_ = s.events.SetAccount(ctx, eventID, token.ID)
|
||||||
_ = s.tokens.TouchLastUsed(ctx, token.ID)
|
_ = s.tokens.TouchLastUsed(ctx, token.ID)
|
||||||
@@ -1384,15 +1385,13 @@ func (s *V1Service) tryAccount(ctx context.Context, eventID, pool string, token
|
|||||||
return nil, err, true, false
|
return nil, err, true, false
|
||||||
}
|
}
|
||||||
if isTemp {
|
if isTemp {
|
||||||
if tempAsDead {
|
if tempFailover {
|
||||||
// Ops policy (adobe): a temporary upstream error ("system under
|
// Ops policy (adobe): a temporary upstream error ("system under
|
||||||
// load" etc.) BANS this account — mark it dead/disabled and fail
|
// load" / rate-limit) is NOT the account's fault — record the
|
||||||
// over to the next account. The pool driver caps how many accounts
|
// failure but keep the account active, and fail over to the next
|
||||||
// this is allowed to burn per request (maxTempDeadAccounts). Note:
|
// account. The pool driver caps how many accounts one request may
|
||||||
// dead accounts do NOT auto-recover — they need a manual re-enable.
|
// burn this way (maxTempDeadAccounts).
|
||||||
_, _ = s.tokens.Update(ctx, pool, token.ID, map[string]any{
|
_, _ = s.tokens.Update(ctx, pool, token.ID, map[string]any{
|
||||||
"status": "disabled",
|
|
||||||
"dead": true,
|
|
||||||
"last_used_at": time.Now(),
|
"last_used_at": time.Now(),
|
||||||
"fail_total": gorm.Expr("fail_total + 1"),
|
"fail_total": gorm.Expr("fail_total + 1"),
|
||||||
"fails": gorm.Expr("fails + 1"),
|
"fails": gorm.Expr("fails + 1"),
|
||||||
@@ -1455,9 +1454,9 @@ func (s *V1Service) generateAdobeImage(ctx context.Context, eventID string, mode
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Round-robin order. Adobe uses tempAsDead=true: a temporary upstream error
|
// Round-robin order. Adobe uses tempFailover=true: a temporary upstream error
|
||||||
// ("system under load") marks the account dead (like a 401) and fails over to
|
// ("system under load") fails over to the next account without penalizing the
|
||||||
// the next account, capped at maxTempDeadAccounts; auth/quota also fail over
|
// current one, capped at maxTempDeadAccounts; auth/quota also fail over
|
||||||
// (see runPoolWithFailover).
|
// (see runPoolWithFailover).
|
||||||
return s.runPoolWithFailover(ctx, eventID, "adobe", active, "image", func(token model.TokenAccount) ([]byte, error) {
|
return s.runPoolWithFailover(ctx, eventID, "adobe", active, "image", func(token model.TokenAccount) ([]byte, error) {
|
||||||
var blobIDs []string
|
var blobIDs []string
|
||||||
@@ -1518,8 +1517,8 @@ func (s *V1Service) generateAdobeVideo(ctx context.Context, eventID string, mode
|
|||||||
referenceMode := defaultString(strings.TrimSpace(modelItem.ReferenceMode), "frame")
|
referenceMode := defaultString(strings.TrimSpace(modelItem.ReferenceMode), "frame")
|
||||||
|
|
||||||
// Round-robin order; same-account retry on transient errors, fail over to the
|
// Round-robin order; same-account retry on transient errors, fail over to the
|
||||||
// next account on auth/quota; temporary upstream errors mark the account dead
|
// next account on auth/quota; temporary upstream errors fail over too without
|
||||||
// and fail over too (tempAsDead, capped at maxTempDeadAccounts). videoURL is
|
// penalizing the account (tempFailover, capped at maxTempDeadAccounts). videoURL is
|
||||||
// captured from the successful attempt's meta (the upstream presigned URL).
|
// captured from the successful attempt's meta (the upstream presigned URL).
|
||||||
var videoURL string
|
var videoURL string
|
||||||
data, err := s.runPoolWithFailover(ctx, eventID, "adobe", active, "video", func(token model.TokenAccount) ([]byte, error) {
|
data, err := s.runPoolWithFailover(ctx, eventID, "adobe", active, "video", func(token model.TokenAccount) ([]byte, error) {
|
||||||
|
|||||||
Reference in New Issue
Block a user