修复bug

This commit is contained in:
2026-07-03 18:12:46 +08:00
parent 53dc178804
commit f9e72c0168
5 changed files with 142 additions and 51 deletions
+29 -13
View File
@@ -1303,7 +1303,7 @@ const maxTempDeadAccounts = 3
// auth retry uses a FRESH token instead of replaying the stale one.
func (s *V1Service) runPoolWithFailover(ctx context.Context, eventID, pool string, active []model.TokenAccount, kind string,
attempt func(token model.TokenAccount) ([]byte, error),
classify func(error) (isAuth, isQuota, isTemporary bool),
classify func(error) (isAuth, isQuota, isTemporary, isDead bool),
refreshOnAuth func(tokenID string) (model.TokenAccount, bool),
tempFailover bool,
) ([]byte, error) {
@@ -1357,7 +1357,7 @@ func (s *V1Service) runPoolWithFailover(ctx context.Context, eventID, pool strin
// to the next account. The per-account concurrency gate is held by the caller.
func (s *V1Service) tryAccount(ctx context.Context, eventID, pool string, token model.TokenAccount, kind string,
attempt func(token model.TokenAccount) ([]byte, error),
classify func(error) (isAuth, isQuota, isTemporary bool),
classify func(error) (isAuth, isQuota, isTemporary, isDead bool),
refreshOnAuth func(tokenID string) (model.TokenAccount, bool),
tempFailover bool,
) ([]byte, error, bool, bool) {
@@ -1375,7 +1375,7 @@ func (s *V1Service) tryAccount(ctx context.Context, eventID, pool string, token
})
return data, nil, false, false
}
isAuth, isQuota, isTemp := classify(err)
isAuth, isQuota, isTemp, isDead := classify(err)
if isQuota {
s.markTokenFailure(ctx, pool, token, kind, false, true)
return nil, err, true, false
@@ -1392,6 +1392,10 @@ func (s *V1Service) tryAccount(ctx context.Context, eventID, pool string, token
s.markTokenFailure(ctx, pool, token, kind, true, false)
return nil, err, true, false
}
if isDead {
s.markTokenDead(ctx, pool, token, kind)
return nil, err, true, true
}
if isTemp {
if tempFailover {
// Ops policy (adobe): a temporary upstream error ("system under
@@ -1424,8 +1428,8 @@ func (s *V1Service) tryAccount(ctx context.Context, eventID, pool string, token
}
}
func adobeErrClass(e error) (bool, bool, bool) {
return errors.Is(e, adobe.ErrAuth), errors.Is(e, adobe.ErrQuotaExhausted), errors.Is(e, adobe.ErrTemporaryUpstream)
func adobeErrClass(e error) (bool, bool, bool, bool) {
return errors.Is(e, adobe.ErrAuth), errors.Is(e, adobe.ErrQuotaExhausted), errors.Is(e, adobe.ErrTemporaryUpstream), errors.Is(e, adobe.ErrDeadUpstream)
}
func (s *V1Service) generateAdobeImage(ctx context.Context, eventID string, modelItem *model.ModelConfig, in V1ImageRequest, aspectRatio, resolution string) ([]byte, error) {
@@ -2178,8 +2182,8 @@ func (s *V1Service) generateChatGPTImage(ctx context.Context, eventID string, mo
s.reconcileChatGPTQuota(ctx, token.ID, token.Value)
}
return data, genErr
}, func(e error) (bool, bool, bool) {
return errors.Is(e, chatgpt.ErrAuth), errors.Is(e, chatgpt.ErrQuotaExhausted), errors.Is(e, chatgpt.ErrTemporaryUpstream)
}, func(e error) (bool, bool, bool, bool) {
return errors.Is(e, chatgpt.ErrAuth), errors.Is(e, chatgpt.ErrQuotaExhausted), errors.Is(e, chatgpt.ErrTemporaryUpstream), false
}, nil, false) // chatgpt token IS the credential — no cookie to refresh
}
@@ -2305,8 +2309,8 @@ func (s *V1Service) generateLeonardoImage(ctx context.Context, eventID string, m
// sink to 限额 if below the floor (best-effort; never fails a done render).
s.reconcileLeonardoCredits(ctx, token.ID, token.Value)
return data, nil
}, func(e error) (bool, bool, bool) {
return errors.Is(e, leonardo.ErrAuth), errors.Is(e, leonardo.ErrQuotaExhausted), errors.Is(e, leonardo.ErrTemporaryUpstream)
}, func(e error) (bool, bool, bool, bool) {
return errors.Is(e, leonardo.ErrAuth), errors.Is(e, leonardo.ErrQuotaExhausted), errors.Is(e, leonardo.ErrTemporaryUpstream), false
}, nil, false)
}
@@ -2435,8 +2439,8 @@ func (s *V1Service) generateKreaImage(ctx context.Context, eventID string, model
}
data, _, genErr := s.krea.GenerateImage(ctx, cookie, in.Prompt, width, height, refs)
return data, genErr
}, func(e error) (bool, bool, bool) {
return errors.Is(e, krea.ErrAuth), errors.Is(e, krea.ErrQuotaExhausted), errors.Is(e, krea.ErrTemporaryUpstream)
}, func(e error) (bool, bool, bool, bool) {
return errors.Is(e, krea.ErrAuth), errors.Is(e, krea.ErrQuotaExhausted), errors.Is(e, krea.ErrTemporaryUpstream), false
}, nil, false)
}
@@ -2507,8 +2511,8 @@ func (s *V1Service) generateImagineImage(ctx context.Context, eventID string, mo
return nil, genErr
}
return data, nil
}, func(e error) (bool, bool, bool) {
return errors.Is(e, imagine.ErrAuth), errors.Is(e, imagine.ErrQuotaExhausted), errors.Is(e, imagine.ErrTemporaryUpstream)
}, func(e error) (bool, bool, bool, bool) {
return errors.Is(e, imagine.ErrAuth), errors.Is(e, imagine.ErrQuotaExhausted), errors.Is(e, imagine.ErrTemporaryUpstream), false
}, nil, false)
}
@@ -2955,6 +2959,18 @@ func (s *V1Service) markTokenFailure(ctx context.Context, pool string, token mod
_, _ = s.tokens.Update(ctx, pool, token.ID, patch)
}
// markTokenDead disables an account and marks it dead on a fatal upstream error
// (a non-overload temporary Adobe failure that ops policy treats as account death).
func (s *V1Service) markTokenDead(ctx context.Context, pool string, token model.TokenAccount, kind string) {
_, _ = s.tokens.Update(ctx, pool, token.ID, map[string]any{
"last_used_at": time.Now(),
"fail_total": gorm.Expr("fail_total + 1"),
"fails": gorm.Expr("fails + 1"),
"status": "disabled",
"dead": true,
})
}
// nextCursor returns the pool's current round-robin position and atomically
// advances it by one. Concurrent callers each get a distinct value, so parallel
// picks land on different accounts instead of racing onto the same one. The