修复bug

This commit is contained in:
2026-07-03 18:12:46 +08:00
parent 53dc178804
commit f9e72c0168
5 changed files with 142 additions and 51 deletions
+12 -6
View File
@@ -131,12 +131,14 @@ func (m *MaintenanceService) tick(ctx context.Context) {
log.Printf("maintenance: roll_reset: %v", err)
}
// 1b. Runway/grok tokens have no refresh — once the reset marker passes, mark
// them dead directly instead of letting them 401 on next use. Runway's
// marker is the JWT expiry; grok's is the credits reset (grok sso can't be
// renewed either — 失效就失效 — so a purchased short-lived account that has
// lapsed by its reset time is treated as dead rather than re-scheduled).
for _, pool := range []string{"runway", "grok"} {
// 1b. Runway tokens have no refresh — once the JWT expiry marker passes the
// token can only 401, so flip it to disabled+dead proactively instead of
// leaving a doomed account "active". Grok is intentionally NOT swept here:
// its reset marker is billingPeriodEnd (a credits-renewal date), not a
// death deadline — a grok sso keeps working past billingPeriodEnd, so
// expiring on it kills live accounts. Grok death is caught for real by the
// import-time FetchSession check and by marking dead on a 401 at use.
for _, pool := range []string{"runway"} {
if n, err := m.tokens.ExpireByReset(ctx, pool); err != nil {
log.Printf("maintenance: expire_%s: %v", pool, err)
} else if n > 0 {
@@ -155,6 +157,10 @@ func (m *MaintenanceService) tick(ctx context.Context) {
// always-active account (never went 限额) would otherwise read 0 / 402
// after each reset. Self-guarded + background; no-op once all are done.
m.tokenSvc.ActivateKreaDue(ctx)
// 1e. Re-validate grok accounts: an empty /rest/subscriptions (or 401)
// means the membership lapsed → disable+dead; otherwise re-sync the
// credits balance and 恢复时间 (from the credits' weekly reset).
m.tokenSvc.RefreshGrokLiveness(ctx)
}
// 2. Auto-renew Adobe cookies whose refresh interval has elapsed.
+64 -5
View File
@@ -944,6 +944,66 @@ func (s *TokenService) checkPendingGrok(tokenID, ssoToken string) {
s.finishPending(ctx, "grok", tokenID, "active", false, quotaMeta)
}
// RefreshGrokLiveness re-validates every live grok account each maintenance tick.
// Grok sso can't be renewed and has no reset-based death deadline (billingPeriodEnd
// is only a credits-renewal date — the sso keeps working past it), so liveness is
// probed directly: GET /rest/subscriptions. No ACTIVE entry — a lapsed membership
// flips to SUBSCRIPTION_STATUS_INACTIVE (empty array / 401 also count) — means the
// paid membership is gone → the account is disabled+dead. Otherwise the credits
// balance is re-synced and 恢复时间 is refreshed from the credits' own weekly
// reset (NOT the subscription's billing-period end).
func (s *TokenService) RefreshGrokLiveness(ctx context.Context) {
if s.grok == nil {
return
}
items, err := s.tokens.List(ctx)
if err != nil {
return
}
s.applyProxy(ctx)
for i := range items {
it := items[i]
if it.Pool != "grok" || it.Dead || it.Status == "disabled" || strings.TrimSpace(it.Value) == "" {
continue
}
sub, serr := s.grok.FetchSubscription(ctx, it.Value)
if serr != nil {
if errors.Is(serr, grok.ErrAuth) {
_, _ = s.tokens.Update(ctx, "grok", it.ID, map[string]any{"status": "disabled", "dead": true})
}
continue // transient upstream error → leave as-is, retry next tick
}
if sub == nil || !sub.Member {
// no ACTIVE subscription (INACTIVE / empty) → membership lapsed → dead.
_, _ = s.tokens.Update(ctx, "grok", it.ID, map[string]any{"status": "disabled", "dead": true})
continue
}
data, derr := s.grok.FetchCreditsBalance(ctx, it.Value)
if derr != nil {
if errors.Is(derr, grok.ErrAuth) {
_, _ = s.tokens.Update(ctx, "grok", it.ID, map[string]any{"status": "disabled", "dead": true})
}
continue
}
meta := cloneJSONMap(it.Meta)
meta["cached_quota_at"] = int(time.Now().Unix())
if rem, ok := data["remaining"].(int); ok {
meta["cached_quota_remaining"] = rem
}
if used, ok := data["used"].(int); ok {
meta["cached_quota_used"] = used
}
if total, ok := data["total"].(int); ok {
meta["cached_quota_total"] = total
}
patch := map[string]any{"meta": meta}
if reset := strings.TrimSpace(stringValue(data["reset_after"])); reset != "" {
patch["cached_quota_reset_after"] = reset
}
_, _ = s.tokens.Update(ctx, "grok", it.ID, patch)
}
}
// ImportCustomAccount adds an upstream as a custom account: base_url + key, the
// csv list of model ids it serves (empty = all), plus optional weight and
// per-account concurrency. No probe — the account goes active immediately and is
@@ -1446,11 +1506,10 @@ func (s *TokenService) Quota(ctx context.Context, pool, id string) (map[string]a
meta["cached_quota_total"] = total
}
patch["meta"] = meta
// Recovery time is the death deadline: the maintenance sweep expires a grok
// account once this marker passes (grok sso can't be renewed). Only stamp it
// when still unset (import couldn't resolve it) — never move it forward on a
// later refresh, so an admin opening 账号管理 can't push the death time out.
if reset := strings.TrimSpace(stringValue(data["reset_after"])); reset != "" && strings.TrimSpace(item.CachedQuotaResetAfter) == "" {
// Recovery time is the credits' weekly reset (when the grant refills) —
// purely informational, NOT a death deadline (liveness is judged by the
// subscriptions sweep / real 401s), so it's safe to refresh every time.
if reset := strings.TrimSpace(stringValue(data["reset_after"])); reset != "" {
patch["cached_quota_reset_after"] = reset
item.CachedQuotaResetAfter = reset
}
+29 -13
View File
@@ -1303,7 +1303,7 @@ const maxTempDeadAccounts = 3
// auth retry uses a FRESH token instead of replaying the stale one.
func (s *V1Service) runPoolWithFailover(ctx context.Context, eventID, pool string, active []model.TokenAccount, kind string,
attempt func(token model.TokenAccount) ([]byte, error),
classify func(error) (isAuth, isQuota, isTemporary bool),
classify func(error) (isAuth, isQuota, isTemporary, isDead bool),
refreshOnAuth func(tokenID string) (model.TokenAccount, bool),
tempFailover bool,
) ([]byte, error) {
@@ -1357,7 +1357,7 @@ func (s *V1Service) runPoolWithFailover(ctx context.Context, eventID, pool strin
// to the next account. The per-account concurrency gate is held by the caller.
func (s *V1Service) tryAccount(ctx context.Context, eventID, pool string, token model.TokenAccount, kind string,
attempt func(token model.TokenAccount) ([]byte, error),
classify func(error) (isAuth, isQuota, isTemporary bool),
classify func(error) (isAuth, isQuota, isTemporary, isDead bool),
refreshOnAuth func(tokenID string) (model.TokenAccount, bool),
tempFailover bool,
) ([]byte, error, bool, bool) {
@@ -1375,7 +1375,7 @@ func (s *V1Service) tryAccount(ctx context.Context, eventID, pool string, token
})
return data, nil, false, false
}
isAuth, isQuota, isTemp := classify(err)
isAuth, isQuota, isTemp, isDead := classify(err)
if isQuota {
s.markTokenFailure(ctx, pool, token, kind, false, true)
return nil, err, true, false
@@ -1392,6 +1392,10 @@ func (s *V1Service) tryAccount(ctx context.Context, eventID, pool string, token
s.markTokenFailure(ctx, pool, token, kind, true, false)
return nil, err, true, false
}
if isDead {
s.markTokenDead(ctx, pool, token, kind)
return nil, err, true, true
}
if isTemp {
if tempFailover {
// Ops policy (adobe): a temporary upstream error ("system under
@@ -1424,8 +1428,8 @@ func (s *V1Service) tryAccount(ctx context.Context, eventID, pool string, token
}
}
func adobeErrClass(e error) (bool, bool, bool) {
return errors.Is(e, adobe.ErrAuth), errors.Is(e, adobe.ErrQuotaExhausted), errors.Is(e, adobe.ErrTemporaryUpstream)
func adobeErrClass(e error) (bool, bool, bool, bool) {
return errors.Is(e, adobe.ErrAuth), errors.Is(e, adobe.ErrQuotaExhausted), errors.Is(e, adobe.ErrTemporaryUpstream), errors.Is(e, adobe.ErrDeadUpstream)
}
func (s *V1Service) generateAdobeImage(ctx context.Context, eventID string, modelItem *model.ModelConfig, in V1ImageRequest, aspectRatio, resolution string) ([]byte, error) {
@@ -2178,8 +2182,8 @@ func (s *V1Service) generateChatGPTImage(ctx context.Context, eventID string, mo
s.reconcileChatGPTQuota(ctx, token.ID, token.Value)
}
return data, genErr
}, func(e error) (bool, bool, bool) {
return errors.Is(e, chatgpt.ErrAuth), errors.Is(e, chatgpt.ErrQuotaExhausted), errors.Is(e, chatgpt.ErrTemporaryUpstream)
}, func(e error) (bool, bool, bool, bool) {
return errors.Is(e, chatgpt.ErrAuth), errors.Is(e, chatgpt.ErrQuotaExhausted), errors.Is(e, chatgpt.ErrTemporaryUpstream), false
}, nil, false) // chatgpt token IS the credential — no cookie to refresh
}
@@ -2305,8 +2309,8 @@ func (s *V1Service) generateLeonardoImage(ctx context.Context, eventID string, m
// sink to 限额 if below the floor (best-effort; never fails a done render).
s.reconcileLeonardoCredits(ctx, token.ID, token.Value)
return data, nil
}, func(e error) (bool, bool, bool) {
return errors.Is(e, leonardo.ErrAuth), errors.Is(e, leonardo.ErrQuotaExhausted), errors.Is(e, leonardo.ErrTemporaryUpstream)
}, func(e error) (bool, bool, bool, bool) {
return errors.Is(e, leonardo.ErrAuth), errors.Is(e, leonardo.ErrQuotaExhausted), errors.Is(e, leonardo.ErrTemporaryUpstream), false
}, nil, false)
}
@@ -2435,8 +2439,8 @@ func (s *V1Service) generateKreaImage(ctx context.Context, eventID string, model
}
data, _, genErr := s.krea.GenerateImage(ctx, cookie, in.Prompt, width, height, refs)
return data, genErr
}, func(e error) (bool, bool, bool) {
return errors.Is(e, krea.ErrAuth), errors.Is(e, krea.ErrQuotaExhausted), errors.Is(e, krea.ErrTemporaryUpstream)
}, func(e error) (bool, bool, bool, bool) {
return errors.Is(e, krea.ErrAuth), errors.Is(e, krea.ErrQuotaExhausted), errors.Is(e, krea.ErrTemporaryUpstream), false
}, nil, false)
}
@@ -2507,8 +2511,8 @@ func (s *V1Service) generateImagineImage(ctx context.Context, eventID string, mo
return nil, genErr
}
return data, nil
}, func(e error) (bool, bool, bool) {
return errors.Is(e, imagine.ErrAuth), errors.Is(e, imagine.ErrQuotaExhausted), errors.Is(e, imagine.ErrTemporaryUpstream)
}, func(e error) (bool, bool, bool, bool) {
return errors.Is(e, imagine.ErrAuth), errors.Is(e, imagine.ErrQuotaExhausted), errors.Is(e, imagine.ErrTemporaryUpstream), false
}, nil, false)
}
@@ -2955,6 +2959,18 @@ func (s *V1Service) markTokenFailure(ctx context.Context, pool string, token mod
_, _ = s.tokens.Update(ctx, pool, token.ID, patch)
}
// markTokenDead disables an account and marks it dead on a fatal upstream error
// (a non-overload temporary Adobe failure that ops policy treats as account death).
func (s *V1Service) markTokenDead(ctx context.Context, pool string, token model.TokenAccount, kind string) {
_, _ = s.tokens.Update(ctx, pool, token.ID, map[string]any{
"last_used_at": time.Now(),
"fail_total": gorm.Expr("fail_total + 1"),
"fails": gorm.Expr("fails + 1"),
"status": "disabled",
"dead": true,
})
}
// nextCursor returns the pool's current round-robin position and atomically
// advances it by one. Concurrent callers each get a distinct value, so parallel
// picks land on different accounts instead of racing onto the same one. The