修复bug
This commit is contained in:
@@ -131,12 +131,14 @@ func (m *MaintenanceService) tick(ctx context.Context) {
|
||||
log.Printf("maintenance: roll_reset: %v", err)
|
||||
}
|
||||
|
||||
// 1b. Runway/grok tokens have no refresh — once the reset marker passes, mark
|
||||
// them dead directly instead of letting them 401 on next use. Runway's
|
||||
// marker is the JWT expiry; grok's is the credits reset (grok sso can't be
|
||||
// renewed either — 失效就失效 — so a purchased short-lived account that has
|
||||
// lapsed by its reset time is treated as dead rather than re-scheduled).
|
||||
for _, pool := range []string{"runway", "grok"} {
|
||||
// 1b. Runway tokens have no refresh — once the JWT expiry marker passes the
|
||||
// token can only 401, so flip it to disabled+dead proactively instead of
|
||||
// leaving a doomed account "active". Grok is intentionally NOT swept here:
|
||||
// its reset marker is billingPeriodEnd (a credits-renewal date), not a
|
||||
// death deadline — a grok sso keeps working past billingPeriodEnd, so
|
||||
// expiring on it kills live accounts. Grok death is caught for real by the
|
||||
// import-time FetchSession check and by marking dead on a 401 at use.
|
||||
for _, pool := range []string{"runway"} {
|
||||
if n, err := m.tokens.ExpireByReset(ctx, pool); err != nil {
|
||||
log.Printf("maintenance: expire_%s: %v", pool, err)
|
||||
} else if n > 0 {
|
||||
@@ -155,6 +157,10 @@ func (m *MaintenanceService) tick(ctx context.Context) {
|
||||
// always-active account (never went 限额) would otherwise read 0 / 402
|
||||
// after each reset. Self-guarded + background; no-op once all are done.
|
||||
m.tokenSvc.ActivateKreaDue(ctx)
|
||||
// 1e. Re-validate grok accounts: an empty /rest/subscriptions (or 401)
|
||||
// means the membership lapsed → disable+dead; otherwise re-sync the
|
||||
// credits balance and 恢复时间 (from the credits' weekly reset).
|
||||
m.tokenSvc.RefreshGrokLiveness(ctx)
|
||||
}
|
||||
|
||||
// 2. Auto-renew Adobe cookies whose refresh interval has elapsed.
|
||||
|
||||
@@ -944,6 +944,66 @@ func (s *TokenService) checkPendingGrok(tokenID, ssoToken string) {
|
||||
s.finishPending(ctx, "grok", tokenID, "active", false, quotaMeta)
|
||||
}
|
||||
|
||||
// RefreshGrokLiveness re-validates every live grok account each maintenance tick.
|
||||
// Grok sso can't be renewed and has no reset-based death deadline (billingPeriodEnd
|
||||
// is only a credits-renewal date — the sso keeps working past it), so liveness is
|
||||
// probed directly: GET /rest/subscriptions. No ACTIVE entry — a lapsed membership
|
||||
// flips to SUBSCRIPTION_STATUS_INACTIVE (empty array / 401 also count) — means the
|
||||
// paid membership is gone → the account is disabled+dead. Otherwise the credits
|
||||
// balance is re-synced and 恢复时间 is refreshed from the credits' own weekly
|
||||
// reset (NOT the subscription's billing-period end).
|
||||
func (s *TokenService) RefreshGrokLiveness(ctx context.Context) {
|
||||
if s.grok == nil {
|
||||
return
|
||||
}
|
||||
items, err := s.tokens.List(ctx)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
s.applyProxy(ctx)
|
||||
for i := range items {
|
||||
it := items[i]
|
||||
if it.Pool != "grok" || it.Dead || it.Status == "disabled" || strings.TrimSpace(it.Value) == "" {
|
||||
continue
|
||||
}
|
||||
sub, serr := s.grok.FetchSubscription(ctx, it.Value)
|
||||
if serr != nil {
|
||||
if errors.Is(serr, grok.ErrAuth) {
|
||||
_, _ = s.tokens.Update(ctx, "grok", it.ID, map[string]any{"status": "disabled", "dead": true})
|
||||
}
|
||||
continue // transient upstream error → leave as-is, retry next tick
|
||||
}
|
||||
if sub == nil || !sub.Member {
|
||||
// no ACTIVE subscription (INACTIVE / empty) → membership lapsed → dead.
|
||||
_, _ = s.tokens.Update(ctx, "grok", it.ID, map[string]any{"status": "disabled", "dead": true})
|
||||
continue
|
||||
}
|
||||
data, derr := s.grok.FetchCreditsBalance(ctx, it.Value)
|
||||
if derr != nil {
|
||||
if errors.Is(derr, grok.ErrAuth) {
|
||||
_, _ = s.tokens.Update(ctx, "grok", it.ID, map[string]any{"status": "disabled", "dead": true})
|
||||
}
|
||||
continue
|
||||
}
|
||||
meta := cloneJSONMap(it.Meta)
|
||||
meta["cached_quota_at"] = int(time.Now().Unix())
|
||||
if rem, ok := data["remaining"].(int); ok {
|
||||
meta["cached_quota_remaining"] = rem
|
||||
}
|
||||
if used, ok := data["used"].(int); ok {
|
||||
meta["cached_quota_used"] = used
|
||||
}
|
||||
if total, ok := data["total"].(int); ok {
|
||||
meta["cached_quota_total"] = total
|
||||
}
|
||||
patch := map[string]any{"meta": meta}
|
||||
if reset := strings.TrimSpace(stringValue(data["reset_after"])); reset != "" {
|
||||
patch["cached_quota_reset_after"] = reset
|
||||
}
|
||||
_, _ = s.tokens.Update(ctx, "grok", it.ID, patch)
|
||||
}
|
||||
}
|
||||
|
||||
// ImportCustomAccount adds an upstream as a custom account: base_url + key, the
|
||||
// csv list of model ids it serves (empty = all), plus optional weight and
|
||||
// per-account concurrency. No probe — the account goes active immediately and is
|
||||
@@ -1446,11 +1506,10 @@ func (s *TokenService) Quota(ctx context.Context, pool, id string) (map[string]a
|
||||
meta["cached_quota_total"] = total
|
||||
}
|
||||
patch["meta"] = meta
|
||||
// Recovery time is the death deadline: the maintenance sweep expires a grok
|
||||
// account once this marker passes (grok sso can't be renewed). Only stamp it
|
||||
// when still unset (import couldn't resolve it) — never move it forward on a
|
||||
// later refresh, so an admin opening 账号管理 can't push the death time out.
|
||||
if reset := strings.TrimSpace(stringValue(data["reset_after"])); reset != "" && strings.TrimSpace(item.CachedQuotaResetAfter) == "" {
|
||||
// Recovery time is the credits' weekly reset (when the grant refills) —
|
||||
// purely informational, NOT a death deadline (liveness is judged by the
|
||||
// subscriptions sweep / real 401s), so it's safe to refresh every time.
|
||||
if reset := strings.TrimSpace(stringValue(data["reset_after"])); reset != "" {
|
||||
patch["cached_quota_reset_after"] = reset
|
||||
item.CachedQuotaResetAfter = reset
|
||||
}
|
||||
|
||||
@@ -1303,7 +1303,7 @@ const maxTempDeadAccounts = 3
|
||||
// auth retry uses a FRESH token instead of replaying the stale one.
|
||||
func (s *V1Service) runPoolWithFailover(ctx context.Context, eventID, pool string, active []model.TokenAccount, kind string,
|
||||
attempt func(token model.TokenAccount) ([]byte, error),
|
||||
classify func(error) (isAuth, isQuota, isTemporary bool),
|
||||
classify func(error) (isAuth, isQuota, isTemporary, isDead bool),
|
||||
refreshOnAuth func(tokenID string) (model.TokenAccount, bool),
|
||||
tempFailover bool,
|
||||
) ([]byte, error) {
|
||||
@@ -1357,7 +1357,7 @@ func (s *V1Service) runPoolWithFailover(ctx context.Context, eventID, pool strin
|
||||
// to the next account. The per-account concurrency gate is held by the caller.
|
||||
func (s *V1Service) tryAccount(ctx context.Context, eventID, pool string, token model.TokenAccount, kind string,
|
||||
attempt func(token model.TokenAccount) ([]byte, error),
|
||||
classify func(error) (isAuth, isQuota, isTemporary bool),
|
||||
classify func(error) (isAuth, isQuota, isTemporary, isDead bool),
|
||||
refreshOnAuth func(tokenID string) (model.TokenAccount, bool),
|
||||
tempFailover bool,
|
||||
) ([]byte, error, bool, bool) {
|
||||
@@ -1375,7 +1375,7 @@ func (s *V1Service) tryAccount(ctx context.Context, eventID, pool string, token
|
||||
})
|
||||
return data, nil, false, false
|
||||
}
|
||||
isAuth, isQuota, isTemp := classify(err)
|
||||
isAuth, isQuota, isTemp, isDead := classify(err)
|
||||
if isQuota {
|
||||
s.markTokenFailure(ctx, pool, token, kind, false, true)
|
||||
return nil, err, true, false
|
||||
@@ -1392,6 +1392,10 @@ func (s *V1Service) tryAccount(ctx context.Context, eventID, pool string, token
|
||||
s.markTokenFailure(ctx, pool, token, kind, true, false)
|
||||
return nil, err, true, false
|
||||
}
|
||||
if isDead {
|
||||
s.markTokenDead(ctx, pool, token, kind)
|
||||
return nil, err, true, true
|
||||
}
|
||||
if isTemp {
|
||||
if tempFailover {
|
||||
// Ops policy (adobe): a temporary upstream error ("system under
|
||||
@@ -1424,8 +1428,8 @@ func (s *V1Service) tryAccount(ctx context.Context, eventID, pool string, token
|
||||
}
|
||||
}
|
||||
|
||||
func adobeErrClass(e error) (bool, bool, bool) {
|
||||
return errors.Is(e, adobe.ErrAuth), errors.Is(e, adobe.ErrQuotaExhausted), errors.Is(e, adobe.ErrTemporaryUpstream)
|
||||
func adobeErrClass(e error) (bool, bool, bool, bool) {
|
||||
return errors.Is(e, adobe.ErrAuth), errors.Is(e, adobe.ErrQuotaExhausted), errors.Is(e, adobe.ErrTemporaryUpstream), errors.Is(e, adobe.ErrDeadUpstream)
|
||||
}
|
||||
|
||||
func (s *V1Service) generateAdobeImage(ctx context.Context, eventID string, modelItem *model.ModelConfig, in V1ImageRequest, aspectRatio, resolution string) ([]byte, error) {
|
||||
@@ -2178,8 +2182,8 @@ func (s *V1Service) generateChatGPTImage(ctx context.Context, eventID string, mo
|
||||
s.reconcileChatGPTQuota(ctx, token.ID, token.Value)
|
||||
}
|
||||
return data, genErr
|
||||
}, func(e error) (bool, bool, bool) {
|
||||
return errors.Is(e, chatgpt.ErrAuth), errors.Is(e, chatgpt.ErrQuotaExhausted), errors.Is(e, chatgpt.ErrTemporaryUpstream)
|
||||
}, func(e error) (bool, bool, bool, bool) {
|
||||
return errors.Is(e, chatgpt.ErrAuth), errors.Is(e, chatgpt.ErrQuotaExhausted), errors.Is(e, chatgpt.ErrTemporaryUpstream), false
|
||||
}, nil, false) // chatgpt token IS the credential — no cookie to refresh
|
||||
}
|
||||
|
||||
@@ -2305,8 +2309,8 @@ func (s *V1Service) generateLeonardoImage(ctx context.Context, eventID string, m
|
||||
// sink to 限额 if below the floor (best-effort; never fails a done render).
|
||||
s.reconcileLeonardoCredits(ctx, token.ID, token.Value)
|
||||
return data, nil
|
||||
}, func(e error) (bool, bool, bool) {
|
||||
return errors.Is(e, leonardo.ErrAuth), errors.Is(e, leonardo.ErrQuotaExhausted), errors.Is(e, leonardo.ErrTemporaryUpstream)
|
||||
}, func(e error) (bool, bool, bool, bool) {
|
||||
return errors.Is(e, leonardo.ErrAuth), errors.Is(e, leonardo.ErrQuotaExhausted), errors.Is(e, leonardo.ErrTemporaryUpstream), false
|
||||
}, nil, false)
|
||||
}
|
||||
|
||||
@@ -2435,8 +2439,8 @@ func (s *V1Service) generateKreaImage(ctx context.Context, eventID string, model
|
||||
}
|
||||
data, _, genErr := s.krea.GenerateImage(ctx, cookie, in.Prompt, width, height, refs)
|
||||
return data, genErr
|
||||
}, func(e error) (bool, bool, bool) {
|
||||
return errors.Is(e, krea.ErrAuth), errors.Is(e, krea.ErrQuotaExhausted), errors.Is(e, krea.ErrTemporaryUpstream)
|
||||
}, func(e error) (bool, bool, bool, bool) {
|
||||
return errors.Is(e, krea.ErrAuth), errors.Is(e, krea.ErrQuotaExhausted), errors.Is(e, krea.ErrTemporaryUpstream), false
|
||||
}, nil, false)
|
||||
}
|
||||
|
||||
@@ -2507,8 +2511,8 @@ func (s *V1Service) generateImagineImage(ctx context.Context, eventID string, mo
|
||||
return nil, genErr
|
||||
}
|
||||
return data, nil
|
||||
}, func(e error) (bool, bool, bool) {
|
||||
return errors.Is(e, imagine.ErrAuth), errors.Is(e, imagine.ErrQuotaExhausted), errors.Is(e, imagine.ErrTemporaryUpstream)
|
||||
}, func(e error) (bool, bool, bool, bool) {
|
||||
return errors.Is(e, imagine.ErrAuth), errors.Is(e, imagine.ErrQuotaExhausted), errors.Is(e, imagine.ErrTemporaryUpstream), false
|
||||
}, nil, false)
|
||||
}
|
||||
|
||||
@@ -2955,6 +2959,18 @@ func (s *V1Service) markTokenFailure(ctx context.Context, pool string, token mod
|
||||
_, _ = s.tokens.Update(ctx, pool, token.ID, patch)
|
||||
}
|
||||
|
||||
// markTokenDead disables an account and marks it dead on a fatal upstream error
|
||||
// (a non-overload temporary Adobe failure that ops policy treats as account death).
|
||||
func (s *V1Service) markTokenDead(ctx context.Context, pool string, token model.TokenAccount, kind string) {
|
||||
_, _ = s.tokens.Update(ctx, pool, token.ID, map[string]any{
|
||||
"last_used_at": time.Now(),
|
||||
"fail_total": gorm.Expr("fail_total + 1"),
|
||||
"fails": gorm.Expr("fails + 1"),
|
||||
"status": "disabled",
|
||||
"dead": true,
|
||||
})
|
||||
}
|
||||
|
||||
// nextCursor returns the pool's current round-robin position and atomically
|
||||
// advances it by one. Concurrent callers each get a distinct value, so parallel
|
||||
// picks land on different accounts instead of racing onto the same one. The
|
||||
|
||||
Reference in New Issue
Block a user