chiyi/image2api
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(upstream): custom 模型按媒体分类配置参考文件上限

Browse Source 
- ModelConfig 新增 max_reference_videos / max_reference_audios
  (配合既有 max_reference_images);admin 读写接口透传
- custom 模型表单去掉「参考模式」下拉,改为参考图/参考视频/
  参考音频三个数量输入;合计张数走 totalRefLimit 校验
- vivid 上游 /v1/uploads 按文件魔数自动选 kind(image/video/
  audio)与 mime;openai 协议上游遇视频/音频参考自动跳过换号
- videoCaps 让 custom 模型的分类上限在画图台/文档徽标生效
- reference_mode 请求级 override 放行 "media"

Generated with [Devin](https://devin.ai)

Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
chiyi
2026-10-14 14:58:34 +08:00
co-authored by Devin
parent babfc71f13
commit 1be1b00511
9 changed files with 236 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/internal/service/admin_read.go
+2
View File
@@ -102,6 +102,8 @@ func (s *AdminReadService) ModelsView(ctx context.Context) ([]map[string]any, er
"duration_prices_agent": map[string]any(item.DurationPricesAgent),
"durations": repo.JSONStrings(item.Durations),
"max_reference_images": item.MaxReferenceImages,
"max_reference_videos": item.MaxReferenceVideos,
"max_reference_audios": item.MaxReferenceAudios,
"reference_mode": item.ReferenceMode,
"weight": item.Weight,
"generation_count": item.GenerationCount,
backend/internal/service/admin_write.go
+8
View File
@@ -401,6 +401,8 @@ func (s *AdminWriteService) CreateModel(ctx context.Context, body map[string]any
DurationPricesAgent: jsonMap(body["duration_prices_agent"]),
Durations: jsonArray(body["durations"]),
MaxReferenceImages: intValue(body["max_reference_images"]),
MaxReferenceVideos: intValue(body["max_reference_videos"]),
MaxReferenceAudios: intValue(body["max_reference_audios"]),
ReferenceMode: defaultString(strings.TrimSpace(stringValue(body["reference_mode"])), "none"),
Weight: intValue(body["weight"]),
CreatedAt: time.Now(),
@@ -477,6 +479,12 @@ func (s *AdminWriteService) UpdateModel(ctx context.Context, modelID string, bod
if _, ok := body["max_reference_images"]; ok {
patch["max_reference_images"] = intValue(body["max_reference_images"])
}
if _, ok := body["max_reference_videos"]; ok {
patch["max_reference_videos"] = intValue(body["max_reference_videos"])
}
if _, ok := body["max_reference_audios"]; ok {
patch["max_reference_audios"] = intValue(body["max_reference_audios"])
}
if _, ok := body["reference_mode"]; ok {
patch["reference_mode"] = defaultString(strings.TrimSpace(stringValue(body["reference_mode"])), "none")
}
backend/internal/service/user_generation.go
+6 -5
View File
@@ -95,9 +95,10 @@ func (s *UserGenerationService) Generate(ctx context.Context, user *model.User,
}
}
// validateReferenceMode mirrors the /v1 checks: the override must be "frame"
// or "asset", the model must support references at all, and frame mode carries
// at most 2 images (first+last frame).
// validateReferenceMode mirrors the /v1 checks: the override must be "frame",
// "asset" or "media", the model must support references at all, and frame mode
// carries at most 2 images (first+last frame). "media" = 参考视频/音频 — refs
// may be video or audio blobs (vivid-style upstreams that accept all kinds).
func validateReferenceMode(rm string, modelItem *model.ModelConfig, refCount int) error {
rm = strings.TrimSpace(rm)
if rm == "" {
@@ -107,8 +108,8 @@ func validateReferenceMode(rm string, modelItem *model.ModelConfig, refCount int
if supported == "" || supported == "none" {
return fmt.Errorf("%w: reference_mode not supported for this model", ErrUnsupportedParams)
}
if rm != "frame" && rm != "asset" {
return fmt.Errorf("%w: reference_mode must be 'frame' or 'asset'", ErrUnsupportedParams)
if rm != "frame" && rm != "asset" && rm != "media" {
return fmt.Errorf("%w: reference_mode must be 'frame', 'asset' or 'media'", ErrUnsupportedParams)
}
if rm == "frame" && refCount > 2 {
return fmt.Errorf("%w: frame mode supports at most 2 reference images (first+last frame), got %d", ErrUnsupportedParams, refCount)
backend/internal/service/v1.go
+67 -9
View File
@@ -224,7 +224,7 @@ type V1VideoRequest struct {
AspectRatio string
Resolution string
ReferenceImages []string
ReferenceMode string // "frame" or "asset", overrides model default
ReferenceMode string // "frame", "asset" or "media", overrides model default
// BaseURL — see V1ImageRequest.BaseURL.
BaseURL string
// AccountID — see V1ImageRequest.AccountID.
@@ -1273,7 +1273,7 @@ func (s *V1Service) prepareImage(ctx context.Context, principal *APIPrincipal, i
}
refLimit := 0
if modelItem.ImageToImage {
refLimit = modelItem.MaxReferenceImages
refLimit = totalRefLimit(modelItem)
if refLimit <= 0 {
refLimit = 1
}
@@ -1362,7 +1362,7 @@ func (s *V1Service) prepareVideo(ctx context.Context, principal *APIPrincipal, i
} else if !ok {
return nil, "", "", "", 0, ErrNoProviderAccount
}
refLimit := modelItem.MaxReferenceImages
refLimit := totalRefLimit(modelItem)
if refLimit <= 0 {
refLimit = 10
}
@@ -2120,6 +2120,44 @@ func (s *V1Service) generateRunwayVideo(ctx context.Context, eventID string, mod
return nil, "", lastErr
}
// totalRefLimit is the aggregate reference-file cap across all media kinds.
// Models with per-kind caps (max_reference_videos/audios) total them; legacy
// models only have max_reference_images, so the sum degenerates correctly.
func totalRefLimit(m *model.ModelConfig) int {
return m.MaxReferenceImages + m.MaxReferenceVideos + m.MaxReferenceAudios
}
// refKinds counts decoded refs by media class (image / video / audio) using the
// same magic-byte detection the leonardo upload path relies on.
func refKinds(refs [][]byte) (images, videos, audios int) {
for _, r := range refs {
switch detectMediaType(r) {
case "video":
videos++
case "audio":
audios++
default:
images++
}
}
return
}
// checkCustomRefCaps validates decoded refs against a custom model's per-kind
// caps (Chinese messages mirror the leonardo spec errors).
func checkCustomRefCaps(modelItem *model.ModelConfig, refs [][]byte) (images, videos, audios int, err error) {
images, videos, audios = refKinds(refs)
switch {
case videos > modelItem.MaxReferenceVideos:
err = fmt.Errorf("最多 %d 段视频参考", modelItem.MaxReferenceVideos)
case audios > modelItem.MaxReferenceAudios:
err = fmt.Errorf("最多 %d 段音频参考", modelItem.MaxReferenceAudios)
case images > modelItem.MaxReferenceImages:
err = fmt.Errorf("最多 %d 张图片参考", modelItem.MaxReferenceImages)
}
return
}
// customAccountServes reports whether a custom (upstream) account is usable for a
// given model id: active, not dead, has a base_url, and its meta.models list (csv
// of model ids it serves) contains the id. An empty models list serves ALL ids.
@@ -2201,7 +2239,11 @@ func (s *V1Service) generateCustomImage(ctx context.Context, eventID string, mod
if s.custom == nil {
return nil, "", errors.New("custom client not configured")
}
refs, err := decodeReferenceImages(in.ReferenceImages, max(1, modelItem.MaxReferenceImages))
refs, err := decodeReferenceImages(in.ReferenceImages, max(1, totalRefLimit(modelItem)))
if err != nil {
return nil, "", err
}
_, vids, auds, err := checkCustomRefCaps(modelItem, refs)
if err != nil {
return nil, "", err
}
@@ -2229,10 +2271,17 @@ func (s *V1Service) generateCustomImage(ctx context.Context, eventID string, mod
_ = s.events.SetAccount(ctx, eventID, token.ID, token.AccountEmail)
_ = s.tokens.TouchLastUsed(ctx, token.ID)
baseURL := stringValue(token.Meta["base_url"])
vivid := strings.EqualFold(stringValue(token.Meta["protocol"]), "vivid")
if !vivid && (vids > 0 || auds > 0) {
// OpenAI-protocol upstreams only accept image refs — skip this
// account; another vivid-protocol account may still serve.
lastErr = errors.New("upstream does not accept video/audio refs")
return false, true
}
var d []byte
var u string
var genErr error
if strings.EqualFold(stringValue(token.Meta["protocol"]), "vivid") {
if vivid {
d, u, genErr = s.custom.GenerateVividImage(ctx, baseURL, token.Value, modelItem.ID, in.Prompt, aspectRatio, vividQuality(resolution), refs, !urlOnly)
} else {
d, u, genErr = s.custom.GenerateImage(ctx, baseURL, token.Value, modelItem.ID, in.Prompt, size, quality, refs, !urlOnly)
@@ -2291,9 +2340,13 @@ func (s *V1Service) generateCustomVideo(ctx context.Context, eventID string, mod
return nil, "", ErrNoProviderAccount
}
size := upstreamVideoSize(aspectRatio, resolution)
// Optional reference frames (image-to-video / first-last frames) — forwarded
// to the upstream as multipart input_reference[] files.
frames, err := decodeReferenceImages(in.ReferenceImages, max(1, modelItem.MaxReferenceImages))
// Optional reference files (images / videos / audios for vivid upstreams;
// images only for OpenAI-protocol ones).
frames, err := decodeReferenceImages(in.ReferenceImages, max(1, totalRefLimit(modelItem)))
if err != nil {
return nil, "", err
}
_, vids, auds, err := checkCustomRefCaps(modelItem, frames)
if err != nil {
return nil, "", err
}
@@ -2311,10 +2364,15 @@ func (s *V1Service) generateCustomVideo(ctx context.Context, eventID string, mod
_ = s.events.SetAccount(ctx, eventID, token.ID, token.AccountEmail)
_ = s.tokens.TouchLastUsed(ctx, token.ID)
baseURL := stringValue(token.Meta["base_url"])
vivid := strings.EqualFold(stringValue(token.Meta["protocol"]), "vivid")
if !vivid && (vids > 0 || auds > 0) {
lastErr = errors.New("upstream does not accept video/audio refs")
return false, true
}
var d []byte
var url string
var genErr error
if strings.EqualFold(stringValue(token.Meta["protocol"]), "vivid") {
if vivid {
d, url, genErr = s.custom.GenerateVividVideo(ctx, baseURL, token.Value, modelItem.ID, in.Prompt, aspectRatio, vividQuality(resolution), durationSeconds, frames, downloadResult)
} else {
d, url, genErr = s.custom.GenerateVideo(ctx, baseURL, token.Value, modelItem.ID, in.Prompt, size, durationSeconds, frames, downloadResult)