feat(upstream): custom 模型按媒体分类配置参考文件上限
- ModelConfig 新增 max_reference_videos / max_reference_audios (配合既有 max_reference_images);admin 读写接口透传 - custom 模型表单去掉「参考模式」下拉,改为参考图/参考视频/ 参考音频三个数量输入;合计张数走 totalRefLimit 校验 - vivid 上游 /v1/uploads 按文件魔数自动选 kind(image/video/ audio)与 mime;openai 协议上游遇视频/音频参考自动跳过换号 - videoCaps 让 custom 模型的分类上限在画图台/文档徽标生效 - reference_mode 请求级 override 放行 "media" Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
@@ -139,6 +139,10 @@ type ModelConfig struct {
|
|||||||
DurationPricesAgent datatypes.JSONMap `gorm:"type:jsonb;column:duration_prices_agent"`
|
DurationPricesAgent datatypes.JSONMap `gorm:"type:jsonb;column:duration_prices_agent"`
|
||||||
Durations datatypes.JSON `gorm:"type:jsonb"`
|
Durations datatypes.JSON `gorm:"type:jsonb"`
|
||||||
MaxReferenceImages int `gorm:"not null;default:0"`
|
MaxReferenceImages int `gorm:"not null;default:0"`
|
||||||
|
// Per-kind reference caps for upstreams that accept video/audio refs
|
||||||
|
// (vivid-protocol custom upstreams). 0 = that kind is not accepted.
|
||||||
|
MaxReferenceVideos int `gorm:"not null;default:0"`
|
||||||
|
MaxReferenceAudios int `gorm:"not null;default:0"`
|
||||||
ReferenceMode string `gorm:"size:32;not null;default:'none'"`
|
ReferenceMode string `gorm:"size:32;not null;default:'none'"`
|
||||||
// Custom-upstream models (provider="custom"): UpstreamModel is the model name
|
// Custom-upstream models (provider="custom"): UpstreamModel is the model name
|
||||||
// sent to the upstream OpenAI-compatible API; the base_url + key live on the
|
// sent to the upstream OpenAI-compatible API; the base_url + key live on the
|
||||||
|
|||||||
@@ -178,14 +178,13 @@ func (c *Client) vividPayload(ctx context.Context, baseURL, apiKey, model, promp
|
|||||||
return payload, nil
|
return payload, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// vividUploadRef runs the two-step upload: POST /v1/uploads {kind:"image",mime}
|
// vividUploadRef runs the two-step upload: POST /v1/uploads {kind,mime} → PUT
|
||||||
// → PUT the bytes to the presigned url → returns the key for refs[].
|
// the bytes to the presigned url → returns the key for refs[]. kind is image /
|
||||||
|
// video / audio, derived from the file magic (same classes the upstream
|
||||||
|
// accepts: png/jpeg/webp · mp4/webm/mov · mp3/wav/m4a/ogg).
|
||||||
func (c *Client) vividUploadRef(ctx context.Context, baseURL, apiKey string, data []byte) (string, error) {
|
func (c *Client) vividUploadRef(ctx context.Context, baseURL, apiKey string, data []byte) (string, error) {
|
||||||
mime := http.DetectContentType(data)
|
kind, mime := vividRefKindMime(data)
|
||||||
if mime == "" || mime == "application/octet-stream" {
|
raw, _ := json.Marshal(map[string]any{"kind": kind, "mime": mime})
|
||||||
mime = "image/png"
|
|
||||||
}
|
|
||||||
raw, _ := json.Marshal(map[string]any{"kind": "image", "mime": mime})
|
|
||||||
up, err := c.doJSON(ctx, http.MethodPost, baseURL+"/v1/uploads", apiKey, raw)
|
up, err := c.doJSON(ctx, http.MethodPost, baseURL+"/v1/uploads", apiKey, raw)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
@@ -213,6 +212,58 @@ func (c *Client) vividUploadRef(ctx context.Context, baseURL, apiKey string, dat
|
|||||||
return key, nil
|
return key, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// vividRefKindMime classifies a reference blob into the vivid "kind" (image /
|
||||||
|
// video / audio) plus a concrete mime. Magic bytes first (DetectContentType
|
||||||
|
// misses mov/m4a/ogg-audio), then sniffing, then a png default.
|
||||||
|
func vividRefKindMime(data []byte) (kind, mime string) {
|
||||||
|
n := len(data)
|
||||||
|
// ISOBMFF box: ftyp — mp4/mov/m4a share the container; the brand decides.
|
||||||
|
if n >= 12 && string(data[4:8]) == "ftyp" {
|
||||||
|
brand := string(data[8:12])
|
||||||
|
switch {
|
||||||
|
case strings.Contains(brand, "M4A"), strings.Contains(brand, "M4B"):
|
||||||
|
return "audio", "audio/mp4"
|
||||||
|
case strings.Contains(brand, "qt"):
|
||||||
|
return "video", "video/quicktime"
|
||||||
|
default:
|
||||||
|
return "video", "video/mp4"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// WebM/MKV EBML
|
||||||
|
if n >= 4 && data[0] == 0x1A && data[1] == 0x45 && data[2] == 0xDF && data[3] == 0xA3 {
|
||||||
|
return "video", "video/webm"
|
||||||
|
}
|
||||||
|
// MP3: ID3 tag or MPEG sync
|
||||||
|
if n >= 3 && string(data[:3]) == "ID3" {
|
||||||
|
return "audio", "audio/mpeg"
|
||||||
|
}
|
||||||
|
if n >= 2 && data[0] == 0xFF && (data[1]&0xE0) == 0xE0 {
|
||||||
|
return "audio", "audio/mpeg"
|
||||||
|
}
|
||||||
|
// WAV / OGG / FLAC
|
||||||
|
if n >= 4 && string(data[:4]) == "RIFF" {
|
||||||
|
return "audio", "audio/wav"
|
||||||
|
}
|
||||||
|
if n >= 4 && string(data[:4]) == "OggS" {
|
||||||
|
return "audio", "audio/ogg"
|
||||||
|
}
|
||||||
|
if n >= 4 && string(data[:4]) == "fLaC" {
|
||||||
|
return "audio", "audio/x-flac"
|
||||||
|
}
|
||||||
|
// Sniffed type (images land here; also catches anything odd above missed).
|
||||||
|
if m := http.DetectContentType(data); m != "" && m != "application/octet-stream" {
|
||||||
|
switch {
|
||||||
|
case strings.HasPrefix(m, "video/"):
|
||||||
|
return "video", m
|
||||||
|
case strings.HasPrefix(m, "audio/"), m == "application/ogg":
|
||||||
|
return "audio", m
|
||||||
|
case strings.HasPrefix(m, "image/"):
|
||||||
|
return "image", m
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return "image", "image/png"
|
||||||
|
}
|
||||||
|
|
||||||
// vividGenerate POSTs /v1/generate and reads the NDJSON stream to the terminal
|
// vividGenerate POSTs /v1/generate and reads the NDJSON stream to the terminal
|
||||||
// frame. If the stream drops before terminal, it reconnects once with the
|
// frame. If the stream drops before terminal, it reconnects once with the
|
||||||
// jobId (same-request resume — no new task, no double charge).
|
// jobId (same-request resume — no new task, no double charge).
|
||||||
|
|||||||
@@ -141,6 +141,56 @@ func TestVividErrorCodes(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestVividRefKindMime(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
data []byte
|
||||||
|
kind string
|
||||||
|
}{
|
||||||
|
{"png", []byte{0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, 1, 2, 3, 4}, "image"},
|
||||||
|
{"jpeg", []byte{0xFF, 0xD8, 0xFF, 0xE0, 1, 2, 3, 4, 5, 6}, "image"},
|
||||||
|
{"mp4", []byte{0, 0, 0, 0, 'f', 't', 'y', 'p', 'i', 's', 'o', 'm'}, "video"},
|
||||||
|
{"mov", []byte{0, 0, 0, 0, 'f', 't', 'y', 'p', 'q', 't', ' ', ' '}, "video"},
|
||||||
|
{"webm", []byte{0x1A, 0x45, 0xDF, 0xA3, 1, 2, 3, 4}, "video"},
|
||||||
|
{"mp3-id3", []byte{'I', 'D', '3', 4, 0, 0, 1, 2}, "audio"},
|
||||||
|
{"mp3-sync", []byte{0xFF, 0xFB, 1, 2, 3, 4}, "audio"},
|
||||||
|
{"wav", []byte{'R', 'I', 'F', 'F', 1, 2, 3, 4, 'W', 'A', 'V', 'E'}, "audio"},
|
||||||
|
{"ogg", []byte{'O', 'g', 'g', 'S', 1, 2, 3, 4}, "audio"},
|
||||||
|
{"m4a", []byte{0, 0, 0, 0, 'f', 't', 'y', 'p', 'M', '4', 'A', ' '}, "audio"},
|
||||||
|
{"empty", []byte{}, "image"},
|
||||||
|
}
|
||||||
|
for _, tc := range cases {
|
||||||
|
kind, mime := vividRefKindMime(tc.data)
|
||||||
|
if kind != tc.kind {
|
||||||
|
t.Errorf("%s: kind=%s mime=%s", tc.name, kind, mime)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestVividUploadRefKindVideo(t *testing.T) {
|
||||||
|
var gotKind, gotMime string
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.URL.Path == "/v1/uploads" {
|
||||||
|
var req map[string]any
|
||||||
|
_ = json.NewDecoder(r.Body).Decode(&req)
|
||||||
|
gotKind, _ = req["kind"].(string)
|
||||||
|
gotMime, _ = req["mime"].(string)
|
||||||
|
fmt.Fprintf(w, `{"key":"k1","url":"http://%s/up"}`, r.Host)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
webm := []byte{0x1A, 0x45, 0xDF, 0xA3, 1, 2, 3, 4, 5, 6, 7, 8}
|
||||||
|
key, err := NewClient().vividUploadRef(context.Background(), srv.URL, "vk", webm)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("upload: %v", err)
|
||||||
|
}
|
||||||
|
if key != "k1" || gotKind != "video" || gotMime != "video/webm" {
|
||||||
|
t.Fatalf("key=%q kind=%q mime=%q", key, gotKind, gotMime)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestVividUploadRefFlow(t *testing.T) {
|
func TestVividUploadRefFlow(t *testing.T) {
|
||||||
var putOK bool
|
var putOK bool
|
||||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
|||||||
@@ -102,6 +102,8 @@ func (s *AdminReadService) ModelsView(ctx context.Context) ([]map[string]any, er
|
|||||||
"duration_prices_agent": map[string]any(item.DurationPricesAgent),
|
"duration_prices_agent": map[string]any(item.DurationPricesAgent),
|
||||||
"durations": repo.JSONStrings(item.Durations),
|
"durations": repo.JSONStrings(item.Durations),
|
||||||
"max_reference_images": item.MaxReferenceImages,
|
"max_reference_images": item.MaxReferenceImages,
|
||||||
|
"max_reference_videos": item.MaxReferenceVideos,
|
||||||
|
"max_reference_audios": item.MaxReferenceAudios,
|
||||||
"reference_mode": item.ReferenceMode,
|
"reference_mode": item.ReferenceMode,
|
||||||
"weight": item.Weight,
|
"weight": item.Weight,
|
||||||
"generation_count": item.GenerationCount,
|
"generation_count": item.GenerationCount,
|
||||||
|
|||||||
@@ -401,6 +401,8 @@ func (s *AdminWriteService) CreateModel(ctx context.Context, body map[string]any
|
|||||||
DurationPricesAgent: jsonMap(body["duration_prices_agent"]),
|
DurationPricesAgent: jsonMap(body["duration_prices_agent"]),
|
||||||
Durations: jsonArray(body["durations"]),
|
Durations: jsonArray(body["durations"]),
|
||||||
MaxReferenceImages: intValue(body["max_reference_images"]),
|
MaxReferenceImages: intValue(body["max_reference_images"]),
|
||||||
|
MaxReferenceVideos: intValue(body["max_reference_videos"]),
|
||||||
|
MaxReferenceAudios: intValue(body["max_reference_audios"]),
|
||||||
ReferenceMode: defaultString(strings.TrimSpace(stringValue(body["reference_mode"])), "none"),
|
ReferenceMode: defaultString(strings.TrimSpace(stringValue(body["reference_mode"])), "none"),
|
||||||
Weight: intValue(body["weight"]),
|
Weight: intValue(body["weight"]),
|
||||||
CreatedAt: time.Now(),
|
CreatedAt: time.Now(),
|
||||||
@@ -477,6 +479,12 @@ func (s *AdminWriteService) UpdateModel(ctx context.Context, modelID string, bod
|
|||||||
if _, ok := body["max_reference_images"]; ok {
|
if _, ok := body["max_reference_images"]; ok {
|
||||||
patch["max_reference_images"] = intValue(body["max_reference_images"])
|
patch["max_reference_images"] = intValue(body["max_reference_images"])
|
||||||
}
|
}
|
||||||
|
if _, ok := body["max_reference_videos"]; ok {
|
||||||
|
patch["max_reference_videos"] = intValue(body["max_reference_videos"])
|
||||||
|
}
|
||||||
|
if _, ok := body["max_reference_audios"]; ok {
|
||||||
|
patch["max_reference_audios"] = intValue(body["max_reference_audios"])
|
||||||
|
}
|
||||||
if _, ok := body["reference_mode"]; ok {
|
if _, ok := body["reference_mode"]; ok {
|
||||||
patch["reference_mode"] = defaultString(strings.TrimSpace(stringValue(body["reference_mode"])), "none")
|
patch["reference_mode"] = defaultString(strings.TrimSpace(stringValue(body["reference_mode"])), "none")
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -95,9 +95,10 @@ func (s *UserGenerationService) Generate(ctx context.Context, user *model.User,
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// validateReferenceMode mirrors the /v1 checks: the override must be "frame"
|
// validateReferenceMode mirrors the /v1 checks: the override must be "frame",
|
||||||
// or "asset", the model must support references at all, and frame mode carries
|
// "asset" or "media", the model must support references at all, and frame mode
|
||||||
// at most 2 images (first+last frame).
|
// carries at most 2 images (first+last frame). "media" = 参考视频/音频 — refs
|
||||||
|
// may be video or audio blobs (vivid-style upstreams that accept all kinds).
|
||||||
func validateReferenceMode(rm string, modelItem *model.ModelConfig, refCount int) error {
|
func validateReferenceMode(rm string, modelItem *model.ModelConfig, refCount int) error {
|
||||||
rm = strings.TrimSpace(rm)
|
rm = strings.TrimSpace(rm)
|
||||||
if rm == "" {
|
if rm == "" {
|
||||||
@@ -107,8 +108,8 @@ func validateReferenceMode(rm string, modelItem *model.ModelConfig, refCount int
|
|||||||
if supported == "" || supported == "none" {
|
if supported == "" || supported == "none" {
|
||||||
return fmt.Errorf("%w: reference_mode not supported for this model", ErrUnsupportedParams)
|
return fmt.Errorf("%w: reference_mode not supported for this model", ErrUnsupportedParams)
|
||||||
}
|
}
|
||||||
if rm != "frame" && rm != "asset" {
|
if rm != "frame" && rm != "asset" && rm != "media" {
|
||||||
return fmt.Errorf("%w: reference_mode must be 'frame' or 'asset'", ErrUnsupportedParams)
|
return fmt.Errorf("%w: reference_mode must be 'frame', 'asset' or 'media'", ErrUnsupportedParams)
|
||||||
}
|
}
|
||||||
if rm == "frame" && refCount > 2 {
|
if rm == "frame" && refCount > 2 {
|
||||||
return fmt.Errorf("%w: frame mode supports at most 2 reference images (first+last frame), got %d", ErrUnsupportedParams, refCount)
|
return fmt.Errorf("%w: frame mode supports at most 2 reference images (first+last frame), got %d", ErrUnsupportedParams, refCount)
|
||||||
|
|||||||
@@ -224,7 +224,7 @@ type V1VideoRequest struct {
|
|||||||
AspectRatio string
|
AspectRatio string
|
||||||
Resolution string
|
Resolution string
|
||||||
ReferenceImages []string
|
ReferenceImages []string
|
||||||
ReferenceMode string // "frame" or "asset", overrides model default
|
ReferenceMode string // "frame", "asset" or "media", overrides model default
|
||||||
// BaseURL — see V1ImageRequest.BaseURL.
|
// BaseURL — see V1ImageRequest.BaseURL.
|
||||||
BaseURL string
|
BaseURL string
|
||||||
// AccountID — see V1ImageRequest.AccountID.
|
// AccountID — see V1ImageRequest.AccountID.
|
||||||
@@ -1273,7 +1273,7 @@ func (s *V1Service) prepareImage(ctx context.Context, principal *APIPrincipal, i
|
|||||||
}
|
}
|
||||||
refLimit := 0
|
refLimit := 0
|
||||||
if modelItem.ImageToImage {
|
if modelItem.ImageToImage {
|
||||||
refLimit = modelItem.MaxReferenceImages
|
refLimit = totalRefLimit(modelItem)
|
||||||
if refLimit <= 0 {
|
if refLimit <= 0 {
|
||||||
refLimit = 1
|
refLimit = 1
|
||||||
}
|
}
|
||||||
@@ -1362,7 +1362,7 @@ func (s *V1Service) prepareVideo(ctx context.Context, principal *APIPrincipal, i
|
|||||||
} else if !ok {
|
} else if !ok {
|
||||||
return nil, "", "", "", 0, ErrNoProviderAccount
|
return nil, "", "", "", 0, ErrNoProviderAccount
|
||||||
}
|
}
|
||||||
refLimit := modelItem.MaxReferenceImages
|
refLimit := totalRefLimit(modelItem)
|
||||||
if refLimit <= 0 {
|
if refLimit <= 0 {
|
||||||
refLimit = 10
|
refLimit = 10
|
||||||
}
|
}
|
||||||
@@ -2120,6 +2120,44 @@ func (s *V1Service) generateRunwayVideo(ctx context.Context, eventID string, mod
|
|||||||
return nil, "", lastErr
|
return nil, "", lastErr
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// totalRefLimit is the aggregate reference-file cap across all media kinds.
|
||||||
|
// Models with per-kind caps (max_reference_videos/audios) total them; legacy
|
||||||
|
// models only have max_reference_images, so the sum degenerates correctly.
|
||||||
|
func totalRefLimit(m *model.ModelConfig) int {
|
||||||
|
return m.MaxReferenceImages + m.MaxReferenceVideos + m.MaxReferenceAudios
|
||||||
|
}
|
||||||
|
|
||||||
|
// refKinds counts decoded refs by media class (image / video / audio) using the
|
||||||
|
// same magic-byte detection the leonardo upload path relies on.
|
||||||
|
func refKinds(refs [][]byte) (images, videos, audios int) {
|
||||||
|
for _, r := range refs {
|
||||||
|
switch detectMediaType(r) {
|
||||||
|
case "video":
|
||||||
|
videos++
|
||||||
|
case "audio":
|
||||||
|
audios++
|
||||||
|
default:
|
||||||
|
images++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// checkCustomRefCaps validates decoded refs against a custom model's per-kind
|
||||||
|
// caps (Chinese messages mirror the leonardo spec errors).
|
||||||
|
func checkCustomRefCaps(modelItem *model.ModelConfig, refs [][]byte) (images, videos, audios int, err error) {
|
||||||
|
images, videos, audios = refKinds(refs)
|
||||||
|
switch {
|
||||||
|
case videos > modelItem.MaxReferenceVideos:
|
||||||
|
err = fmt.Errorf("最多 %d 段视频参考", modelItem.MaxReferenceVideos)
|
||||||
|
case audios > modelItem.MaxReferenceAudios:
|
||||||
|
err = fmt.Errorf("最多 %d 段音频参考", modelItem.MaxReferenceAudios)
|
||||||
|
case images > modelItem.MaxReferenceImages:
|
||||||
|
err = fmt.Errorf("最多 %d 张图片参考", modelItem.MaxReferenceImages)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
// customAccountServes reports whether a custom (upstream) account is usable for a
|
// customAccountServes reports whether a custom (upstream) account is usable for a
|
||||||
// given model id: active, not dead, has a base_url, and its meta.models list (csv
|
// given model id: active, not dead, has a base_url, and its meta.models list (csv
|
||||||
// of model ids it serves) contains the id. An empty models list serves ALL ids.
|
// of model ids it serves) contains the id. An empty models list serves ALL ids.
|
||||||
@@ -2201,7 +2239,11 @@ func (s *V1Service) generateCustomImage(ctx context.Context, eventID string, mod
|
|||||||
if s.custom == nil {
|
if s.custom == nil {
|
||||||
return nil, "", errors.New("custom client not configured")
|
return nil, "", errors.New("custom client not configured")
|
||||||
}
|
}
|
||||||
refs, err := decodeReferenceImages(in.ReferenceImages, max(1, modelItem.MaxReferenceImages))
|
refs, err := decodeReferenceImages(in.ReferenceImages, max(1, totalRefLimit(modelItem)))
|
||||||
|
if err != nil {
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
_, vids, auds, err := checkCustomRefCaps(modelItem, refs)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, "", err
|
return nil, "", err
|
||||||
}
|
}
|
||||||
@@ -2229,10 +2271,17 @@ func (s *V1Service) generateCustomImage(ctx context.Context, eventID string, mod
|
|||||||
_ = s.events.SetAccount(ctx, eventID, token.ID, token.AccountEmail)
|
_ = s.events.SetAccount(ctx, eventID, token.ID, token.AccountEmail)
|
||||||
_ = s.tokens.TouchLastUsed(ctx, token.ID)
|
_ = s.tokens.TouchLastUsed(ctx, token.ID)
|
||||||
baseURL := stringValue(token.Meta["base_url"])
|
baseURL := stringValue(token.Meta["base_url"])
|
||||||
|
vivid := strings.EqualFold(stringValue(token.Meta["protocol"]), "vivid")
|
||||||
|
if !vivid && (vids > 0 || auds > 0) {
|
||||||
|
// OpenAI-protocol upstreams only accept image refs — skip this
|
||||||
|
// account; another vivid-protocol account may still serve.
|
||||||
|
lastErr = errors.New("upstream does not accept video/audio refs")
|
||||||
|
return false, true
|
||||||
|
}
|
||||||
var d []byte
|
var d []byte
|
||||||
var u string
|
var u string
|
||||||
var genErr error
|
var genErr error
|
||||||
if strings.EqualFold(stringValue(token.Meta["protocol"]), "vivid") {
|
if vivid {
|
||||||
d, u, genErr = s.custom.GenerateVividImage(ctx, baseURL, token.Value, modelItem.ID, in.Prompt, aspectRatio, vividQuality(resolution), refs, !urlOnly)
|
d, u, genErr = s.custom.GenerateVividImage(ctx, baseURL, token.Value, modelItem.ID, in.Prompt, aspectRatio, vividQuality(resolution), refs, !urlOnly)
|
||||||
} else {
|
} else {
|
||||||
d, u, genErr = s.custom.GenerateImage(ctx, baseURL, token.Value, modelItem.ID, in.Prompt, size, quality, refs, !urlOnly)
|
d, u, genErr = s.custom.GenerateImage(ctx, baseURL, token.Value, modelItem.ID, in.Prompt, size, quality, refs, !urlOnly)
|
||||||
@@ -2291,9 +2340,13 @@ func (s *V1Service) generateCustomVideo(ctx context.Context, eventID string, mod
|
|||||||
return nil, "", ErrNoProviderAccount
|
return nil, "", ErrNoProviderAccount
|
||||||
}
|
}
|
||||||
size := upstreamVideoSize(aspectRatio, resolution)
|
size := upstreamVideoSize(aspectRatio, resolution)
|
||||||
// Optional reference frames (image-to-video / first-last frames) — forwarded
|
// Optional reference files (images / videos / audios for vivid upstreams;
|
||||||
// to the upstream as multipart input_reference[] files.
|
// images only for OpenAI-protocol ones).
|
||||||
frames, err := decodeReferenceImages(in.ReferenceImages, max(1, modelItem.MaxReferenceImages))
|
frames, err := decodeReferenceImages(in.ReferenceImages, max(1, totalRefLimit(modelItem)))
|
||||||
|
if err != nil {
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
_, vids, auds, err := checkCustomRefCaps(modelItem, frames)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, "", err
|
return nil, "", err
|
||||||
}
|
}
|
||||||
@@ -2311,10 +2364,15 @@ func (s *V1Service) generateCustomVideo(ctx context.Context, eventID string, mod
|
|||||||
_ = s.events.SetAccount(ctx, eventID, token.ID, token.AccountEmail)
|
_ = s.events.SetAccount(ctx, eventID, token.ID, token.AccountEmail)
|
||||||
_ = s.tokens.TouchLastUsed(ctx, token.ID)
|
_ = s.tokens.TouchLastUsed(ctx, token.ID)
|
||||||
baseURL := stringValue(token.Meta["base_url"])
|
baseURL := stringValue(token.Meta["base_url"])
|
||||||
|
vivid := strings.EqualFold(stringValue(token.Meta["protocol"]), "vivid")
|
||||||
|
if !vivid && (vids > 0 || auds > 0) {
|
||||||
|
lastErr = errors.New("upstream does not accept video/audio refs")
|
||||||
|
return false, true
|
||||||
|
}
|
||||||
var d []byte
|
var d []byte
|
||||||
var url string
|
var url string
|
||||||
var genErr error
|
var genErr error
|
||||||
if strings.EqualFold(stringValue(token.Meta["protocol"]), "vivid") {
|
if vivid {
|
||||||
d, url, genErr = s.custom.GenerateVividVideo(ctx, baseURL, token.Value, modelItem.ID, in.Prompt, aspectRatio, vividQuality(resolution), durationSeconds, frames, downloadResult)
|
d, url, genErr = s.custom.GenerateVividVideo(ctx, baseURL, token.Value, modelItem.ID, in.Prompt, aspectRatio, vividQuality(resolution), durationSeconds, frames, downloadResult)
|
||||||
} else {
|
} else {
|
||||||
d, url, genErr = s.custom.GenerateVideo(ctx, baseURL, token.Value, modelItem.ID, in.Prompt, size, durationSeconds, frames, downloadResult)
|
d, url, genErr = s.custom.GenerateVideo(ctx, baseURL, token.Value, modelItem.ID, in.Prompt, size, durationSeconds, frames, downloadResult)
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
<script setup>
|
<script setup>
|
||||||
import { ref, computed, watch } from 'vue'
|
import { ref, computed } from 'vue'
|
||||||
import { api, jsonBody } from '../api'
|
import { api, jsonBody } from '../api'
|
||||||
import Icon from './Icon.vue'
|
import Icon from './Icon.vue'
|
||||||
import SelectMenu from './SelectMenu.vue'
|
import SelectMenu from './SelectMenu.vue'
|
||||||
@@ -19,7 +19,8 @@ const alias = ref('')
|
|||||||
const type = ref('image')
|
const type = ref('image')
|
||||||
const ratios = ref(['1:1', '16:9', '9:16'])
|
const ratios = ref(['1:1', '16:9', '9:16'])
|
||||||
const maxRefs = ref(0)
|
const maxRefs = ref(0)
|
||||||
const refMode = ref('none')
|
const maxRefVids = ref(0)
|
||||||
|
const maxRefAuds = ref(0)
|
||||||
const weight = ref(0)
|
const weight = ref(0)
|
||||||
// tier -> { price, agent } — blank price means the tier is NOT supported.
|
// tier -> { price, agent } — blank price means the tier is NOT supported.
|
||||||
const res = ref(Object.fromEntries(ALL_RES.map((r) => [r, { price: '', agent: '' }])))
|
const res = ref(Object.fromEntries(ALL_RES.map((r) => [r, { price: '', agent: '' }])))
|
||||||
@@ -52,12 +53,6 @@ const isVideo = computed(() => type.value === 'video')
|
|||||||
// Resolution tiers depend on type: image = 1K/2K/4K, video = 540p/720p/1080p/2K/4K.
|
// Resolution tiers depend on type: image = 1K/2K/4K, video = 540p/720p/1080p/2K/4K.
|
||||||
const resOpts = computed(() => (isVideo.value ? VID_RES : IMG_RES))
|
const resOpts = computed(() => (isVideo.value ? VID_RES : IMG_RES))
|
||||||
|
|
||||||
// 首尾帧(frame) only has first+last slots → cap reference images at 2.
|
|
||||||
const refsCap = computed(() => (refMode.value === 'frame' ? 2 : 99))
|
|
||||||
watch([refMode, maxRefs], () => {
|
|
||||||
if (refMode.value === 'frame' && Number(maxRefs.value) > 2) maxRefs.value = 2
|
|
||||||
})
|
|
||||||
|
|
||||||
function toggleRatio(r) {
|
function toggleRatio(r) {
|
||||||
const i = ratios.value.indexOf(r)
|
const i = ratios.value.indexOf(r)
|
||||||
if (i >= 0) ratios.value.splice(i, 1)
|
if (i >= 0) ratios.value.splice(i, 1)
|
||||||
@@ -84,7 +79,6 @@ function collect(tiers, allowed) {
|
|||||||
async function save() {
|
async function save() {
|
||||||
const mid = id.value.trim()
|
const mid = id.value.trim()
|
||||||
if (!mid) { error.value = '请填写模型 id'; return }
|
if (!mid) { error.value = '请填写模型 id'; return }
|
||||||
if (refMode.value === 'frame' && Number(maxRefs.value) > 2) maxRefs.value = 2
|
|
||||||
const r = collect(res.value, resOpts.value)
|
const r = collect(res.value, resOpts.value)
|
||||||
if (!r.keys.length) { error.value = '请至少勾选一个分辨率并填价格'; return }
|
if (!r.keys.length) { error.value = '请至少勾选一个分辨率并填价格'; return }
|
||||||
const body = {
|
const body = {
|
||||||
@@ -97,9 +91,12 @@ async function save() {
|
|||||||
prices_agent: r.agent,
|
prices_agent: r.agent,
|
||||||
ratios: ratios.value.slice(),
|
ratios: ratios.value.slice(),
|
||||||
max_reference_images: Number(maxRefs.value) || 0,
|
max_reference_images: Number(maxRefs.value) || 0,
|
||||||
reference_mode: refMode.value,
|
max_reference_videos: Number(maxRefVids.value) || 0,
|
||||||
|
max_reference_audios: Number(maxRefAuds.value) || 0,
|
||||||
|
// asset 语义 = 通用参考文件;vivid 上游按 refs[] 直传,模式字段不再做区分
|
||||||
|
reference_mode: (Number(maxRefs.value) + Number(maxRefVids.value) + Number(maxRefAuds.value)) > 0 ? 'asset' : 'none',
|
||||||
weight: Number(weight.value) || 0,
|
weight: Number(weight.value) || 0,
|
||||||
image_to_image: (Number(maxRefs.value) || 0) > 0,
|
image_to_image: (Number(maxRefs.value) + Number(maxRefVids.value) + Number(maxRefAuds.value)) > 0,
|
||||||
}
|
}
|
||||||
if (isVideo.value) {
|
if (isVideo.value) {
|
||||||
body.resolutions = r.keys
|
body.resolutions = r.keys
|
||||||
@@ -192,14 +189,21 @@ async function save() {
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<div>
|
||||||
|
<label class="text-xs text-slate-500 block mb-1">参考文件上限(各类数量,vivid 上游支持三种;openai 协议只收图)</label>
|
||||||
<div class="flex gap-3">
|
<div class="flex gap-3">
|
||||||
<div class="flex-1">
|
<div class="flex-1">
|
||||||
<label class="text-xs text-slate-500 block mb-1">参考图张数<span v-if="refMode==='frame'" class="text-white/40">(首尾帧最多 2)</span></label>
|
<label class="text-[11px] text-slate-400 block mb-1">参考图</label>
|
||||||
<input v-model.number="maxRefs" type="number" min="0" :max="refsCap" class="field h-10" />
|
<input v-model.number="maxRefs" type="number" min="0" class="field h-10" placeholder="0" />
|
||||||
</div>
|
</div>
|
||||||
<div class="flex-1">
|
<div class="flex-1">
|
||||||
<label class="text-xs text-slate-500 block mb-1">参考模式</label>
|
<label class="text-[11px] text-slate-400 block mb-1">参考视频</label>
|
||||||
<SelectMenu v-model="refMode" :options="[{value:'none',label:'无'},{value:'asset',label:'参考图'},{value:'frame',label:'首尾帧(视频)'}]" />
|
<input v-model.number="maxRefVids" type="number" min="0" class="field h-10" placeholder="0" />
|
||||||
|
</div>
|
||||||
|
<div class="flex-1">
|
||||||
|
<label class="text-[11px] text-slate-400 block mb-1">参考音频</label>
|
||||||
|
<input v-model.number="maxRefAuds" type="number" min="0" class="field h-10" placeholder="0" />
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
|||||||
@@ -15,10 +15,11 @@ export function mediaCaps(model, preset) {
|
|||||||
// creativefabrica 上游只收图片参考(VIDEO_FRAME_TYPE_REFERENCE),不收视频/音频;
|
// creativefabrica 上游只收图片参考(VIDEO_FRAME_TYPE_REFERENCE),不收视频/音频;
|
||||||
// 它虽然叫 seedance,但要排除,否则会错误地显示「视频 3 音频 3」。
|
// 它虽然叫 seedance,但要排除,否则会错误地显示「视频 3 音频 3」。
|
||||||
const isSeedance = /^seedance/.test(model?.id || '') && (model?.provider || '') !== 'creativefabrica'
|
const isSeedance = /^seedance/.test(model?.id || '') && (model?.provider || '') !== 'creativefabrica'
|
||||||
const videos = declared(preset?.max_videos) ?? (isSeedance ? SEEDANCE_FALLBACK.videos : 0)
|
// preset 优先;custom 上游模型直接在模型字段里声明(max_reference_videos/audios)。
|
||||||
const audios = declared(preset?.max_audios) ?? (isSeedance ? SEEDANCE_FALLBACK.audios : 0)
|
const videos = declared(preset?.max_videos) ?? declared(model?.max_reference_videos) ?? (isSeedance ? SEEDANCE_FALLBACK.videos : 0)
|
||||||
|
const audios = declared(preset?.max_audios) ?? declared(model?.max_reference_audios) ?? (isSeedance ? SEEDANCE_FALLBACK.audios : 0)
|
||||||
if (!videos && !audios) return null
|
if (!videos && !audios) return null
|
||||||
return { images: declared(preset?.max_images) ?? 0, videos, audios }
|
return { images: declared(preset?.max_images) ?? declared(model?.max_reference_images) ?? 0, videos, audios }
|
||||||
}
|
}
|
||||||
|
|
||||||
// presetMap 把 /video-presets 的列表转成 key → preset,便于按模型 id 取预设。
|
// presetMap 把 /video-presets 的列表转成 key → preset,便于按模型 id 取预设。
|
||||||
|
|||||||
Reference in New Issue
Block a user