413 lines
12 KiB
Go
413 lines
12 KiB
Go
package leonardo
|
|
|
|
import (
|
|
"context"
|
|
"encoding/binary"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// qGenerationVideos polls one video generation's status AND its produced clip in
|
|
// a single round-trip (motionMP4URL carries the mp4).
|
|
const qGenerationVideos = `query GenerationVideos($where: generations_bool_exp = {}) {
|
|
generations(where: $where) {
|
|
id
|
|
status
|
|
generated_images {
|
|
id
|
|
motionMP4URL
|
|
__typename
|
|
}
|
|
__typename
|
|
}
|
|
}`
|
|
|
|
// VideoAssets are the decoded reference assets a video request can carry:
|
|
// image references (strength MID), one audio track and video references. The
|
|
// caller enforces the per-type caps; durations are derived from the bytes here
|
|
// because Leonardo requires them for audio/video guidances.
|
|
type VideoAssets struct {
|
|
Images [][]byte
|
|
Audios [][]byte
|
|
Videos [][]byte
|
|
}
|
|
|
|
// GenerateVideo runs the Leonardo video pipeline (seedance-2.0 / -fast, hailuo) against
|
|
// one account cookie: upload every reference asset, submit the Generate mutation
|
|
// as a PRIVATE generation, poll until COMPLETE, then optionally download the mp4.
|
|
func (c *Client) GenerateVideo(ctx context.Context, cookie, model, prompt string, width, height, durationSeconds int, refs VideoAssets, downloadResult bool) ([]byte, map[string]any, error) {
|
|
sess, err := c.GetSession(ctx, cookie)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
if strings.TrimSpace(model) == "" {
|
|
model = "seedance-2.0-fast"
|
|
}
|
|
|
|
guidances := map[string]any{}
|
|
var imageRefs []map[string]any
|
|
for _, img := range refs.Images {
|
|
if len(img) == 0 {
|
|
continue
|
|
}
|
|
uploadID, upErr := c.uploadInitImage(ctx, cookie, assetExtension(img, "png"), img)
|
|
if upErr != nil {
|
|
return nil, nil, upErr
|
|
}
|
|
imageRefs = append(imageRefs, map[string]any{
|
|
"image": map[string]any{"id": uploadID, "type": "UPLOADED"},
|
|
"strength": "MID",
|
|
})
|
|
}
|
|
if len(imageRefs) > 0 {
|
|
guidances["image_reference"] = imageRefs
|
|
}
|
|
var videoRefs []map[string]any
|
|
for _, vid := range refs.Videos {
|
|
if len(vid) == 0 {
|
|
continue
|
|
}
|
|
uploadID, upErr := c.uploadAsset(ctx, cookie, assetExtension(vid, "mp4"), vid)
|
|
if upErr != nil {
|
|
return nil, nil, upErr
|
|
}
|
|
video := map[string]any{"id": uploadID, "type": "UPLOADED"}
|
|
if secs := MediaDurationSeconds(vid); secs > 0 {
|
|
video["duration"] = secs
|
|
}
|
|
videoRefs = append(videoRefs, map[string]any{"video": video})
|
|
}
|
|
if len(videoRefs) > 0 {
|
|
guidances["video_reference_base"] = videoRefs
|
|
}
|
|
// Leonardo rejects an audio reference that isn't paired with an image or
|
|
// video reference (audio_reference_only_compatible_with_image_or_video_reference).
|
|
var audioRefs []map[string]any
|
|
for _, aud := range refs.Audios {
|
|
if len(aud) == 0 {
|
|
continue
|
|
}
|
|
if len(imageRefs) == 0 && len(videoRefs) == 0 {
|
|
return nil, nil, errors.New("leonardo: audio reference requires an image or video reference")
|
|
}
|
|
uploadID, upErr := c.uploadAsset(ctx, cookie, assetExtension(aud, "mp3"), aud)
|
|
if upErr != nil {
|
|
return nil, nil, upErr
|
|
}
|
|
audio := map[string]any{"id": uploadID, "type": "UPLOADED"}
|
|
if secs := MediaDurationSeconds(aud); secs > 0 {
|
|
audio["duration"] = secs
|
|
}
|
|
audioRefs = append(audioRefs, map[string]any{"audio": audio})
|
|
}
|
|
if len(audioRefs) > 0 {
|
|
guidances["audio_reference"] = audioRefs
|
|
}
|
|
|
|
parameters := map[string]any{
|
|
"height": height,
|
|
"width": width,
|
|
"duration": durationSeconds,
|
|
"motion_has_audio": true,
|
|
"quantity": 1,
|
|
"prompt": prompt,
|
|
"guidances": guidances,
|
|
}
|
|
// seedance 走随机种子;hailuo 的生成页 seedEnabled=false,请求里不带 seed。
|
|
if !strings.HasPrefix(model, "hailuo") {
|
|
parameters["seed"] = -1
|
|
}
|
|
genReq := map[string]any{
|
|
"operationName": "Generate",
|
|
"query": mGenerate,
|
|
"variables": map[string]any{
|
|
"request": map[string]any{
|
|
"model": model,
|
|
// 私有生成：不进公开 feed。
|
|
"public": false,
|
|
"parameters": parameters,
|
|
},
|
|
},
|
|
}
|
|
payload, _ := json.Marshal(genReq)
|
|
body, err := c.callGraphQL(ctx, cookie, payload, true, "generate-video")
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
var genResp struct {
|
|
Data struct {
|
|
Generate struct {
|
|
GenerationID string `json:"generationId"`
|
|
} `json:"generate"`
|
|
} `json:"data"`
|
|
}
|
|
if err := json.Unmarshal(body, &genResp); err != nil {
|
|
return nil, nil, fmt.Errorf("%w: generate non-json", ErrTemporaryUpstream)
|
|
}
|
|
genID := strings.TrimSpace(genResp.Data.Generate.GenerationID)
|
|
if genID == "" {
|
|
return nil, nil, fmt.Errorf("%w: no generationId: %s", ErrTemporaryUpstream, clip(body, 200))
|
|
}
|
|
|
|
videoURL, err := c.pollVideo(ctx, cookie, genID)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
info := map[string]any{
|
|
"generation_id": genID,
|
|
"video_url": videoURL,
|
|
"user_id": sess.UserID,
|
|
}
|
|
if !downloadResult {
|
|
return nil, info, nil
|
|
}
|
|
data, err := c.downloadImage(ctx, videoURL)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
return data, info, nil
|
|
}
|
|
|
|
// pollVideo polls one generation until COMPLETE (returning the mp4 url) or
|
|
// FAILED. Temporary upstream hiccups don't abort the wait — only ctx/deadline do.
|
|
func (c *Client) pollVideo(ctx context.Context, cookie, genID string) (string, error) {
|
|
payload, _ := json.Marshal(map[string]any{
|
|
"operationName": "GenerationVideos",
|
|
"query": qGenerationVideos,
|
|
"variables": map[string]any{
|
|
"where": map[string]any{"id": map[string]any{"_in": []string{genID}}},
|
|
},
|
|
})
|
|
|
|
ticker := time.NewTicker(5 * time.Second)
|
|
defer ticker.Stop()
|
|
deadline := time.Now().Add(10 * time.Minute)
|
|
if dl, ok := ctx.Deadline(); ok {
|
|
deadline = dl.Add(-60 * time.Second)
|
|
}
|
|
|
|
for {
|
|
body, err := c.callGraphQL(ctx, cookie, payload, false, "poll-video")
|
|
if errors.Is(err, ErrAuth) {
|
|
return "", err
|
|
}
|
|
if err == nil {
|
|
var pr struct {
|
|
Data struct {
|
|
Generations []struct {
|
|
Status string `json:"status"`
|
|
GeneratedImages []struct {
|
|
MotionMP4URL string `json:"motionMP4URL"`
|
|
} `json:"generated_images"`
|
|
} `json:"generations"`
|
|
} `json:"data"`
|
|
}
|
|
if err := json.Unmarshal(body, &pr); err == nil && len(pr.Data.Generations) > 0 {
|
|
g := pr.Data.Generations[0]
|
|
switch strings.ToUpper(g.Status) {
|
|
case "COMPLETE":
|
|
for _, img := range g.GeneratedImages {
|
|
if u := strings.TrimSpace(img.MotionMP4URL); u != "" {
|
|
return u, nil
|
|
}
|
|
}
|
|
return "", fmt.Errorf("%w: complete but no video url", ErrTemporaryUpstream)
|
|
case "FAILED":
|
|
return "", fmt.Errorf("%w: generation failed", ErrTemporaryUpstream)
|
|
}
|
|
}
|
|
}
|
|
|
|
if time.Now().After(deadline) {
|
|
return "", fmt.Errorf("%w: generation timed out", ErrTemporaryUpstream)
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
return "", ctx.Err()
|
|
case <-ticker.C:
|
|
}
|
|
}
|
|
}
|
|
|
|
// assetExtension sniffs the upload extension Leonardo expects for a reference
|
|
// asset; fallback is used when the bytes aren't recognized.
|
|
func assetExtension(data []byte, fallback string) string {
|
|
n := len(data)
|
|
switch {
|
|
case n >= 8 && string(data[1:4]) == "PNG":
|
|
return "png"
|
|
case n >= 3 && data[0] == 0xFF && data[1] == 0xD8 && data[2] == 0xFF:
|
|
return "jpg"
|
|
case n >= 12 && string(data[0:4]) == "RIFF" && string(data[8:12]) == "WEBP":
|
|
return "webp"
|
|
case n >= 12 && string(data[4:8]) == "ftyp":
|
|
if n >= 12 && strings.HasPrefix(string(data[8:12]), "qt") {
|
|
return "mov"
|
|
}
|
|
if n >= 12 && strings.HasPrefix(string(data[8:12]), "M4A") {
|
|
return "m4a"
|
|
}
|
|
return "mp4"
|
|
case n >= 4 && data[0] == 0x1A && data[1] == 0x45 && data[2] == 0xDF && data[3] == 0xA3:
|
|
return "webm"
|
|
case n >= 3 && string(data[0:3]) == "ID3":
|
|
return "mp3"
|
|
case n >= 2 && data[0] == 0xFF && (data[1]&0xE0) == 0xE0:
|
|
return "mp3"
|
|
case n >= 12 && string(data[0:4]) == "RIFF" && string(data[8:12]) == "WAVE":
|
|
return "wav"
|
|
}
|
|
return fallback
|
|
}
|
|
|
|
// MediaDurationSeconds returns a media asset's duration in seconds (0 when it
|
|
// can't be determined). Leonardo's audio_reference / video_reference_base
|
|
// guidances carry the duration, and the caller validates video lengths with it,
|
|
// so it's derived from the uploaded bytes: mp4/mov via the mvhd box, wav via the
|
|
// fmt byte rate, mp3 from the first frame's bitrate.
|
|
func MediaDurationSeconds(data []byte) float64 {
|
|
if secs := mp4Duration(data); secs > 0 {
|
|
return secs
|
|
}
|
|
if secs := wavDuration(data); secs > 0 {
|
|
return secs
|
|
}
|
|
return mp3Duration(data)
|
|
}
|
|
|
|
// mp4Duration walks the ISOBMFF box tree to moov/mvhd and reads timescale+duration.
|
|
func mp4Duration(data []byte) float64 {
|
|
if len(data) < 12 || string(data[4:8]) != "ftyp" {
|
|
return 0
|
|
}
|
|
moov := findBox(data, "moov")
|
|
if moov == nil {
|
|
return 0
|
|
}
|
|
mvhd := findBox(moov, "mvhd")
|
|
if len(mvhd) < 20 {
|
|
return 0
|
|
}
|
|
version := mvhd[0]
|
|
if version == 1 {
|
|
if len(mvhd) < 32 {
|
|
return 0
|
|
}
|
|
timescale := binary.BigEndian.Uint32(mvhd[20:24])
|
|
duration := binary.BigEndian.Uint64(mvhd[24:32])
|
|
if timescale == 0 {
|
|
return 0
|
|
}
|
|
return float64(duration) / float64(timescale)
|
|
}
|
|
timescale := binary.BigEndian.Uint32(mvhd[12:16])
|
|
duration := binary.BigEndian.Uint32(mvhd[16:20])
|
|
if timescale == 0 {
|
|
return 0
|
|
}
|
|
return float64(duration) / float64(timescale)
|
|
}
|
|
|
|
// findBox returns the payload of the first box named typ among data's boxes
|
|
// (callers descend one level at a time by passing a parent's payload back in).
|
|
func findBox(data []byte, typ string) []byte {
|
|
for off := 0; off+8 <= len(data); {
|
|
size := int(binary.BigEndian.Uint32(data[off : off+4]))
|
|
name := string(data[off+4 : off+8])
|
|
header := 8
|
|
if size == 1 { // 64-bit size
|
|
if off+16 > len(data) {
|
|
return nil
|
|
}
|
|
size = int(binary.BigEndian.Uint64(data[off+8 : off+16]))
|
|
header = 16
|
|
} else if size == 0 {
|
|
size = len(data) - off
|
|
}
|
|
if size < header || off+size > len(data) {
|
|
return nil
|
|
}
|
|
if name == typ {
|
|
return data[off+header : off+size]
|
|
}
|
|
off += size
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// wavDuration reads the RIFF fmt chunk's byte rate and the data chunk size.
|
|
func wavDuration(data []byte) float64 {
|
|
if len(data) < 44 || string(data[0:4]) != "RIFF" || string(data[8:12]) != "WAVE" {
|
|
return 0
|
|
}
|
|
byteRate := 0
|
|
dataSize := 0
|
|
for off := 12; off+8 <= len(data); {
|
|
name := string(data[off : off+4])
|
|
size := int(binary.LittleEndian.Uint32(data[off+4 : off+8]))
|
|
body := off + 8
|
|
if size < 0 || body > len(data) {
|
|
break
|
|
}
|
|
switch name {
|
|
case "fmt ":
|
|
if body+16 <= len(data) {
|
|
byteRate = int(binary.LittleEndian.Uint32(data[body+8 : body+12]))
|
|
}
|
|
case "data":
|
|
dataSize = size
|
|
if body+size > len(data) {
|
|
dataSize = len(data) - body
|
|
}
|
|
}
|
|
off = body + size + size%2
|
|
}
|
|
if byteRate <= 0 || dataSize <= 0 {
|
|
return 0
|
|
}
|
|
return float64(dataSize) / float64(byteRate)
|
|
}
|
|
|
|
// mp3Bitrates are the Layer III bitrate tables (kbps) indexed by the frame
|
|
// header's bitrate index: 1 = MPEG1, 2 = MPEG2/2.5.
|
|
var mp3Bitrates = map[int][]int{
|
|
1: {0, 32, 40, 48, 56, 64, 80, 96, 112, 128, 160, 192, 224, 256, 320, 0},
|
|
2: {0, 8, 16, 24, 32, 40, 48, 56, 64, 80, 96, 112, 128, 144, 160, 0},
|
|
}
|
|
|
|
// mp3Duration estimates the duration from the first frame header's bitrate
|
|
// (constant-bitrate assumption — good enough for the guidance duration field).
|
|
func mp3Duration(data []byte) float64 {
|
|
off := 0
|
|
if len(data) >= 10 && string(data[0:3]) == "ID3" {
|
|
// syncsafe int: 7 bits per byte
|
|
tagSize := int(data[6]&0x7F)<<21 | int(data[7]&0x7F)<<14 | int(data[8]&0x7F)<<7 | int(data[9]&0x7F)
|
|
off = 10 + tagSize
|
|
}
|
|
for ; off+4 <= len(data); off++ {
|
|
if data[off] != 0xFF || data[off+1]&0xE0 != 0xE0 {
|
|
continue
|
|
}
|
|
versionBits := (data[off+1] >> 3) & 0x03
|
|
layerBits := (data[off+1] >> 1) & 0x03
|
|
if layerBits != 0x01 { // Layer III only
|
|
continue
|
|
}
|
|
table := 1
|
|
if versionBits != 0x03 { // MPEG2 / 2.5
|
|
table = 2
|
|
}
|
|
idx := int((data[off+2] >> 4) & 0x0F)
|
|
rates := mp3Bitrates[table]
|
|
if idx <= 0 || idx >= len(rates) || rates[idx] == 0 {
|
|
continue
|
|
}
|
|
kbps := rates[idx]
|
|
return float64(len(data)-off) * 8 / float64(kbps*1000)
|
|
}
|
|
return 0
|
|
}
|