diff --git a/apps/frontend/src/components/layout/settings.component.tsx b/apps/frontend/src/components/layout/settings.component.tsx index 0850f213..f8f39311 100644 --- a/apps/frontend/src/components/layout/settings.component.tsx +++ b/apps/frontend/src/components/layout/settings.component.tsx @@ -129,7 +129,9 @@ export const SettingsPopup: FC<{ {t('auto_post', 'Auto Post')} )} - {t('sets', 'Sets')} + {user?.tier.current !== 'FREE' && ( + {t('sets', 'Sets')} + )} {user?.tier.current !== 'FREE' && ( {t('signatures', 'Signatures')} diff --git a/libraries/nestjs-libraries/src/videos/images-slides/images.slides.ts b/libraries/nestjs-libraries/src/videos/images-slides/images.slides.ts index 9aa7ebad..1d48f1de 100644 --- a/libraries/nestjs-libraries/src/videos/images-slides/images.slides.ts +++ b/libraries/nestjs-libraries/src/videos/images-slides/images.slides.ts @@ -31,6 +31,7 @@ async function getAudioDuration(buffer: Buffer): Promise { title: 'Image Text Slides', description: 'Generate videos slides from images and text', placement: 'text-to-image', + trial: true, available: !!process.env.ELEVENSLABS_API_KEY && !!process.env.TRANSLOADIT_AUTH && diff --git a/libraries/nestjs-libraries/src/videos/veo3/veo3.ts b/libraries/nestjs-libraries/src/videos/veo3/veo3.ts new file mode 100644 index 00000000..28bc0d56 --- /dev/null +++ b/libraries/nestjs-libraries/src/videos/veo3/veo3.ts @@ -0,0 +1,244 @@ +import { OpenaiService } from '@gitroom/nestjs-libraries/openai/openai.service'; +import { + Prompt, + URL, + Video, + VideoAbstract, +} from '@gitroom/nestjs-libraries/videos/video.interface'; +import { chunk } from 'lodash'; +import Transloadit from 'transloadit'; +import { UploadFactory } from '@gitroom/nestjs-libraries/upload/upload.factory'; +import { Readable } from 'stream'; +import { parseBuffer } from 'music-metadata'; +import { stringifySync } from 'subtitle'; + +import pLimit from 'p-limit'; +import { FalService } from '@gitroom/nestjs-libraries/openai/fal.service'; +const limit = pLimit(2); + +const transloadit = new Transloadit({ + authKey: process.env.TRANSLOADIT_AUTH || 'just empty text', + authSecret: process.env.TRANSLOADIT_SECRET || 'just empty text', +}); + +async function getAudioDuration(buffer: Buffer): Promise { + const metadata = await parseBuffer(buffer, 'audio/mpeg'); + return metadata.format.duration || 0; +} + +@Video({ + identifier: 'veo3', + title: 'Veo3 (Audio + Video)', + description: 'Generate videos with the most advanced video model.', + placement: 'text-to-image', + trial: false, + available: + !!process.env.TRANSLOADIT_AUTH && + !!process.env.TRANSLOADIT_SECRET && + !!process.env.OPENAI_API_KEY && + !!process.env.KIEAI_API_KEY, +}) +export class Veo3 extends VideoAbstract { + private storage = UploadFactory.createStorage(); + constructor( + private _openaiService: OpenaiService, + private _falService: FalService + ) { + super(); + } + + async process( + prompt: Prompt[], + output: 'vertical' | 'horizontal', + customParams: { voice: string } + ): Promise { + const list = await this._openaiService.generateSlidesFromText( + prompt[0].value + ); + + const generated = await Promise.all( + list.reduce((all, current) => { + all.push( + new Promise(async (res) => { + res({ + len: 0, + url: await this._falService.generateImageFromText( + 'ideogram/v2', + current.imagePrompt, + output === 'vertical' + ), + }); + }) + ); + + all.push( + new Promise(async (res) => { + const buffer = Buffer.from( + await ( + await limit(() => + fetch( + `https://api.elevenlabs.io/v1/text-to-speech/${customParams.voice}?output_format=mp3_44100_128`, + { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'xi-api-key': process.env.ELEVENSLABS_API_KEY || '', + }, + body: JSON.stringify({ + text: current.voiceText, + model_id: 'eleven_multilingual_v2' + }), + } + ) + ) + ).arrayBuffer() + ); + + const { path } = await this.storage.uploadFile({ + buffer, + mimetype: 'audio/mp3', + size: buffer.length, + path: '', + fieldname: '', + destination: '', + stream: new Readable(), + filename: '', + originalname: '', + encoding: '', + }); + + res({ + len: await getAudioDuration(buffer), + url: + path.indexOf('http') === -1 + ? process.env.FRONTEND_URL + + '/' + + process.env.NEXT_PUBLIC_UPLOAD_STATIC_DIRECTORY + + path + : path, + }); + }) + ); + + return all; + }, [] as Promise[]) + ); + + const split = chunk(generated, 2); + + const srt = stringifySync( + list + .reduce((all, current, index) => { + const start = all.length ? all[all.length - 1].end : 0; + const end = start + split[index][1].len * 1000 + 1000; + all.push({ + start: start, + end: end, + text: current.voiceText, + }); + + return all; + }, [] as { start: number; end: number; text: string }[]) + .map((item) => ({ + type: 'cue', + data: item, + })), + { format: 'SRT' } + ); + + const { results } = await transloadit.createAssembly({ + uploads: { + 'subtitles.srt': srt, + }, + waitForCompletion: true, + params: { + steps: { + ...split.reduce((all, current, index) => { + all[`image${index}`] = { + robot: '/http/import', + url: current[0].url, + }; + all[`audio${index}`] = { + robot: '/http/import', + url: current[1].url, + }; + all[`merge${index}`] = { + use: [ + { + name: `image${index}`, + as: 'image', + }, + { + name: `audio${index}`, + as: 'audio', + }, + ], + robot: '/video/merge', + duration: current[1].len + 1, + audio_delay: 0.5, + preset: 'hls-1080p', + resize_strategy: 'min_fit', + loop: true, + }; + return all; + }, {} as any), + concatenated: { + robot: '/video/concat', + result: false, + video_fade_seconds: 0.5, + use: split.map((p, index) => ({ + name: `merge${index}`, + as: `video_${index + 1}`, + })), + }, + subtitled: { + robot: '/video/subtitle', + result: true, + preset: 'hls-1080p', + use: { + bundle_steps: true, + steps: [ + { + name: 'concatenated', + as: 'video', + }, + { + name: ':original', + as: 'subtitles', + }, + ], + }, + position: 'center', + font_size: 8, + subtitles_type: 'burned', + }, + }, + }, + }); + + return results.subtitled[0].url; + } + + async loadVoices(data: any) { + const { voices } = await ( + await fetch( + 'https://api.elevenlabs.io/v2/voices?page_size=40&category=premade', + { + method: 'GET', + headers: { + 'Content-Type': 'application/json', + 'xi-api-key': process.env.ELEVENSLABS_API_KEY || '', + }, + } + ) + ).json(); + + return { + voices: voices.map((voice: any) => ({ + id: voice.voice_id, + name: voice.name, + preview_url: voice.preview_url, + })), + }; + } +} diff --git a/libraries/nestjs-libraries/src/videos/video.interface.ts b/libraries/nestjs-libraries/src/videos/video.interface.ts index 1373347d..6daf6e37 100644 --- a/libraries/nestjs-libraries/src/videos/video.interface.ts +++ b/libraries/nestjs-libraries/src/videos/video.interface.ts @@ -21,6 +21,7 @@ export interface VideoParams { description: string; placement: 'text-to-image' | 'image-to-video' | 'video-to-video'; available: boolean; + trial: boolean; } export function Video(params: VideoParams) { diff --git a/libraries/nestjs-libraries/src/videos/video.manager.ts b/libraries/nestjs-libraries/src/videos/video.manager.ts index 87ec1ee5..bc6ab78a 100644 --- a/libraries/nestjs-libraries/src/videos/video.manager.ts +++ b/libraries/nestjs-libraries/src/videos/video.manager.ts @@ -17,6 +17,7 @@ export class VideoManager { title: p.title, description: p.description, placement: p.placement, + trial: p.trial, }) ); } diff --git a/libraries/nestjs-libraries/src/videos/video.module.ts b/libraries/nestjs-libraries/src/videos/video.module.ts index c7d8f06a..cced4cc5 100644 --- a/libraries/nestjs-libraries/src/videos/video.module.ts +++ b/libraries/nestjs-libraries/src/videos/video.module.ts @@ -1,10 +1,11 @@ import { Global, Module } from '@nestjs/common'; import { ImagesSlides } from '@gitroom/nestjs-libraries/videos/images-slides/images.slides'; import { VideoManager } from '@gitroom/nestjs-libraries/videos/video.manager'; +import { Veo3 } from '@gitroom/nestjs-libraries/videos/veo3/veo3'; @Global() @Module({ - providers: [ImagesSlides, VideoManager], + providers: [ImagesSlides, Veo3, VideoManager], get exports() { return this.providers; },