feat: generate videos in public api

apps/backend/src/api/routes/media.controller.ts

@@ -38,6 +38,16 @@ export class MediaController {
   deleteMedia(@GetOrgFromRequest() org: Organization, @Param('id') id: string) {
     return this._mediaService.deleteMedia(org.id, id);
   }
+
+  @Post('/generate-video')
+  generateVideo(
+    @GetOrgFromRequest() org: Organization,
+    @Body() body: VideoDto
+  ) {
+    console.log('hello');
+    return this._mediaService.generateVideo(org, body);
+  }
+
   @Post('/generate-image')
   async generateImage(
     @GetOrgFromRequest() org: Organization,
@@ -187,13 +197,4 @@ export class MediaController {
   ) {
     return this._mediaService.generateVideoAllowed(org, type);
   }
-
-  @Post('/generate-video/:type')
-  generateVideo(
-    @GetOrgFromRequest() org: Organization,
-    @Body() body: VideoDto,
-    @Param('type') type: string
-  ) {
-    return this._mediaService.generateVideo(org, body, type);
-  }
 }

apps/backend/src/api/routes/stripe.controller.ts

@@ -23,6 +23,7 @@ export class StripeController {
   stripeConnect(@Req() req: RawBodyRequest<Request>) {
     const event = this._stripeService.validateRequest(
       req.rawBody,
+      // @ts-ignore
       req.headers['stripe-signature'],
       process.env.STRIPE_SIGNING_KEY_CONNECT
     );
@@ -35,8 +36,6 @@ export class StripeController {
     }
 
     switch (event.type) {
-      case 'checkout.session.completed':
-        return this._stripeService.updateOrder(event);
       case 'account.updated':
         return this._stripeService.updateAccount(event);
       default:
@@ -48,6 +47,7 @@ export class StripeController {
   stripe(@Req() req: RawBodyRequest<Request>) {
     const event = this._stripeService.validateRequest(
       req.rawBody,
+      // @ts-ignore
       req.headers['stripe-signature'],
       process.env.STRIPE_SIGNING_KEY
     );
@@ -66,8 +66,6 @@ export class StripeController {
       switch (event.type) {
         case 'invoice.payment_succeeded':
           return this._stripeService.paymentSucceeded(event);
-        case 'checkout.session.completed':
-          return this._stripeService.updateOrder(event);
         case 'account.updated':
           return this._stripeService.updateAccount(event);
         case 'customer.subscription.created':

apps/backend/src/public-api/routes/v1/public.integrations.controller.ts

@@ -20,7 +20,11 @@ import { FileInterceptor } from '@nestjs/platform-express';
 import { UploadFactory } from '@gitroom/nestjs-libraries/upload/upload.factory';
 import { MediaService } from '@gitroom/nestjs-libraries/database/prisma/media/media.service';
 import { GetPostsDto } from '@gitroom/nestjs-libraries/dtos/posts/get.posts.dto';
-import { AuthorizationActions, Sections } from '@gitroom/backend/services/auth/permissions/permission.exception.class';
+import {
+  AuthorizationActions,
+  Sections,
+} from '@gitroom/backend/services/auth/permissions/permission.exception.class';
+import { VideoDto } from '@gitroom/nestjs-libraries/dtos/videos/video.dto';
 
 @ApiTags('Public API')
 @Controller('/public/v1')
@@ -69,7 +73,11 @@ export class PublicIntegrationsController {
     @GetOrgFromRequest() org: Organization,
     @Body() rawBody: any
   ) {
-    const body = await this._postsService.mapTypeToPost(rawBody, org.id, rawBody.type === 'draft');
+    const body = await this._postsService.mapTypeToPost(
+      rawBody,
+      org.id,
+      rawBody.type === 'draft'
+    );
     body.type = rawBody.type;
 
     console.log(JSON.stringify(body, null, 2));
@@ -87,7 +95,7 @@ export class PublicIntegrationsController {
 
   @Get('/is-connected')
   async getActiveIntegrations(@GetOrgFromRequest() org: Organization) {
-    return {connected: true};
+    return { connected: true };
   }
 
   @Get('/integrations')
@@ -109,4 +117,12 @@ export class PublicIntegrationsController {
       })
     );
   }
+
+  @Post('/generate-video')
+  generateVideo(
+    @GetOrgFromRequest() org: Organization,
+    @Body() body: VideoDto
+  ) {
+    return this._mediaService.generateVideo(org, body);
+  }
 }

apps/frontend/src/components/launches/ai.video.tsx

@@ -49,10 +49,10 @@ export const Modal: FC<{
       return ;
     }
     try {
-      const image = await fetch(`/media/generate-video/${type.identifier}`, {
+      const image = await fetch(`/media/generate-video`, {
         method: 'POST',
         body: JSON.stringify({
-          prompt: [{ type: 'prompt', value }],
+          type: type.identifier,
           output: position,
           customParams,
         }),

apps/frontend/src/components/videos/providers/image-text-slides.provider.tsx

@@ -5,6 +5,7 @@ import useSWR from 'swr';
 import { useFormContext } from 'react-hook-form';
 import { Button } from '@gitroom/react/form/button';
 import clsx from 'clsx';
+import { useVideo } from '@gitroom/frontend/components/videos/video.context.wrapper';
 
 export interface Voices {
   voices: Voice[];
@@ -22,6 +23,11 @@ const VoiceSelector: FC = () => {
   const [currentlyPlaying, setCurrentlyPlaying] = useState<string | null>(null);
   const [loadingVoice, setLoadingVoice] = useState<string | null>(null);
   const audioRef = useRef<HTMLAudioElement | null>(null);
+  const { value } = useVideo();
+
+  register('prompt', {
+    value,
+  });
 
   const loadVideos = useCallback(() => {
     return videoFunction('loadVoices', {});

apps/frontend/src/components/videos/providers/veo3.provider.tsx

@@ -14,7 +14,6 @@ export interface Voice {
 const VEO3Settings: FC = () => {
   const { register, watch, setValue, formState } = useFormContext();
   const { value } = useVideo();
-  const [videoValue, setVideoValue] = useState(value);
 
   const media = register('media', {
     value: [],

libraries/helpers/src/swagger/load.swagger.ts

@@ -3,7 +3,7 @@ import { INestApplication } from '@nestjs/common';
 
 export const loadSwagger = (app: INestApplication) => {
   const config = new DocumentBuilder()
-    .setTitle('crosspublic Swagger file')
+    .setTitle('Postiz Swagger file')
     .setDescription('API description')
     .setVersion('1.0')
     .build();

libraries/nestjs-libraries/src/database/prisma/media/media.service.ts

@@ -78,7 +78,7 @@ export class MediaService {
     return true;
   }
 
-  async generateVideo(org: Organization, body: VideoDto, type: string) {
+  async generateVideo(org: Organization, body: VideoDto) {
     const totalCredits = await this._subscriptionService.checkCredits(
       org,
       'ai_videos'
@@ -90,17 +90,16 @@ export class MediaService {
       });
     }
 
-    const video = this._videoManager.getVideoByName(type);
+    const video = this._videoManager.getVideoByName(body.type);
     if (!video) {
-      throw new Error(`Video type ${type} not found`);
+      throw new Error(`Video type ${body.type} not found`);
     }
 
     if (!video.trial && org.isTrailing) {
       throw new HttpException('This video is not available in trial mode', 406);
     }
 
-    const loadedData = await video.instance.process(
-      body.prompt,
+    const loadedData = await video.instance.processAndValidate(
       body.output,
       body.customParams
     );

libraries/nestjs-libraries/src/dtos/videos/video.dto.ts

@@ -1,19 +1,31 @@
-import { Type } from 'class-transformer';
-import { IsArray, IsIn, IsString, ValidateNested } from 'class-validator';
+import {
+  IsIn, Validate, ValidationArguments, ValidatorConstraint, ValidatorConstraintInterface
+} from 'class-validator';
+import { VideoAbstract } from '@gitroom/nestjs-libraries/videos/video.interface';
 
-export class Prompt {
-  @IsIn(['prompt', 'image'])
-  type: 'prompt' | 'image';
+@ValidatorConstraint({ name: 'checkInRuntime', async: false })
+export class ValidIn implements ValidatorConstraintInterface {
+  private _load() {
+    return (Reflect.getMetadata('video', VideoAbstract) || [])
+      .filter((f: any) => f.available)
+      .map((p: any) => p.identifier);
+  }
 
-  @IsString()
-  value: string;
+  validate(text: string, args: ValidationArguments) {
+    // Check if the text is in the list of valid video types
+    const validTypes = this._load();
+    return validTypes.includes(text);
+  }
+
+  defaultMessage(args: ValidationArguments) {
+    // here you can provide default error message if validation failed
+    return 'type must be any of: ' + this._load().join(', ');
+  }
 }
 
 export class VideoDto {
-  @ValidateNested({ each: true })
-  @IsArray()
-  @Type(() => Prompt)
-  prompt: Prompt[];
+  @Validate(ValidIn)
+  type: string;
 
   @IsIn(['vertical', 'horizontal'])
   output: 'vertical' | 'horizontal';

libraries/nestjs-libraries/src/services/stripe.service.ts

@@ -685,27 +685,6 @@ export class StripeService {
     return { ok: true };
   }
 
-  async updateOrder(event: Stripe.CheckoutSessionCompletedEvent) {
-    if (event?.data?.object?.metadata?.type !== 'marketplace') {
-      return { ok: true };
-    }
-
-    const { orderId } = event?.data?.object?.metadata || { orderId: '' };
-    if (!orderId) {
-      return;
-    }
-
-    const charge = (
-      await stripe.paymentIntents.retrieve(
-        event.data.object.payment_intent as string
-      )
-    ).latest_charge;
-    const id = typeof charge === 'string' ? charge : charge?.id;
-
-    await this._messagesService.changeOrderStatus(orderId, 'ACCEPTED', id);
-    return { ok: true };
-  }
-
   async payout(
     orderId: string,
     charge: string,

libraries/nestjs-libraries/src/videos/images-slides/images.slides.ts

@@ -1,6 +1,5 @@
 import { OpenaiService } from '@gitroom/nestjs-libraries/openai/openai.service';
 import {
-  Prompt,
   URL,
   Video,
   VideoAbstract,
@@ -14,6 +13,7 @@ import { stringifySync } from 'subtitle';
 
 import pLimit from 'p-limit';
 import { FalService } from '@gitroom/nestjs-libraries/openai/fal.service';
+import { IsString } from 'class-validator';
 const limit = pLimit(2);
 
 const transloadit = new Transloadit({
@@ -26,6 +26,14 @@ async function getAudioDuration(buffer: Buffer): Promise<number> {
   return metadata.format.duration || 0;
 }
 
+class Params {
+  @IsString()
+  voice: string;
+
+  @IsString()
+  prompt: string;
+}
+
 @Video({
   identifier: 'image-text-slides',
   title: 'Image Text Slides',
@@ -39,7 +47,8 @@ async function getAudioDuration(buffer: Buffer): Promise<number> {
     !!process.env.OPENAI_API_KEY &&
     !!process.env.FAL_KEY,
 })
-export class ImagesSlides extends VideoAbstract {
+export class ImagesSlides extends VideoAbstract<Params> {
+  override dto = Params;
   private storage = UploadFactory.createStorage();
   constructor(
     private _openaiService: OpenaiService,
@@ -49,12 +58,11 @@ export class ImagesSlides extends VideoAbstract {
   }
 
   async process(
-    prompt: Prompt[],
     output: 'vertical' | 'horizontal',
-    customParams: { voice: string }
+    customParams: Params
   ): Promise<URL> {
     const list = await this._openaiService.generateSlidesFromText(
-      prompt[0].value
+      customParams.prompt
     );
 
     const generated = await Promise.all(
@@ -87,7 +95,7 @@ export class ImagesSlides extends VideoAbstract {
                       },
                       body: JSON.stringify({
                         text: current.voiceText,
-                        model_id: 'eleven_multilingual_v2'
+                        model_id: 'eleven_multilingual_v2',
                       }),
                     }
                   )

libraries/nestjs-libraries/src/videos/veo3/veo3.ts

@@ -1,10 +1,29 @@
 import {
-  Prompt,
   URL,
   Video,
   VideoAbstract,
 } from '@gitroom/nestjs-libraries/videos/video.interface';
 import { timer } from '@gitroom/helpers/utils/timer';
+import { ArrayMaxSize, IsArray, IsString, ValidateNested } from 'class-validator';
+import { Type } from 'class-transformer';
+
+class Image {
+  @IsString()
+  id: string;
+
+  @IsString()
+  path: string;
+}
+class Params {
+  @IsString()
+  prompt: string;
+
+  @Type(() => Image)
+  @ValidateNested({ each: true })
+  @IsArray()
+  @ArrayMaxSize(3)
+  images: Image[];
+}
 
 @Video({
   identifier: 'veo3',
@@ -14,11 +33,11 @@ import { timer } from '@gitroom/helpers/utils/timer';
   trial: false,
   available: !!process.env.KIEAI_API_KEY,
 })
-export class Veo3 extends VideoAbstract {
+export class Veo3 extends VideoAbstract<Params> {
+  override dto = Params;
   async process(
-    prompt: Prompt[],
     output: 'vertical' | 'horizontal',
-    customParams: { prompt: string; images: { id: string; path: string }[] }
+    customParams: Params
   ): Promise<URL> {
     console.log({
       headers: {

libraries/nestjs-libraries/src/videos/video.interface.ts

@@ -1,17 +1,33 @@
-import { Injectable } from '@nestjs/common';
-
-export interface Prompt {
-  type: 'prompt' | 'image';
-  value: string;
-}
+import { Injectable, Type, ValidationPipe } from '@nestjs/common';
 
 export type URL = string;
 
-export abstract class VideoAbstract {
-  abstract process(
-    prompt: Prompt[],
+export abstract class VideoAbstract<T> {
+  dto: Type<T>;
+
+  async processAndValidate(
     output: 'vertical' | 'horizontal',
-    customParams?: any
+    customParams?: T
+  ) {
+    const validationPipe = new ValidationPipe({
+      skipMissingProperties: false,
+      transform: true,
+      transformOptions: {
+        enableImplicitConversion: true,
+      },
+    });
+
+    const transformed = await validationPipe.transform(customParams, {
+      type: 'body',
+      metatype: this.dto,
+    });
+
+    return this.process(output, transformed);
+  }
+
+  protected abstract process(
+    output: 'vertical' | 'horizontal',
+    customParams?: T
   ): Promise<URL>;
 }
 

libraries/nestjs-libraries/src/videos/video.manager.ts

@@ -24,7 +24,7 @@ export class VideoManager {
 
   getVideoByName(
     identifier: string
-  ): (VideoParams & { instance: VideoAbstract }) | undefined {
+  ): (VideoParams & { instance: VideoAbstract<any> }) | undefined {
     const video = (Reflect.getMetadata('video', VideoAbstract) || []).find(
       (p: any) => p.identifier === identifier
     );