From 7d4351adeaca04dabb2d6235bb5d9ce213b96376 Mon Sep 17 00:00:00 2001 From: Nevo David Date: Mon, 30 Dec 2024 19:38:27 +0700 Subject: [PATCH] feat: ai image --- .../src/api/routes/billing.controller.ts | 4 +- .../src/api/routes/media.controller.ts | 59 +++++++-- .../src/api/routes/posts.controller.ts | 6 + .../components/launches/add.edit.model.tsx | 1 + .../src/components/launches/ai.image.tsx | 114 ++++++++++++++++++ .../providers/high.order.provider.tsx | 13 +- .../src/components/media/media.component.tsx | 85 +++++++------ .../database/prisma/media/media.service.ts | 8 +- .../notifications/notification.service.ts | 4 +- .../src/emails/email.interface.ts | 11 +- .../src/emails/resend.provider.ts | 10 +- .../src/openai/openai.service.ts | 43 ++++++- .../src/services/email.service.ts | 5 +- 13 files changed, 301 insertions(+), 62 deletions(-) create mode 100644 apps/frontend/src/components/launches/ai.image.tsx diff --git a/apps/backend/src/api/routes/billing.controller.ts b/apps/backend/src/api/routes/billing.controller.ts index f14c8161..5bf7f73b 100644 --- a/apps/backend/src/api/routes/billing.controller.ts +++ b/apps/backend/src/api/routes/billing.controller.ts @@ -61,12 +61,14 @@ export class BillingController { @Post('/cancel') async cancel( @GetOrgFromRequest() org: Organization, + @GetUserFromRequest() user: User, @Body() body: { feedback: string } ) { await this._notificationService.sendEmail( process.env.EMAIL_FROM_ADDRESS, 'Subscription Cancelled', - `Organization ${org.name} has cancelled their subscription because: ${body.feedback}` + `${user.name} from Organization ${org.name} has cancelled their subscription because: ${body.feedback}`, + user.email ); return this._stripeService.setToCancel(org.id); diff --git a/apps/backend/src/api/routes/media.controller.ts b/apps/backend/src/api/routes/media.controller.ts index 6e7719f3..d9a0b906 100644 --- a/apps/backend/src/api/routes/media.controller.ts +++ b/apps/backend/src/api/routes/media.controller.ts @@ -1,5 +1,15 @@ import { - Body, Controller, Get, Param, Post, Query, Req, Res, UploadedFile, UseInterceptors, UsePipes + Body, + Controller, + Get, + Param, + Post, + Query, + Req, + Res, + UploadedFile, + UseInterceptors, + UsePipes, } from '@nestjs/common'; import { Request, Response } from 'express'; import { GetOrgFromRequest } from '@gitroom/nestjs-libraries/user/org.from.request'; @@ -25,14 +35,35 @@ export class MediaController { async generateImage( @GetOrgFromRequest() org: Organization, @Req() req: Request, - @Body('prompt') prompt: string + @Body('prompt') prompt: string, + isPicturePrompt = false ) { const total = await this._subscriptionService.checkCredits(org); if (total.credits <= 0) { return false; } - return {output: 'data:image/png;base64,' + await this._mediaService.generateImage(prompt, org)}; + return { + output: + (isPicturePrompt ? '' : 'data:image/png;base64,') + + (await this._mediaService.generateImage(prompt, org, isPicturePrompt)), + }; + } + + @Post('/generate-image-with-prompt') + async generateImageFromText( + @GetOrgFromRequest() org: Organization, + @Req() req: Request, + @Body('prompt') prompt: string + ) { + const image = await this.generateImage(org, req, prompt, true); + if (!image) { + return false; + } + + const file = await this.storage.uploadSimple(image.output); + + return this._mediaService.saveFile(org.id, file.split('/').pop(), file); } @Post('/upload-server') @@ -43,7 +74,11 @@ export class MediaController { @UploadedFile() file: Express.Multer.File ) { const uploadedFile = await this.storage.uploadFile(file); - return this._mediaService.saveFile(org.id, uploadedFile.originalname, uploadedFile.path); + return this._mediaService.saveFile( + org.id, + uploadedFile.originalname, + uploadedFile.path + ); } @Post('/upload-simple') @@ -53,7 +88,11 @@ export class MediaController { @UploadedFile('file') file: Express.Multer.File ) { const getFile = await this.storage.uploadFile(file); - return this._mediaService.saveFile(org.id, getFile.originalname, getFile.path); + return this._mediaService.saveFile( + org.id, + getFile.originalname, + getFile.path + ); } @Post('/:endpoint') @@ -75,10 +114,14 @@ export class MediaController { // @ts-ignore const name = upload.Location.split('/').pop(); - // @ts-ignore - const saveFile = await this._mediaService.saveFile(org.id, name, upload.Location); + const saveFile = await this._mediaService.saveFile( + org.id, + name, + // @ts-ignore + upload.Location + ); - res.status(200).json({...upload, saved: saveFile}); + res.status(200).json({ ...upload, saved: saveFile }); // const filePath = // file.path.indexOf('http') === 0 // ? file.path diff --git a/apps/backend/src/api/routes/posts.controller.ts b/apps/backend/src/api/routes/posts.controller.ts index e02f743e..1aea47c7 100644 --- a/apps/backend/src/api/routes/posts.controller.ts +++ b/apps/backend/src/api/routes/posts.controller.ts @@ -45,6 +45,12 @@ export class PostsController { return this._messagesService.getMarketplaceAvailableOffers(org.id, id); } + @Post('/posts/generate-image') + @CheckPolicies([AuthorizationActions.Create, Sections.POSTS_PER_MONTH]) + generateImage(@Body() body: { text: string; type: string }) { + + } + @Get('/') async getPosts( @GetOrgFromRequest() org: Organization, diff --git a/apps/frontend/src/components/launches/add.edit.model.tsx b/apps/frontend/src/components/launches/add.edit.model.tsx index 147b21b7..2f31271e 100644 --- a/apps/frontend/src/components/launches/add.edit.model.tsx +++ b/apps/frontend/src/components/launches/add.edit.model.tsx @@ -655,6 +655,7 @@ export const AddEditModal: FC<{
void; +}> = (props) => { + const { value, onChange } = props; + const [loading, setLoading] = useState(false); + const fetch = useFetch(); + + const generateImage = useCallback( + (type: string) => async () => { + setLoading(true); + const image = await ( + await fetch('/media/generate-image-with-prompt', { + method: 'POST', + body: JSON.stringify({ + prompt: ` + +${value} + + + +${type} + + +`, + }), + }) + ).json(); + setLoading(false); + onChange(image); + }, + [value, onChange] + ); + + return ( +
+ + {value.length >= 30 && !loading && ( +
+
    + {list.map((p) => ( +
  • + {p} +
    • + ))} +
+
+ )} +
+ ); +}; diff --git a/apps/frontend/src/components/launches/providers/high.order.provider.tsx b/apps/frontend/src/components/launches/providers/high.order.provider.tsx index daa9d133..4f9d2be6 100644 --- a/apps/frontend/src/components/launches/providers/high.order.provider.tsx +++ b/apps/frontend/src/components/launches/providers/high.order.provider.tsx @@ -474,6 +474,7 @@ export const withProvider = function (
( ? undefined : typeof maximumCharacters === 'number' ? maximumCharacters - : maximumCharacters(JSON.parse(integration?.additionalSettings || '[]')) + : maximumCharacters( + JSON.parse( + integration?.additionalSettings || '[]' + ) + ) } /> ) : ( @@ -568,7 +573,11 @@ export const withProvider = function ( ? undefined : typeof maximumCharacters === 'number' ? maximumCharacters - : maximumCharacters(JSON.parse(integration?.additionalSettings || '[]')) + : maximumCharacters( + JSON.parse( + integration?.additionalSettings || '[]' + ) + ) } /> ) diff --git a/apps/frontend/src/components/media/media.component.tsx b/apps/frontend/src/components/media/media.component.tsx index b44f5a22..d715ebb1 100644 --- a/apps/frontend/src/components/media/media.component.tsx +++ b/apps/frontend/src/components/media/media.component.tsx @@ -15,6 +15,7 @@ import { LoadingComponent } from '@gitroom/frontend/components/layout/loading'; import { MultipartFileUploader } from '@gitroom/frontend/components/media/new.uploader'; import dynamic from 'next/dynamic'; import { useUser } from '@gitroom/frontend/components/layout/user.context'; +import { AiImage } from '@gitroom/frontend/components/launches/ai.image'; const Polonto = dynamic( () => import('@gitroom/frontend/components/launches/polonto') ); @@ -152,7 +153,7 @@ export const MediaBox: FC<{ )} > {!mediaList.length && ( -
+
You don{"'"}t have any assets yet.
Click the button below to upload one
@@ -212,13 +213,14 @@ export const MultiMediaComponent: FC<{ label: string; description: string; value?: Array<{ path: string; id: string }>; + text: string; name: string; error?: any; onChange: (event: { target: { name: string; value?: Array<{ id: string; path: string }> }; }) => void; }> = (props) => { - const { name, label, error, description, onChange, value } = props; + const { name, label, error, text, description, onChange, value } = props; const user = useUser(); useEffect(() => { @@ -276,48 +278,59 @@ export const MultiMediaComponent: FC<{ onClick={showModal} className="ml-[10px] rounded-[4px] mb-[10px] gap-[8px] !text-primary justify-center items-center w-[127px] flex border border-dashed border-customColor21 bg-input" > -
- - - -
-
- Insert Media +
+
+ + + +
+
+ Insert Media +
+ + {!!user?.tier?.ai && ( + + )}
{!!currentMedia && diff --git a/libraries/nestjs-libraries/src/database/prisma/media/media.service.ts b/libraries/nestjs-libraries/src/database/prisma/media/media.service.ts index ddc8574e..7ea8467f 100644 --- a/libraries/nestjs-libraries/src/database/prisma/media/media.service.ts +++ b/libraries/nestjs-libraries/src/database/prisma/media/media.service.ts @@ -12,8 +12,12 @@ export class MediaService { private _subscriptionService: SubscriptionService ){} - async generateImage(prompt: string, org: Organization) { - const image = await this._openAi.generateImage(prompt); + async generateImage(prompt: string, org: Organization, generatePromptFirst?: boolean) { + if (generatePromptFirst) { + prompt = await this._openAi.generatePromptForPicture(prompt); + console.log('Prompt:', prompt); + } + const image = await this._openAi.generateImage(prompt, !!generatePromptFirst); await this._subscriptionService.useCredit(org); return image; } diff --git a/libraries/nestjs-libraries/src/database/prisma/notifications/notification.service.ts b/libraries/nestjs-libraries/src/database/prisma/notifications/notification.service.ts index 52f9ce3c..af007660 100644 --- a/libraries/nestjs-libraries/src/database/prisma/notifications/notification.service.ts +++ b/libraries/nestjs-libraries/src/database/prisma/notifications/notification.service.ts @@ -37,8 +37,8 @@ export class NotificationService { } } - async sendEmail(to: string, subject: string, html: string) { - await this._emailService.sendEmail(to, subject, html); + async sendEmail(to: string, subject: string, html: string, replyTo?: string) { + await this._emailService.sendEmail(to, subject, html, replyTo); } hasEmailProvider() { diff --git a/libraries/nestjs-libraries/src/emails/email.interface.ts b/libraries/nestjs-libraries/src/emails/email.interface.ts index 7e6acbc0..a5ebf824 100644 --- a/libraries/nestjs-libraries/src/emails/email.interface.ts +++ b/libraries/nestjs-libraries/src/emails/email.interface.ts @@ -1,5 +1,12 @@ export interface EmailInterface { name: string; validateEnvKeys: string[]; - sendEmail(to: string, subject: string, html: string, emailFromName: string, emailFromAddress: string): Promise; -} \ No newline at end of file + sendEmail( + to: string, + subject: string, + html: string, + emailFromName: string, + emailFromAddress: string, + replyTo?: string + ): Promise; +} diff --git a/libraries/nestjs-libraries/src/emails/resend.provider.ts b/libraries/nestjs-libraries/src/emails/resend.provider.ts index 15e34cf5..b08c0eed 100644 --- a/libraries/nestjs-libraries/src/emails/resend.provider.ts +++ b/libraries/nestjs-libraries/src/emails/resend.provider.ts @@ -6,12 +6,20 @@ const resend = new Resend(process.env.RESEND_API_KEY || 're_132'); export class ResendProvider implements EmailInterface { name = 'resend'; validateEnvKeys = ['RESEND_API_KEY']; - async sendEmail(to: string, subject: string, html: string, emailFromName: string, emailFromAddress: string) { + async sendEmail( + to: string, + subject: string, + html: string, + emailFromName: string, + emailFromAddress: string, + replyTo?: string + ) { const sends = await resend.emails.send({ from: `${emailFromName} <${emailFromAddress}>`, to, subject, html, + ...(replyTo && { reply_to: replyTo }), }); return sends; diff --git a/libraries/nestjs-libraries/src/openai/openai.service.ts b/libraries/nestjs-libraries/src/openai/openai.service.ts index 077ad9c7..1f740fdc 100644 --- a/libraries/nestjs-libraries/src/openai/openai.service.ts +++ b/libraries/nestjs-libraries/src/openai/openai.service.ts @@ -1,19 +1,50 @@ import { Injectable } from '@nestjs/common'; import OpenAI from 'openai'; import { shuffle } from 'lodash'; +import { zodResponseFormat } from 'openai/helpers/zod'; +import { z } from 'zod'; const openai = new OpenAI({ apiKey: process.env.OPENAI_API_KEY || 'sk-proj-', }); +const PicturePrompt = z.object({ + prompt: z.string(), +}); + @Injectable() export class OpenaiService { - async generateImage(prompt: string) { - return (await openai.images.generate({ - prompt, - response_format: 'b64_json', - model: 'dall-e-3', - })).data[0].b64_json; + async generateImage(prompt: string, isUrl: boolean) { + const generate = ( + await openai.images.generate({ + prompt, + response_format: isUrl ? 'url' : 'b64_json', + model: 'dall-e-3', + }) + ).data[0]; + + return isUrl ? generate.url : generate.b64_json; + } + + async generatePromptForPicture(prompt: string) { + return ( + ( + await openai.beta.chat.completions.parse({ + model: 'gpt-4o-2024-08-06', + messages: [ + { + role: 'system', + content: `You are an assistant that take a description and style and generate a prompt that will be used later to generate images, make it a very long and descriptive explanation, and write a lot of things for the renderer like, if it${"'"}s realistic describe the camera`, + }, + { + role: 'user', + content: `prompt: ${prompt}`, + }, + ], + response_format: zodResponseFormat(PicturePrompt, 'picturePrompt'), + }) + ).choices[0].message.parsed?.prompt || '' + ); } async generatePosts(content: string) { diff --git a/libraries/nestjs-libraries/src/services/email.service.ts b/libraries/nestjs-libraries/src/services/email.service.ts index 903111af..135dceb7 100644 --- a/libraries/nestjs-libraries/src/services/email.service.ts +++ b/libraries/nestjs-libraries/src/services/email.service.ts @@ -32,7 +32,7 @@ export class EmailService { } } - async sendEmail(to: string, subject: string, html: string) { + async sendEmail(to: string, subject: string, html: string, replyTo?: string) { if (!process.env.EMAIL_FROM_ADDRESS || !process.env.EMAIL_FROM_NAME) { console.log( 'Email sender information not found in environment variables' @@ -96,7 +96,8 @@ export class EmailService { subject, modifiedHtml, process.env.EMAIL_FROM_NAME, - process.env.EMAIL_FROM_ADDRESS + process.env.EMAIL_FROM_ADDRESS, + replyTo ); console.log(sends); }