From a0054ac2de69de1a30e6d291c7e5f5c76f4a0091 Mon Sep 17 00:00:00 2001 From: Nevo David Date: Wed, 25 Dec 2024 17:52:10 +0700 Subject: [PATCH] feat: agents --- .../src/api/routes/posts.controller.ts | 21 +- .../src/api/routes/public.controller.ts | 17 +- apps/backend/src/app.module.ts | 2 + apps/commands/project.json | 10 - apps/commands/src/command.module.ts | 11 +- apps/commands/src/tasks/agent.run.ts | 15 + apps/frontend/src/app/global.scss | 21 + .../components/launches/add.edit.model.tsx | 9 +- .../src/components/launches/editor.tsx | 2 +- .../launches/generator/generator.tsx | 458 ++--- .../launches/launches.component.tsx | 13 +- .../src/agent/agent.categories.ts | 32 + .../src/agent/agent.graph.insert.service.ts | 138 ++ .../src/agent/agent.graph.service.ts | 419 ++++ .../src/agent/agent.module.ts | 12 + .../src/agent/agent.topics.ts | 82 + .../integrations/integration.repository.ts | 17 +- .../integrations/integration.service.ts | 16 +- .../database/prisma/posts/posts.repository.ts | 85 +- .../database/prisma/posts/posts.service.ts | 84 +- .../src/database/prisma/schema.prisma | 10 + .../src/dtos/generator/generator.dto.ts | 28 +- .../src/form/textarea.tsx | 2 +- package-lock.json | 1815 +++++++++++------ package.json | 10 +- 25 files changed, 2414 insertions(+), 915 deletions(-) create mode 100644 apps/commands/src/tasks/agent.run.ts create mode 100644 libraries/nestjs-libraries/src/agent/agent.categories.ts create mode 100644 libraries/nestjs-libraries/src/agent/agent.graph.insert.service.ts create mode 100644 libraries/nestjs-libraries/src/agent/agent.graph.service.ts create mode 100644 libraries/nestjs-libraries/src/agent/agent.module.ts create mode 100644 libraries/nestjs-libraries/src/agent/agent.topics.ts diff --git a/apps/backend/src/api/routes/posts.controller.ts b/apps/backend/src/api/routes/posts.controller.ts index 1590411a..e02f743e 100644 --- a/apps/backend/src/api/routes/posts.controller.ts +++ b/apps/backend/src/api/routes/posts.controller.ts @@ -7,6 +7,7 @@ import { Post, Put, Query, + Res, } from '@nestjs/common'; import { PostsService } from '@gitroom/nestjs-libraries/database/prisma/posts/posts.service'; import { GetOrgFromRequest } from '@gitroom/nestjs-libraries/user/org.from.request'; @@ -23,6 +24,8 @@ import { ApiTags } from '@nestjs/swagger'; import { MessagesService } from '@gitroom/nestjs-libraries/database/prisma/marketplace/messages.service'; import { GeneratorDto } from '@gitroom/nestjs-libraries/dtos/generator/generator.dto'; import { CreateGeneratedPostsDto } from '@gitroom/nestjs-libraries/dtos/generator/create.generated.posts.dto'; +import { AgentGraphService } from '@gitroom/nestjs-libraries/agent/agent.graph.service'; +import { Response } from 'express'; @ApiTags('Posts') @Controller('/posts') @@ -30,7 +33,8 @@ export class PostsController { constructor( private _postsService: PostsService, private _starsService: StarsService, - private _messagesService: MessagesService + private _messagesService: MessagesService, + private _agentGraphService: AgentGraphService ) {} @Get('/marketplace/:id?') @@ -100,11 +104,20 @@ export class PostsController { @Post('/generator') @CheckPolicies([AuthorizationActions.Create, Sections.POSTS_PER_MONTH]) - generatePosts( + async generatePosts( @GetOrgFromRequest() org: Organization, - @Body() body: GeneratorDto + @Body() body: GeneratorDto, + @Res({ passthrough: false }) res: Response ) { - return this._postsService.generatePosts(org.id, body); + res.setHeader('Content-Type', 'application/json; charset=utf-8'); + for await (const event of this._agentGraphService.start( + org.id, + body, + )) { + res.write(JSON.stringify(event) + '\n'); + } + + res.end(); } @Delete('/:group') diff --git a/apps/backend/src/api/routes/public.controller.ts b/apps/backend/src/api/routes/public.controller.ts index dc4d16fc..44ff199f 100644 --- a/apps/backend/src/api/routes/public.controller.ts +++ b/apps/backend/src/api/routes/public.controller.ts @@ -10,14 +10,29 @@ import { GetUserFromRequest } from '@gitroom/nestjs-libraries/user/user.from.req import { User } from '@prisma/client'; import { makeId } from '@gitroom/nestjs-libraries/services/make.is'; import { getCookieUrlFromDomain } from '@gitroom/helpers/subdomain/subdomain.management'; +import { AgentGraphService } from '@gitroom/nestjs-libraries/agent/agent.graph.service'; +import { AgentGraphInsertService } from '@gitroom/nestjs-libraries/agent/agent.graph.insert.service'; @ApiTags('Public') @Controller('/public') export class PublicController { constructor( private _agenciesService: AgenciesService, - private _trackService: TrackService + private _trackService: TrackService, + private _agentGraphInsertService: AgentGraphInsertService ) {} + @Post('/agent') + async createAgent(@Body() body: { text: string; apiKey: string }) { + if ( + !body.apiKey || + !process.env.AGENT_API_KEY || + body.apiKey !== process.env.AGENT_API_KEY + ) { + return; + } + return this._agentGraphInsertService.newPost(body.text); + } + @Get('/agencies-list') async getAgencyByUser() { return this._agenciesService.getAllAgencies(); diff --git a/apps/backend/src/app.module.ts b/apps/backend/src/app.module.ts index f9cc38ba..3412b76a 100644 --- a/apps/backend/src/app.module.ts +++ b/apps/backend/src/app.module.ts @@ -9,6 +9,7 @@ import { PluginModule } from '@gitroom/plugins/plugin.module'; import { PublicApiModule } from '@gitroom/backend/public-api/public.api.module'; import { ThrottlerBehindProxyGuard } from '@gitroom/nestjs-libraries/throttler/throttler.provider'; import { ThrottlerModule } from '@nestjs/throttler'; +import { AgentModule } from '@gitroom/nestjs-libraries/agent/agent.module'; @Global() @Module({ @@ -18,6 +19,7 @@ import { ThrottlerModule } from '@nestjs/throttler'; ApiModule, PluginModule, PublicApiModule, + AgentModule, ThrottlerModule.forRoot([ { ttl: 3600000, diff --git a/apps/commands/project.json b/apps/commands/project.json index bd2b0ad9..3629dc78 100644 --- a/apps/commands/project.json +++ b/apps/commands/project.json @@ -25,17 +25,7 @@ "executor": "nx:run-commands", "defaultConfiguration": "development", "options": { - "buildTarget": "commands:build", - "inspect": false, "command": "cd dist/apps/commands && node main.js" - }, - "configurations": { - "development": { - "buildTarget": "commands:build:development" - }, - "production": { - "buildTarget": "commands:build:production" - } } }, "lint": { diff --git a/apps/commands/src/command.module.ts b/apps/commands/src/command.module.ts index 9430ef56..44d9d5e5 100644 --- a/apps/commands/src/command.module.ts +++ b/apps/commands/src/command.module.ts @@ -5,11 +5,18 @@ import { DatabaseModule } from '@gitroom/nestjs-libraries/database/prisma/databa import { RefreshTokens } from './tasks/refresh.tokens'; import { BullMqModule } from '@gitroom/nestjs-libraries/bull-mq-transport-new/bull.mq.module'; import { ConfigurationTask } from './tasks/configuration'; +import { AgentRun } from './tasks/agent.run'; +import { AgentModule } from '@gitroom/nestjs-libraries/agent/agent.module'; @Module({ - imports: [ExternalCommandModule, DatabaseModule, BullMqModule], + imports: [ + ExternalCommandModule, + DatabaseModule, + BullMqModule, + AgentModule, + ], controllers: [], - providers: [CheckStars, RefreshTokens, ConfigurationTask], + providers: [CheckStars, RefreshTokens, ConfigurationTask, AgentRun], get exports() { return [...this.imports, ...this.providers]; }, diff --git a/apps/commands/src/tasks/agent.run.ts b/apps/commands/src/tasks/agent.run.ts new file mode 100644 index 00000000..ad8e67c0 --- /dev/null +++ b/apps/commands/src/tasks/agent.run.ts @@ -0,0 +1,15 @@ +import { Command } from 'nestjs-command'; +import { Injectable } from '@nestjs/common'; +import { AgentGraphService } from '@gitroom/nestjs-libraries/agent/agent.graph.service'; + +@Injectable() +export class AgentRun { + constructor(private _agentGraphService: AgentGraphService) {} + @Command({ + command: 'run:agent', + describe: 'Run the agent', + }) + async agentRun() { + console.log(await this._agentGraphService.createGraph('hello', true)); + } +} diff --git a/apps/frontend/src/app/global.scss b/apps/frontend/src/app/global.scss index 885b1e7d..f9e668fa 100644 --- a/apps/frontend/src/app/global.scss +++ b/apps/frontend/src/app/global.scss @@ -394,4 +394,25 @@ div div .set-font-family { transform: translate(-50%, -50%); white-space: nowrap; opacity: 30%; +} + +.loading-shimmer { + position: relative; + color: rgba(255, 255, 255, .5); +} + +.loading-shimmer:before { + content: attr(data-text); + position: absolute; + overflow: hidden; + max-width: 100%; + white-space: nowrap; + color: white; + animation: loading 4s linear 0s infinite; + filter: blur(0.4px); +} +@keyframes loading { + 0% { + max-width: 0; + } } \ No newline at end of file diff --git a/apps/frontend/src/components/launches/add.edit.model.tsx b/apps/frontend/src/components/launches/add.edit.model.tsx index 38c434de..79a0a01a 100644 --- a/apps/frontend/src/components/launches/add.edit.model.tsx +++ b/apps/frontend/src/components/launches/add.edit.model.tsx @@ -66,8 +66,13 @@ export const AddEditModal: FC<{ integrations: Integrations[]; reopenModal: () => void; mutate: () => void; + onlyValues?: Array<{ + content: string; + id?: string; + image?: Array<{ id: string; path: string }>; + }>; }> = (props) => { - const { date, integrations: ints, reopenModal, mutate } = props; + const { date, integrations: ints, reopenModal, mutate, onlyValues } = props; const [customer, setCustomer] = useState(''); // selected integrations to allow edit @@ -104,7 +109,7 @@ export const AddEditModal: FC<{ id?: string; image?: Array<{ id: string; path: string }>; }> - >([{ content: '' }]); + >(onlyValues ? onlyValues : [{ content: '' }]); const fetch = useFetch(); diff --git a/apps/frontend/src/components/launches/editor.tsx b/apps/frontend/src/components/launches/editor.tsx index e2906b01..20c743c2 100644 --- a/apps/frontend/src/components/launches/editor.tsx +++ b/apps/frontend/src/components/launches/editor.tsx @@ -22,7 +22,7 @@ export const Editor = forwardRef< const user = useUser(); useCopilotReadable({ description: 'Content of the post number ' + (props.order + 1), - value: props.content, + value: props.value, }); useCopilotAction({ diff --git a/apps/frontend/src/components/launches/generator/generator.tsx b/apps/frontend/src/components/launches/generator/generator.tsx index e75b32c1..00acb205 100644 --- a/apps/frontend/src/components/launches/generator/generator.tsx +++ b/apps/frontend/src/components/launches/generator/generator.tsx @@ -1,193 +1,30 @@ -import React, { - FC, - useCallback, - useMemo, - useState, -} from 'react'; +import React, { FC, useCallback, useMemo, useState } from 'react'; import { useUser } from '@gitroom/frontend/components/layout/user.context'; import { useRouter } from 'next/navigation'; import { deleteDialog } from '@gitroom/react/helpers/delete.dialog'; -import { - Step, - StepSpace, -} from '@gitroom/frontend/components/onboarding/onboarding'; import { useModals } from '@mantine/modals'; -import { Select } from '@gitroom/react/form/select'; -import { Input } from '@gitroom/react/form/input'; -import dayjs from 'dayjs'; import { FormProvider, SubmitHandler, useForm } from 'react-hook-form'; import { classValidatorResolver } from '@hookform/resolvers/class-validator'; import { GeneratorDto } from '@gitroom/nestjs-libraries/dtos/generator/generator.dto'; import { Button } from '@gitroom/react/form/button'; -import { PostSelector } from '@gitroom/frontend/components/post-url-selector/post.url.selector'; import { useFetch } from '@gitroom/helpers/utils/custom.fetch'; +import { Textarea } from '@gitroom/react/form/textarea'; +import { Checkbox } from '@gitroom/react/form/checkbox'; import clsx from 'clsx'; +import { + CalendarWeekProvider, + useCalendar, +} from '@gitroom/frontend/components/launches/calendar.context'; +import { AddEditModal } from '@gitroom/frontend/components/launches/add.edit.model'; +import dayjs from 'dayjs'; +import { Select } from '@gitroom/react/form/select'; -const ThirdStep: FC<{ week: number; year: number }> = (props) => { - const { week, year } = props; - - const gotToPosts = useCallback(() => { - window.location.href = `/launches?week=${week}&year=${year}`; - }, [week, year]); - return ( -
-
- success - Your posts have been scheduled as drafts. -
- -
-
- ); -}; - -const SecondStep: FC<{ - posts: Array>; - url: string; - postId?: string; - nextStep: (params: { week: number; year: number }) => void; -}> = (props) => { - const { posts, nextStep, url, postId } = props; - const fetch = useFetch(); - const [selected, setSelected] = useState>([]); - const [loading, setLoading] = useState(false); - - const form = useForm({ - values: { - date: dayjs().week() + '_' + dayjs().year(), - }, - }); - - const addPost = useCallback( - (index: string) => () => { - if (selected.includes(index)) { - setSelected(selected.filter((i) => i !== index)); - return; - } - setSelected([...selected, index]); - }, - [selected] - ); - - const list = useMemo(() => { - const currentDate = dayjs(); - return [...new Array(52)].map((_, i) => { - const week = currentDate.add(i, 'week'); - return { - value: week.week() + '_' + week.year(), - label: `Week #${week.week()} (${week - .startOf('isoWeek') - .format('YYYY-MM-DD')} - ${week - .endOf('isoWeek') - .format('YYYY-MM-DD')})`, - }; - }); - }, []); - - const createPosts: SubmitHandler<{ - date: any; - }> = useCallback( - async (values) => { - setLoading(true); - await fetch('/posts/generator/draft', { - method: 'POST', - body: JSON.stringify({ - posts: posts - .filter((_, index) => selected.includes(String(index))) - .map((po) => ({ list: po })), - url, - postId: postId ? `(post:${postId})` : undefined, - year: values.date.year, - week: values.date.week, - }), - }); - setLoading(false); - nextStep({ - week: values.date.week, - year: values.date.year, - }); - }, - [selected, postId, url] - ); - - return ( -
- -
- -
- Click on the posts you would like to schedule. -
- They will be saved as drafts and you can edit them later. -
-
- {posts.map((post, index) => ( -
- {post.length > 1 && ( -
- a thread -
- )} -
-
-
- {post[0].post.split('\n\n')[0]} -
-
-
-
- ))} -
-
- -
-
- -
- ); -}; - -const FirstStep: FC<{ - nextStep: ( - posts: Array>, - url: string, - postId?: string - ) => void; -}> = (props) => { - const { nextStep } = props; +const FirstStep: FC = (props) => { + const { integrations, reloadCalendarView } = useCalendar(); + const modal = useModals(); const fetch = useFetch(); const [loading, setLoading] = useState(false); + const [showStep, setShowStep] = useState(''); const resolver = useMemo(() => { return classValidatorResolver(GeneratorDto); }, []); @@ -196,52 +33,130 @@ const FirstStep: FC<{ mode: 'all', resolver, values: { - url: '', - post: undefined as undefined | string, + research: '', + isPicture: false, + format: 'one_short', + tone: 'personal', }, }); - const [url, post] = form.watch(['url', 'post']); + const [research] = form.watch(['research']); - const makeSelect = useCallback( - (post?: string) => { - form.setValue('post', post?.split?.(':')[1]?.split(')')?.[0]); + const generateStep = useCallback( + async (reader: ReadableStreamDefaultReader) => { + const decoder = new TextDecoder('utf-8'); - if (!post && !url) { - form.setError('url', { - message: 'You need to select a post or a URL', - }); - return; + let lastResponse = {} as any; + // eslint-disable-next-line no-constant-condition + while (true) { + const { done, value } = await reader.read(); + if (done) return lastResponse.data.output; + + // Convert chunked binary data to string + const chunkStr = decoder.decode(value, { stream: true }); + for (const chunk of chunkStr + .split('\n') + .filter((f) => f && f.indexOf('{') > -1)) { + try { + const data = JSON.parse(chunk); + switch (data.name) { + case 'agent': + setShowStep('Agent starting'); + break; + case 'research': + setShowStep('Researching your content...'); + break; + case 'find-category': + setShowStep('Understanding the category...'); + break; + case 'find-topic': + setShowStep('Finding the topic...'); + break; + case 'find-popular-posts': + setShowStep('Finding popular posts to match with...'); + break; + case 'generate-hook': + setShowStep('Generating hook...'); + break; + case 'generate-content': + setShowStep('Generating content...'); + break; + case 'generate-picture': + setShowStep('Generating pictures...'); + break; + case 'upload-pictures': + setShowStep('Uploading pictures...'); + break; + case 'post-time': + setShowStep('Finding time to post...'); + break; + } + lastResponse = data; + } catch (e) { + /** don't do anything **/ + } + } } - - if (post && url) { - form.setError('url', { - message: 'You can only have a URL or a post', - }); - return; - } - - form.setError('url', { - message: '', - }); }, - [post, url] + [] ); const onSubmit: SubmitHandler<{ - url: string; - post: string | undefined; - }> = useCallback(async (value) => { - setLoading(true); - const data = await ( - await fetch('/posts/generator', { + research: string; + }> = useCallback( + async (value) => { + setLoading(true); + const response = await fetch('/posts/generator', { method: 'POST', body: JSON.stringify(value), - }) - ).json(); - nextStep(data.list, value.url, value.post); - setLoading(false); - }, []); + }); + + if (!response.body) { + return; + } + + const reader = response.body.getReader(); + const load = await generateStep(reader); + + const messages = load.content.map((p: any, index: number) => { + if (index === 0) { + return { + content: load.hook + '\n' + p.content, + ...(p?.image?.path ? { image: [p.image] } : {}), + }; + } + + return { + content: p.content, + ...(p?.image?.path ? { image: [p.image] } : {}), + }; + }); + + setShowStep(''); + + modal.openModal({ + closeOnClickOutside: false, + closeOnEscape: false, + withCloseButton: false, + classNames: { + modal: 'w-[100%] max-w-[1400px] bg-transparent text-textColor', + }, + children: ( + ({ ...p }))} + mutate={reloadCalendarView} + date={dayjs.utc(load.date).local()} + reopenModal={() => ({})} + onlyValues={messages} + /> + ), + size: '80%', + }); + + setLoading(false); + }, + [integrations, reloadCalendarView] + ); return (
-
+
- -
-
-
-
- null} - onSelect={makeSelect} - date={dayjs().add(1, 'year')} - only="article" + {!showStep ? ( +
 
+ ) : ( +
+ {showStep} +
+ )} +