diff --git a/.env.example b/.env.example index d06208b6..94469552 100644 --- a/.env.example +++ b/.env.example @@ -2,6 +2,8 @@ DATABASE_URL="" REDIS_URL="" UPLOAD_DIRECTORY="" NEXT_PUBLIC_UPLOAD_STATIC_DIRECTORY="" +NEXT_PUBLIC_STORAGE_PROVIDER="local or cloudflare //default is local" +STORAGE_PROVIDER="local or cloudflare //default is local" STRIPE_PUBLISHABLE_KEY="" STRIPE_SECRET_KEY="" STRIPE_SIGNING_KEY="" diff --git a/apps/backend/src/api/routes/media.controller.ts b/apps/backend/src/api/routes/media.controller.ts index fc379d96..3307fef3 100644 --- a/apps/backend/src/api/routes/media.controller.ts +++ b/apps/backend/src/api/routes/media.controller.ts @@ -10,10 +10,13 @@ import handleR2Upload from '@gitroom/nestjs-libraries/upload/r2.uploader'; import { FileInterceptor } from '@nestjs/platform-express'; import { CustomFileValidationPipe } from '@gitroom/nestjs-libraries/upload/custom.upload.validation'; import { SubscriptionService } from '@gitroom/nestjs-libraries/database/prisma/subscriptions/subscription.service'; +import { basename } from 'path'; +import { UploadFactory } from '@gitroom/nestjs-libraries/upload/upload.factory'; @ApiTags('Media') @Controller('/media') export class MediaController { + private storage = UploadFactory.createStorage(); constructor( private _mediaService: MediaService, private _subscriptionService: SubscriptionService @@ -33,6 +36,18 @@ export class MediaController { return {output: 'data:image/png;base64,' + await this._mediaService.generateImage(prompt, org)}; } + @Post('/upload-server') + @UseInterceptors(FileInterceptor('file')) + @UsePipes(new CustomFileValidationPipe()) + async uploadServer( + @GetOrgFromRequest() org: Organization, + @UploadedFile() file: Express.Multer.File + ) { + const uploadedFile = await this.storage.uploadFile(file); + const filePath = uploadedFile.path.replace(process.env.UPLOAD_DIRECTORY, basename(process.env.UPLOAD_DIRECTORY)); + return this._mediaService.saveFile(org.id, uploadedFile.originalname, filePath); + } + @Post('/upload-simple') @UseInterceptors(FileInterceptor('file')) @UsePipes(new CustomFileValidationPipe()) diff --git a/apps/frontend/src/components/media/media.component.tsx b/apps/frontend/src/components/media/media.component.tsx index a90f438f..ea523fc8 100644 --- a/apps/frontend/src/components/media/media.component.tsx +++ b/apps/frontend/src/components/media/media.component.tsx @@ -186,6 +186,7 @@ export const MediaBox: FC<{ media )} diff --git a/apps/frontend/src/components/media/new.uploader.tsx b/apps/frontend/src/components/media/new.uploader.tsx index 3e4f5330..fc53abfa 100644 --- a/apps/frontend/src/components/media/new.uploader.tsx +++ b/apps/frontend/src/components/media/new.uploader.tsx @@ -2,34 +2,14 @@ import React, { useCallback, useEffect, useMemo, useState } from 'react'; // @ts-ignore import Uppy, { UploadResult } from '@uppy/core'; // @ts-ignore -import AwsS3Multipart from '@uppy/aws-s3-multipart'; -// @ts-ignore import { useFetch } from '@gitroom/helpers/utils/custom.fetch'; - -import sha256 from 'sha256'; +import { getUppyUploadPlugin } from '@gitroom/react/helpers/uppy.upload'; import { FileInput, ProgressBar } from '@uppy/react'; // Uppy styles import '@uppy/core/dist/style.min.css'; import '@uppy/dashboard/dist/style.min.css'; -const fetchUploadApiEndpoint = async ( - fetch: any, - endpoint: string, - data: any -) => { - const res = await fetch(`/media/${endpoint}`, { - method: 'POST', - body: JSON.stringify(data), - headers: { - accept: 'application/json', - 'Content-Type': 'application/json', - }, - }); - - return res.json(); -}; - export function MultipartFileUploader({ onUploadSuccess, allowedFileTypes, @@ -41,7 +21,7 @@ export function MultipartFileUploader({ const [loaded, setLoaded] = useState(false); const [reload, setReload] = useState(false); - const onUploadSuccessExtended = useCallback((result: UploadResult) => { + const onUploadSuccessExtended = useCallback((result: UploadResult) => { setReload(true); onUploadSuccess(result); }, [onUploadSuccess]); @@ -78,7 +58,9 @@ export function MultipartFileUploaderAfter({ onUploadSuccess: (result: UploadResult) => void; allowedFileTypes: string; }) { + const storageProvider = process.env.NEXT_PUBLIC_STORAGE_PROVIDER || "local"; const fetch = useFetch(); + const uppy = useMemo(() => { const uppy2 = new Uppy({ autoProceed: true, @@ -87,38 +69,17 @@ export function MultipartFileUploaderAfter({ allowedFileTypes: allowedFileTypes.split(','), maxFileSize: 1000000000, }, - }).use(AwsS3Multipart, { - // @ts-ignore - createMultipartUpload: async (file) => { - const arrayBuffer = await new Response(file.data).arrayBuffer(); - // @ts-ignore - const fileHash = await sha256(arrayBuffer); - const contentType = file.type; - return fetchUploadApiEndpoint(fetch, 'create-multipart-upload', { - file, - fileHash, - contentType, - }); - }, - // @ts-ignore - listParts: (file, props) => - fetchUploadApiEndpoint(fetch, 'list-parts', { file, ...props }), - // @ts-ignore - signPart: (file, props) => - fetchUploadApiEndpoint(fetch, 'sign-part', { file, ...props }), - // @ts-ignore - abortMultipartUpload: (file, props) => - fetchUploadApiEndpoint(fetch, 'abort-multipart-upload', { - file, - ...props, - }), - // @ts-ignore - completeMultipartUpload: (file, props) => - fetchUploadApiEndpoint(fetch, 'complete-multipart-upload', { - file, - ...props, - }), }); + + const { plugin, options } = getUppyUploadPlugin(storageProvider, fetch) + uppy2.use(plugin, options) + // Set additional metadata when a file is added + uppy2.on('file-added', (file) => { + uppy2.setFileMeta(file.id, { + useCloudflare: storageProvider === 'cloudflare' ? 'true' : 'false', // Example of adding a custom field + // Add more fields as needed + }); + }); uppy2.on('complete', (result) => { onUploadSuccess(result); @@ -141,6 +102,7 @@ export function MultipartFileUploaderAfter({ return ( <> + {/* */} n }} - /> + /> ); } diff --git a/libraries/nestjs-libraries/src/upload/cloudflare.storage.ts b/libraries/nestjs-libraries/src/upload/cloudflare.storage.ts index 55930943..8476abaa 100644 --- a/libraries/nestjs-libraries/src/upload/cloudflare.storage.ts +++ b/libraries/nestjs-libraries/src/upload/cloudflare.storage.ts @@ -1,19 +1,13 @@ import { S3Client, PutObjectCommand } from '@aws-sdk/client-s3'; -import concat from 'concat-stream'; -import { StorageEngine } from 'multer'; -import type { Request } from 'express'; +import 'multer'; import {makeId} from "@gitroom/nestjs-libraries/services/make.is"; import mime from 'mime-types'; +import { IUploadProvider } from './upload.interface'; -type CallbackFunction = ( - error: Error | null, - info?: Partial -) => void; - -class CloudflareStorage implements StorageEngine { +class CloudflareStorage implements IUploadProvider { private _client: S3Client; - public constructor( + constructor( accountID: string, accessKey: string, secretKey: string, @@ -31,56 +25,44 @@ class CloudflareStorage implements StorageEngine { }); } - public _handleFile( - _req: Request, - file: Express.Multer.File, - callback: CallbackFunction - ): void { - file.stream.pipe( - concat({ encoding: 'buffer' }, async (data) => { - // @ts-ignore - callback(null, await this._uploadFile(data, data.length, file.mimetype, mime.extension(file.mimetype))); - }) - ); - } - - public _removeFile( - _req: Request, - file: Express.Multer.File, - callback: (error: Error | null) => void - ): void { - void this._deleteFile(file.destination, callback); - } - - private async _uploadFile(data: Buffer, size: number, mime: string, extension: string): Promise { + async uploadFile(file: Express.Multer.File): Promise { const id = makeId(10); + const extension = mime.extension(file.mimetype) || ''; + + // Create the PutObjectCommand to upload the file to Cloudflare R2 const command = new PutObjectCommand({ Bucket: this._bucketName, ACL: 'public-read', Key: `${id}.${extension}`, - Body: data, + Body: file.buffer, }); await this._client.send(command); return { filename: `${id}.${extension}`, - mimetype: mime, - size, - buffer: data, + mimetype: file.mimetype, + size: file.size, + buffer: file.buffer, originalname: `${id}.${extension}`, fieldname: 'file', path: `${this._uploadUrl}/${id}.${extension}`, destination: `${this._uploadUrl}/${id}.${extension}`, encoding: '7bit', - stream: data as any, - } + stream: file.buffer as any, + }; } - private async _deleteFile( - filedestination: string, - callback: CallbackFunction - ) { + // Implement the removeFile method from IUploadProvider + async removeFile(filePath: string): Promise { + // const fileName = filePath.split('/').pop(); // Extract the filename from the path + + // const command = new DeleteObjectCommand({ + // Bucket: this._bucketName, + // Key: fileName, + // }); + + // await this._client.send(command); } } diff --git a/libraries/nestjs-libraries/src/upload/local.storage.ts b/libraries/nestjs-libraries/src/upload/local.storage.ts new file mode 100644 index 00000000..70e77443 --- /dev/null +++ b/libraries/nestjs-libraries/src/upload/local.storage.ts @@ -0,0 +1,46 @@ +import { IUploadProvider } from './upload.interface'; +import { mkdirSync, unlink, writeFileSync } from 'fs'; +import { extname } from 'path'; + +export class LocalStorage implements IUploadProvider { + constructor(private uploadDirectory: string) {} + + async uploadFile(file: Express.Multer.File): Promise { + const now = new Date(); + const year = now.getFullYear(); + const month = String(now.getMonth() + 1).padStart(2, '0'); + const day = String(now.getDate()).padStart(2, '0'); + + const dir = `${this.uploadDirectory}/${year}/${month}/${day}`; + mkdirSync(dir, { recursive: true }); + + const randomName = Array(32) + .fill(null) + .map(() => Math.round(Math.random() * 16).toString(16)) + .join(''); + + const filePath = `${dir}/${randomName}${extname(file.originalname)}`; + // Logic to save the file to the filesystem goes here + writeFileSync(filePath, file.buffer) + + return { + filename: `${randomName}${extname(file.originalname)}`, + path: filePath, + mimetype: file.mimetype, + originalname: file.originalname + }; + } + + async removeFile(filePath: string): Promise { + // Logic to remove the file from the filesystem goes here + return new Promise((resolve, reject) => { + unlink(filePath, (err) => { + if (err) { + reject(err); + } else { + resolve(); + } + }); + }); + } +} diff --git a/libraries/nestjs-libraries/src/upload/upload.factory.ts b/libraries/nestjs-libraries/src/upload/upload.factory.ts new file mode 100644 index 00000000..90dce30d --- /dev/null +++ b/libraries/nestjs-libraries/src/upload/upload.factory.ts @@ -0,0 +1,25 @@ +import { CloudflareStorage } from './cloudflare.storage'; +import { IUploadProvider } from './upload.interface'; +import { LocalStorage } from './local.storage'; + +export class UploadFactory { + static createStorage(): IUploadProvider { + const storageProvider = process.env.STORAGE_PROVIDER || 'local'; + + switch (storageProvider) { + case 'local': + return new LocalStorage(process.env.UPLOAD_DIRECTORY!); + case 'cloudflare': + return new CloudflareStorage( + process.env.CLOUDFLARE_ACCOUNT_ID!, + process.env.CLOUDFLARE_ACCESS_KEY!, + process.env.CLOUDFLARE_SECRET_ACCESS_KEY!, + process.env.CLOUDFLARE_REGION!, + process.env.CLOUDFLARE_BUCKETNAME!, + process.env.CLOUDFLARE_BUCKET_URL! + ); + default: + throw new Error(`Invalid storage type ${storageProvider}`); + } + } + } \ No newline at end of file diff --git a/libraries/nestjs-libraries/src/upload/upload.interface.ts b/libraries/nestjs-libraries/src/upload/upload.interface.ts new file mode 100644 index 00000000..281820d7 --- /dev/null +++ b/libraries/nestjs-libraries/src/upload/upload.interface.ts @@ -0,0 +1,4 @@ +export interface IUploadProvider { + uploadFile(file: Express.Multer.File): Promise; + removeFile(filePath: string): Promise; +} \ No newline at end of file diff --git a/libraries/nestjs-libraries/src/upload/upload.module.ts b/libraries/nestjs-libraries/src/upload/upload.module.ts index c74ec29e..513b2d28 100644 --- a/libraries/nestjs-libraries/src/upload/upload.module.ts +++ b/libraries/nestjs-libraries/src/upload/upload.module.ts @@ -1,60 +1,13 @@ import { Global, Module } from '@nestjs/common'; -import { MulterModule } from '@nestjs/platform-express'; -import { diskStorage } from 'multer'; -import { mkdirSync } from 'fs'; -import { extname } from 'path'; -import CloudflareStorage from '@gitroom/nestjs-libraries/upload/cloudflare.storage'; +import { UploadFactory } from './upload.factory'; import { CustomFileValidationPipe } from '@gitroom/nestjs-libraries/upload/custom.upload.validation'; -const storage = - process.env.CLOUDFLARE_ACCOUNT_ID && - process.env.CLOUDFLARE_ACCESS_KEY && - process.env.CLOUDFLARE_SECRET_ACCESS_KEY && - process.env.CLOUDFLARE_REGION && - process.env.CLOUDFLARE_BUCKETNAME && - process.env.CLOUDFLARE_BUCKET_URL - ? new CloudflareStorage( - process.env.CLOUDFLARE_ACCOUNT_ID, - process.env.CLOUDFLARE_ACCESS_KEY, - process.env.CLOUDFLARE_SECRET_ACCESS_KEY, - process.env.CLOUDFLARE_REGION, - process.env.CLOUDFLARE_BUCKETNAME, - process.env.CLOUDFLARE_BUCKET_URL - ) - : diskStorage({ - destination: (req, file, cb) => { - const now = new Date(); - const year = now.getFullYear(); - const month = String(now.getMonth() + 1).padStart(2, '0'); // Month is zero-based, hence +1 - const day = String(now.getDate()).padStart(2, '0'); - - const dir = `${process.env.UPLOAD_DIRECTORY}/${year}/${month}/${day}`; - - // Create the directory if it doesn't exist - mkdirSync(dir, { recursive: true }); - - cb(null, dir); - }, - filename: (req, file, cb) => { - // Generate a unique filename here if needed - const randomName = Array(32) - .fill(null) - .map(() => Math.round(Math.random() * 16).toString(16)) - .join(''); - cb(null, `${randomName}${extname(file.originalname)}`); - }, - }); - @Global() @Module({ - imports: [ - MulterModule.register({ - storage, - }), - ], - providers: [CustomFileValidationPipe], - get exports() { - return [...this.imports, ...this.providers]; - }, + providers: [UploadFactory, CustomFileValidationPipe], + exports: [UploadFactory, CustomFileValidationPipe], }) + export class UploadModule {} + + diff --git a/libraries/react-shared-libraries/src/helpers/uppy.upload.ts b/libraries/react-shared-libraries/src/helpers/uppy.upload.ts new file mode 100644 index 00000000..86d5da70 --- /dev/null +++ b/libraries/react-shared-libraries/src/helpers/uppy.upload.ts @@ -0,0 +1,68 @@ +import XHRUpload from '@uppy/xhr-upload'; +import AwsS3Multipart from '@uppy/aws-s3'; +import sha256 from 'sha256'; + +const fetchUploadApiEndpoint = async ( + fetch: any, + endpoint: string, + data: any + ) => { + const res = await fetch(`/media/${endpoint}`, { + method: 'POST', + body: JSON.stringify(data), + headers: { + accept: 'application/json', + 'Content-Type': 'application/json', + }, + }); + return res.json(); +}; + +// Define the factory to return appropriate Uppy configuration +export const getUppyUploadPlugin = (provider: string, fetch: any) => { + switch (provider) { + case 'cloudflare': + return { + plugin: AwsS3Multipart, + options: { + createMultipartUpload: async (file: any) => { + const arrayBuffer = await new Response(file.data).arrayBuffer(); + const fileHash = sha256(Buffer.from(arrayBuffer)); + const contentType = file.type; + return fetchUploadApiEndpoint(fetch, 'create-multipart-upload', { + file, + fileHash, + contentType, + }); + }, + listParts: (file: any, props: any) => + fetchUploadApiEndpoint(fetch, 'list-parts', { file, ...props }), + signPart: (file: any, props: any) => + fetchUploadApiEndpoint(fetch, 'sign-part', { file, ...props }), + abortMultipartUpload: (file: any, props: any) => + fetchUploadApiEndpoint(fetch, 'abort-multipart-upload', { + file, + ...props, + }), + completeMultipartUpload: (file: any, props: any) => + fetchUploadApiEndpoint(fetch, 'complete-multipart-upload', { + file, + ...props, + }), + }, + }; + + case 'local': + return { + plugin: XHRUpload, + options: { + endpoint: `${process.env.NEXT_PUBLIC_BACKEND_URL}/media/upload-server`, + withCredentials: true, + }, + }; + + // Add more cases for other cloud providers + default: + throw new Error(`Unsupported storage provider: ${provider}`); + } + } \ No newline at end of file diff --git a/libraries/react-shared-libraries/src/helpers/use.media.directory.ts b/libraries/react-shared-libraries/src/helpers/use.media.directory.ts index 39157d56..3484c645 100644 --- a/libraries/react-shared-libraries/src/helpers/use.media.directory.ts +++ b/libraries/react-shared-libraries/src/helpers/use.media.directory.ts @@ -5,7 +5,7 @@ export const useMediaDirectory = () => { if (path.indexOf('https') === 0) { return path; } - return `${process.env.NEXT_PUBLIC_BACKEND_URL}/${process.env.NEXT_PUBLIC_UPLOAD_STATIC_DIRECTORY}${path}`; + return `http://localhost/${path}`; }, []); return { diff --git a/package.json b/package.json index 73715165..1b60f721 100644 --- a/package.json +++ b/package.json @@ -73,13 +73,15 @@ "@types/yup": "^0.32.0", "@uidotdev/usehooks": "^2.4.1", "@uiw/react-md-editor": "^4.0.3", - "@uppy/aws-s3-multipart": "^3.6.0", - "@uppy/core": "^3.5.0", - "@uppy/dashboard": "^3.5.2", - "@uppy/drag-drop": "^3.0.3", - "@uppy/file-input": "^3.0.3", - "@uppy/progress-bar": "^3.0.3", - "@uppy/react": "^3.1.3", + "@uppy/aws-s3": "^4.1.0", + "@uppy/core": "^4.2.0", + "@uppy/dashboard": "^4.1.0", + "@uppy/drag-drop": "^4.0.2", + "@uppy/file-input": "^4.0.1", + "@uppy/progress-bar": "^4.0.0", + "@uppy/react": "^4.0.2", + "@uppy/status-bar": "^4.0.3", + "@uppy/xhr-upload": "^4.1.0", "array-move": "^4.0.0", "axios": "1.6.7", "bcrypt": "^5.1.1",