jeffemmett
/
postiz
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

uppy uploader

This commit is contained in:
SanadKhan 2024-09-23 12:22:37 +05:30
parent 4622c010da
commit 92cd484212
12 changed files with 218 additions and 157 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.env.example
View File

@ -2,6 +2,8 @@ DATABASE_URL=""
REDIS_URL=""
UPLOAD_DIRECTORY=""
NEXT_PUBLIC_UPLOAD_STATIC_DIRECTORY=""
NEXT_PUBLIC_STORAGE_PROVIDER="local or cloudflare //default is local"
STORAGE_PROVIDER="local or cloudflare //default is local"
STRIPE_PUBLISHABLE_KEY=""
STRIPE_SECRET_KEY=""
STRIPE_SIGNING_KEY=""

15
apps/backend/src/api/routes/media.controller.ts
View File

@ -10,10 +10,13 @@ import handleR2Upload from '@gitroom/nestjs-libraries/upload/r2.uploader';
import { FileInterceptor } from '@nestjs/platform-express';
import { CustomFileValidationPipe } from '@gitroom/nestjs-libraries/upload/custom.upload.validation';
import { SubscriptionService } from '@gitroom/nestjs-libraries/database/prisma/subscriptions/subscription.service';
import { basename } from 'path';
import { UploadFactory } from '@gitroom/nestjs-libraries/upload/upload.factory';
@ApiTags('Media')
@Controller('/media')
export class MediaController {
private storage = UploadFactory.createStorage();
constructor(
private _mediaService: MediaService,
private _subscriptionService: SubscriptionService
@ -33,6 +36,18 @@ export class MediaController {
return {output: 'data:image/png;base64,' + await this._mediaService.generateImage(prompt, org)};
}
@Post('/upload-server')
@UseInterceptors(FileInterceptor('file'))
@UsePipes(new CustomFileValidationPipe())
async uploadServer(
@GetOrgFromRequest() org: Organization,
@UploadedFile() file: Express.Multer.File
) {
const uploadedFile = await this.storage.uploadFile(file);
const filePath = uploadedFile.path.replace(process.env.UPLOAD_DIRECTORY, basename(process.env.UPLOAD_DIRECTORY));
return this._mediaService.saveFile(org.id, uploadedFile.originalname, filePath);
}
@Post('/upload-simple')
@UseInterceptors(FileInterceptor('file'))
@UsePipes(new CustomFileValidationPipe())

1
apps/frontend/src/components/media/media.component.tsx
View File

@ -186,6 +186,7 @@ export const MediaBox: FC<{
<img
className="w-full h-full object-cover"
src={mediaDirectory.set(media.path)}
alt='media'
/>
)}
</div>

71
apps/frontend/src/components/media/new.uploader.tsx
View File

@ -2,34 +2,14 @@ import React, { useCallback, useEffect, useMemo, useState } from 'react';
// @ts-ignore
import Uppy, { UploadResult } from '@uppy/core';
// @ts-ignore
import AwsS3Multipart from '@uppy/aws-s3-multipart';
// @ts-ignore
import { useFetch } from '@gitroom/helpers/utils/custom.fetch';
import sha256 from 'sha256';
import { getUppyUploadPlugin } from '@gitroom/react/helpers/uppy.upload';
import { FileInput, ProgressBar } from '@uppy/react';
// Uppy styles
import '@uppy/core/dist/style.min.css';
import '@uppy/dashboard/dist/style.min.css';
const fetchUploadApiEndpoint = async (
fetch: any,
endpoint: string,
data: any
) => {
const res = await fetch(`/media/${endpoint}`, {
method: 'POST',
body: JSON.stringify(data),
headers: {
accept: 'application/json',
'Content-Type': 'application/json',
},
});
return res.json();
};
export function MultipartFileUploader({
onUploadSuccess,
allowedFileTypes,
@ -41,7 +21,7 @@ export function MultipartFileUploader({
const [loaded, setLoaded] = useState(false);
const [reload, setReload] = useState(false);
const onUploadSuccessExtended = useCallback((result: UploadResult) => {
const onUploadSuccessExtended = useCallback((result: UploadResult<any,any>) => {
setReload(true);
onUploadSuccess(result);
}, [onUploadSuccess]);
@ -78,7 +58,9 @@ export function MultipartFileUploaderAfter({
onUploadSuccess: (result: UploadResult) => void;
allowedFileTypes: string;
}) {
const storageProvider = process.env.NEXT_PUBLIC_STORAGE_PROVIDER || "local";
const fetch = useFetch();
const uppy = useMemo(() => {
const uppy2 = new Uppy({
autoProceed: true,
@ -87,38 +69,17 @@ export function MultipartFileUploaderAfter({
allowedFileTypes: allowedFileTypes.split(','),
maxFileSize: 1000000000,
},
}).use(AwsS3Multipart, {
// @ts-ignore
createMultipartUpload: async (file) => {
const arrayBuffer = await new Response(file.data).arrayBuffer();
// @ts-ignore
const fileHash = await sha256(arrayBuffer);
const contentType = file.type;
return fetchUploadApiEndpoint(fetch, 'create-multipart-upload', {
file,
fileHash,
contentType,
});
},
// @ts-ignore
listParts: (file, props) =>
fetchUploadApiEndpoint(fetch, 'list-parts', { file, ...props }),
// @ts-ignore
signPart: (file, props) =>
fetchUploadApiEndpoint(fetch, 'sign-part', { file, ...props }),
// @ts-ignore
abortMultipartUpload: (file, props) =>
fetchUploadApiEndpoint(fetch, 'abort-multipart-upload', {
file,
...props,
}),
// @ts-ignore
completeMultipartUpload: (file, props) =>
fetchUploadApiEndpoint(fetch, 'complete-multipart-upload', {
file,
...props,
}),
});
const { plugin, options } = getUppyUploadPlugin(storageProvider, fetch)
uppy2.use(plugin, options)
// Set additional metadata when a file is added
uppy2.on('file-added', (file) => {
uppy2.setFileMeta(file.id, {
useCloudflare: storageProvider === 'cloudflare' ? 'true' : 'false', // Example of adding a custom field
// Add more fields as needed
});
});
uppy2.on('complete', (result) => {
onUploadSuccess(result);
@ -141,6 +102,7 @@ export function MultipartFileUploaderAfter({
return (
<>
{/* <Dashboard uppy={uppy} /> */}
<ProgressBar uppy={uppy} />
<FileInput
uppy={uppy}
@ -148,8 +110,9 @@ export function MultipartFileUploaderAfter({
strings: {
chooseFiles: 'Upload',
},
pluralize: (n) => n
}}
/>
/>
</>
);
}

66
libraries/nestjs-libraries/src/upload/cloudflare.storage.ts
View File

@ -1,19 +1,13 @@
import { S3Client, PutObjectCommand } from '@aws-sdk/client-s3';
import concat from 'concat-stream';
import { StorageEngine } from 'multer';
import type { Request } from 'express';
import 'multer';
import {makeId} from "@gitroom/nestjs-libraries/services/make.is";
import mime from 'mime-types';
import { IUploadProvider } from './upload.interface';
type CallbackFunction = (
error: Error | null,
info?: Partial<Express.Multer.File>
) => void;
class CloudflareStorage implements StorageEngine {
class CloudflareStorage implements IUploadProvider {
private _client: S3Client;
public constructor(
constructor(
accountID: string,
accessKey: string,
secretKey: string,
@ -31,56 +25,44 @@ class CloudflareStorage implements StorageEngine {
});
}
public _handleFile(
_req: Request,
file: Express.Multer.File,
callback: CallbackFunction
): void {
file.stream.pipe(
concat({ encoding: 'buffer' }, async (data) => {
// @ts-ignore
callback(null, await this._uploadFile(data, data.length, file.mimetype, mime.extension(file.mimetype)));
})
);
}
public _removeFile(
_req: Request,
file: Express.Multer.File,
callback: (error: Error | null) => void
): void {
void this._deleteFile(file.destination, callback);
}
private async _uploadFile(data: Buffer, size: number, mime: string, extension: string): Promise<Express.Multer.File> {
async uploadFile(file: Express.Multer.File): Promise<any> {
const id = makeId(10);
const extension = mime.extension(file.mimetype) || '';
// Create the PutObjectCommand to upload the file to Cloudflare R2
const command = new PutObjectCommand({
Bucket: this._bucketName,
ACL: 'public-read',
Key: `${id}.${extension}`,
Body: data,
Body: file.buffer,
});
await this._client.send(command);
return {
filename: `${id}.${extension}`,
mimetype: mime,
size,
buffer: data,
mimetype: file.mimetype,
size: file.size,
buffer: file.buffer,
originalname: `${id}.${extension}`,
fieldname: 'file',
path: `${this._uploadUrl}/${id}.${extension}`,
destination: `${this._uploadUrl}/${id}.${extension}`,
encoding: '7bit',
stream: data as any,
}
stream: file.buffer as any,
};
}
private async _deleteFile(
filedestination: string,
callback: CallbackFunction
) {
// Implement the removeFile method from IUploadProvider
async removeFile(filePath: string): Promise<void> {
// const fileName = filePath.split('/').pop(); // Extract the filename from the path
// const command = new DeleteObjectCommand({
// Bucket: this._bucketName,
// Key: fileName,
// });
// await this._client.send(command);
}
}

46
libraries/nestjs-libraries/src/upload/local.storage.ts Normal file
View File

@ -0,0 +1,46 @@
import { IUploadProvider } from './upload.interface';
import { mkdirSync, unlink, writeFileSync } from 'fs';
import { extname } from 'path';
export class LocalStorage implements IUploadProvider {
constructor(private uploadDirectory: string) {}
async uploadFile(file: Express.Multer.File): Promise<any> {
const now = new Date();
const year = now.getFullYear();
const month = String(now.getMonth() + 1).padStart(2, '0');
const day = String(now.getDate()).padStart(2, '0');
const dir = `${this.uploadDirectory}/${year}/${month}/${day}`;
mkdirSync(dir, { recursive: true });
const randomName = Array(32)
.fill(null)
.map(() => Math.round(Math.random() * 16).toString(16))
.join('');
const filePath = `${dir}/${randomName}${extname(file.originalname)}`;
// Logic to save the file to the filesystem goes here
writeFileSync(filePath, file.buffer)
return {
filename: `${randomName}${extname(file.originalname)}`,
path: filePath,
mimetype: file.mimetype,
originalname: file.originalname
};
}
async removeFile(filePath: string): Promise<void> {
// Logic to remove the file from the filesystem goes here
return new Promise((resolve, reject) => {
unlink(filePath, (err) => {
if (err) {
reject(err);
} else {
resolve();
}
});
});
}
}

25
libraries/nestjs-libraries/src/upload/upload.factory.ts Normal file
View File

@ -0,0 +1,25 @@
import { CloudflareStorage } from './cloudflare.storage';
import { IUploadProvider } from './upload.interface';
import { LocalStorage } from './local.storage';
export class UploadFactory {
static createStorage(): IUploadProvider {
const storageProvider = process.env.STORAGE_PROVIDER || 'local';
switch (storageProvider) {
case 'local':
return new LocalStorage(process.env.UPLOAD_DIRECTORY!);
case 'cloudflare':
return new CloudflareStorage(
process.env.CLOUDFLARE_ACCOUNT_ID!,
process.env.CLOUDFLARE_ACCESS_KEY!,
process.env.CLOUDFLARE_SECRET_ACCESS_KEY!,
process.env.CLOUDFLARE_REGION!,
process.env.CLOUDFLARE_BUCKETNAME!,
process.env.CLOUDFLARE_BUCKET_URL!
);
default:
throw new Error(`Invalid storage type ${storageProvider}`);
}
}
}

4
libraries/nestjs-libraries/src/upload/upload.interface.ts Normal file
View File

@ -0,0 +1,4 @@
export interface IUploadProvider {
uploadFile(file: Express.Multer.File): Promise<any>;
removeFile(filePath: string): Promise<void>;
}

59
libraries/nestjs-libraries/src/upload/upload.module.ts
View File

@ -1,60 +1,13 @@
import { Global, Module } from '@nestjs/common';
import { MulterModule } from '@nestjs/platform-express';
import { diskStorage } from 'multer';
import { mkdirSync } from 'fs';
import { extname } from 'path';
import CloudflareStorage from '@gitroom/nestjs-libraries/upload/cloudflare.storage';
import { UploadFactory } from './upload.factory';
import { CustomFileValidationPipe } from '@gitroom/nestjs-libraries/upload/custom.upload.validation';
const storage =
process.env.CLOUDFLARE_ACCOUNT_ID &&
process.env.CLOUDFLARE_ACCESS_KEY &&
process.env.CLOUDFLARE_SECRET_ACCESS_KEY &&
process.env.CLOUDFLARE_REGION &&
process.env.CLOUDFLARE_BUCKETNAME &&
process.env.CLOUDFLARE_BUCKET_URL
? new CloudflareStorage(
process.env.CLOUDFLARE_ACCOUNT_ID,
process.env.CLOUDFLARE_ACCESS_KEY,
process.env.CLOUDFLARE_SECRET_ACCESS_KEY,
process.env.CLOUDFLARE_REGION,
process.env.CLOUDFLARE_BUCKETNAME,
process.env.CLOUDFLARE_BUCKET_URL
)
: diskStorage({
destination: (req, file, cb) => {
const now = new Date();
const year = now.getFullYear();
const month = String(now.getMonth() + 1).padStart(2, '0'); // Month is zero-based, hence +1
const day = String(now.getDate()).padStart(2, '0');
const dir = `${process.env.UPLOAD_DIRECTORY}/${year}/${month}/${day}`;
// Create the directory if it doesn't exist
mkdirSync(dir, { recursive: true });
cb(null, dir);
},
filename: (req, file, cb) => {
// Generate a unique filename here if needed
const randomName = Array(32)
.fill(null)
.map(() => Math.round(Math.random() * 16).toString(16))
.join('');
cb(null, `${randomName}${extname(file.originalname)}`);
},
});
@Global()
@Module({
imports: [
MulterModule.register({
storage,
}),
],
providers: [CustomFileValidationPipe],
get exports() {
return [...this.imports, ...this.providers];
},
providers: [UploadFactory, CustomFileValidationPipe],
exports: [UploadFactory, CustomFileValidationPipe],
})
export class UploadModule {}

68
libraries/react-shared-libraries/src/helpers/uppy.upload.ts Normal file
View File

@ -0,0 +1,68 @@
import XHRUpload from '@uppy/xhr-upload';
import AwsS3Multipart from '@uppy/aws-s3';
import sha256 from 'sha256';
const fetchUploadApiEndpoint = async (
fetch: any,
endpoint: string,
data: any
) => {
const res = await fetch(`/media/${endpoint}`, {
method: 'POST',
body: JSON.stringify(data),
headers: {
accept: 'application/json',
'Content-Type': 'application/json',
},
});
return res.json();
};
// Define the factory to return appropriate Uppy configuration
export const getUppyUploadPlugin = (provider: string, fetch: any) => {
switch (provider) {
case 'cloudflare':
return {
plugin: AwsS3Multipart,
options: {
createMultipartUpload: async (file: any) => {
const arrayBuffer = await new Response(file.data).arrayBuffer();
const fileHash = sha256(Buffer.from(arrayBuffer));
const contentType = file.type;
return fetchUploadApiEndpoint(fetch, 'create-multipart-upload', {
file,
fileHash,
contentType,
});
},
listParts: (file: any, props: any) =>
fetchUploadApiEndpoint(fetch, 'list-parts', { file, ...props }),
signPart: (file: any, props: any) =>
fetchUploadApiEndpoint(fetch, 'sign-part', { file, ...props }),
abortMultipartUpload: (file: any, props: any) =>
fetchUploadApiEndpoint(fetch, 'abort-multipart-upload', {
file,
...props,
}),
completeMultipartUpload: (file: any, props: any) =>
fetchUploadApiEndpoint(fetch, 'complete-multipart-upload', {
file,
...props,
}),
},
};
case 'local':
return {
plugin: XHRUpload,
options: {
endpoint: `${process.env.NEXT_PUBLIC_BACKEND_URL}/media/upload-server`,
withCredentials: true,
},
};
// Add more cases for other cloud providers
default:
throw new Error(`Unsupported storage provider: ${provider}`);
}
}

2
libraries/react-shared-libraries/src/helpers/use.media.directory.ts
View File

@ -5,7 +5,7 @@ export const useMediaDirectory = () => {
if (path.indexOf('https') === 0) {
return path;
}
return `${process.env.NEXT_PUBLIC_BACKEND_URL}/${process.env.NEXT_PUBLIC_UPLOAD_STATIC_DIRECTORY}${path}`;
return `http://localhost/${path}`;
}, []);
return {

16
package.json
View File

@ -73,13 +73,15 @@
"@types/yup": "^0.32.0",
"@uidotdev/usehooks": "^2.4.1",
"@uiw/react-md-editor": "^4.0.3",
"@uppy/aws-s3-multipart": "^3.6.0",
"@uppy/core": "^3.5.0",
"@uppy/dashboard": "^3.5.2",
"@uppy/drag-drop": "^3.0.3",
"@uppy/file-input": "^3.0.3",
"@uppy/progress-bar": "^3.0.3",
"@uppy/react": "^3.1.3",
"@uppy/aws-s3": "^4.1.0",
"@uppy/core": "^4.2.0",
"@uppy/dashboard": "^4.1.0",
"@uppy/drag-drop": "^4.0.2",
"@uppy/file-input": "^4.0.1",
"@uppy/progress-bar": "^4.0.0",
"@uppy/react": "^4.0.2",
"@uppy/status-bar": "^4.0.3",
"@uppy/xhr-upload": "^4.1.0",
"array-move": "^4.0.0",
"axios": "1.6.7",
"bcrypt": "^5.1.1",