feat: stream files

This commit is contained in:
Nevo David 2024-07-13 23:13:18 +07:00
parent cf37c9d9af
commit a6d844a56b
8 changed files with 1169 additions and 966 deletions

View File

@ -1,40 +1,45 @@
import {
Controller,
FileTypeValidator,
Get,
MaxFileSizeValidator,
ParseFilePipe,
Post,
Query,
UploadedFile,
UseInterceptors,
UsePipes,
Controller, Get, Param, Post, Query, Req, Res
} from '@nestjs/common';
import { FileInterceptor } from '@nestjs/platform-express';
import { Express } from 'express';
import { Request, Response } from 'express';
import { GetOrgFromRequest } from '@gitroom/nestjs-libraries/user/org.from.request';
import { Organization } from '@prisma/client';
import { MediaService } from '@gitroom/nestjs-libraries/database/prisma/media/media.service';
import { ApiTags } from '@nestjs/swagger';
import { CustomFileValidationPipe } from '@gitroom/nestjs-libraries/upload/custom.upload.validation';
import handleR2Upload from '@gitroom/nestjs-libraries/upload/r2.uploader';
@ApiTags('Media')
@Controller('/media')
export class MediaController {
constructor(private _mediaService: MediaService) {}
@Post('/')
@UseInterceptors(FileInterceptor('file'))
@UsePipes(new CustomFileValidationPipe())
@Post('/:endpoint')
// @UseInterceptors(FileInterceptor('file'))
// @UsePipes(new CustomFileValidationPipe())
async uploadFile(
@GetOrgFromRequest() org: Organization,
@UploadedFile('file')
file: Express.Multer.File
@Req() req: Request,
@Res() res: Response,
@Param('endpoint') endpoint: string
// @UploadedFile('file')
// file: Express.Multer.File
) {
const filePath =
file.path.indexOf('http') === 0
? file.path
: file.path.replace(process.env.UPLOAD_DIRECTORY, '');
return this._mediaService.saveFile(org.id, file.originalname, filePath);
const upload = await handleR2Upload(endpoint, req, res);
if (endpoint !== 'complete-multipart-upload') {
return upload;
}
// @ts-ignore
const name = upload.Location.split('/').pop();
// @ts-ignore
await this._mediaService.saveFile(org.id, name, upload.Location);
res.status(200).json(upload);
// const filePath =
// file.path.indexOf('http') === 0
// ? file.path
// : file.path.replace(process.env.UPLOAD_DIRECTORY, '');
// return this._mediaService.saveFile(org.id, file.originalname, filePath);
}
@Get('/')

View File

@ -256,7 +256,7 @@ html {
/* display: none;*/
/*}*/
.w-md-editor {
background-color: #131B2C !important;
background-color: #131b2c !important;
border: 0 !important;
box-shadow: none !important;
border-radius: 8px !important;
@ -264,12 +264,12 @@ html {
border-bottom-right-radius: 0 !important;
}
.w-md-editor-toolbar {
.w-md-editor-toolbar {
height: 40px !important;
min-height: 40px !important;
background-color: #131B2C !important;
background-color: #131b2c !important;
padding: 0 8px !important;
border-color: #28344F !important;
border-color: #28344f !important;
}
.wmde-markdown {
@ -290,18 +290,18 @@ html {
}
:root {
--copilot-kit-primary-color: #612AD5 !important;
--copilot-kit-background-color: #0B0F1C !important;
--copilot-kit-separator-color: #1F2941 !important;
--copilot-kit-contrast-color: #FFFFFF !important;
--copilot-kit-secondary-contrast-color: #FFFFFF !important;
--copilot-kit-primary-color: #612ad5 !important;
--copilot-kit-background-color: #0b0f1c !important;
--copilot-kit-separator-color: #1f2941 !important;
--copilot-kit-contrast-color: #ffffff !important;
--copilot-kit-secondary-contrast-color: #ffffff !important;
--copilot-kit-secondary-color: #000 !important;
--copilot-kit-response-button-background-color: #000 !important;;
--copilot-kit-response-button-color: #fff !important;;
--copilot-kit-response-button-background-color: #000 !important;
--copilot-kit-response-button-color: #fff !important;
}
.copilotKitWindow {
background-color: #0B0F1C !important;
background-color: #0b0f1c !important;
}
.copilotKitButtonIconOpen svg {
@ -309,13 +309,13 @@ html {
}
.copilotKitButtonIconOpen:after {
content: "";
content: '';
display: block;
position: relative;
z-index: 1;
object-fit: contain;
color: white;
background: url("/magic.svg") no-repeat center center / contain;
background: url('/magic.svg') no-repeat center center / contain;
width: 30px;
height: 30px;
}
@ -327,4 +327,33 @@ html {
.copilotKitWindow {
/*right: -5rem !important;*/
}
}
.uppy-FileInput-container {
@apply cursor-pointer font-[500] flex justify-center items-center gap-[4px] text-[12px] rounded-[4px] w-[107px] h-[25px] bg-[#0b0f1c] text-white border-[2px] border-[#506490];
}
.uppy-ProgressBar {
width: 150px;
position: relative;
}
.uppy-ProgressBar-inner {
background-color: #4f46e5;
height: 25px;
border-radius: 4px;
}
.uppy-ProgressBar-percentage {
position: absolute;
top: 50%;
left: 50%;
transform: translate(-50%, -50%);
}
.uppy-ProgressBar-inner[style='width: 0%;'],
.uppy-ProgressBar-inner[style='width: 0%;'] + div,
.uppy-ProgressBar-inner[style='width: 100%;'],
.uppy-ProgressBar-inner[style='width: 100%;'] + div {
visibility: hidden;
}

View File

@ -14,6 +14,7 @@ import interClass from '@gitroom/react/helpers/inter.font';
import { VideoFrame } from '@gitroom/react/helpers/video.frame';
import { useToaster } from '@gitroom/react/toaster/toaster';
import { LoadingComponent } from '@gitroom/frontend/components/layout/loading';
import { MultipartFileUploader } from '@gitroom/frontend/components/media/new.uploader';
const showModalEmitter = new EventEmitter();
export const ShowMediaBoxModal: FC = () => {
@ -50,6 +51,8 @@ export const showMediaBox = (
showModalEmitter.emit('show-modal', callback);
};
const CHUNK_SIZE = 1024 * 1024;
export const MediaBox: FC<{
setMedia: (params: { id: string; path: string }) => void;
type?: 'image' | 'video';
@ -61,6 +64,8 @@ export const MediaBox: FC<{
const fetch = useFetch();
const mediaDirectory = useMediaDirectory();
const toaster = useToaster();
const [uploadProgress, setUploadProgress] = useState(0);
const [loading, setLoading] = useState(false);
const loadMedia = useCallback(async () => {
@ -68,32 +73,53 @@ export const MediaBox: FC<{
}, []);
const uploadMedia = useCallback(
async (file: ChangeEvent<HTMLInputElement>) => {
const maxFileSize =
(file?.target?.files?.[0].name.indexOf('mp4') || -1) > -1
? 100 * 1024 * 1024
: 10 * 1024 * 1024;
async (event: ChangeEvent<HTMLInputElement>) => {
const file = event.target.files?.[0]!;
const totalChunks = Math.ceil(file.size / CHUNK_SIZE);
for (let i = 0; i < totalChunks; i++) {
const chunk = file.slice(i, i + CHUNK_SIZE);
const formData = new FormData();
formData.append('chunk', chunk);
formData.append('chunkNumber', String(i));
formData.append('totalChunks', String(totalChunks));
formData.append('fileName', file.name);
if (
!file?.target?.files?.length ||
file?.target?.files?.[0]?.size > maxFileSize
) {
toaster.show(
`Maximum file size ${maxFileSize / 1024 / 1024}mb`,
'warning'
);
return;
const percent = ((i + 1) / totalChunks) * 100;
setUploadProgress(percent);
const data = await (
await fetch('/media', {
method: 'POST',
body: formData,
})
).json();
}
const formData = new FormData();
formData.append('file', file?.target?.files?.[0]);
setLoading(true);
const data = await (
await fetch('/media', {
method: 'POST',
body: formData,
})
).json();
setLoading(false);
// const maxFileSize =
// (file?.target?.files?.[0].name.indexOf('mp4') || -1) > -1
// ? 100 * 1024 * 1024
// : 10 * 1024 * 1024;
//
// if (
// !file?.target?.files?.length ||
// file?.target?.files?.[0]?.size > maxFileSize
// ) {
// toaster.show(
// `Maximum file size ${maxFileSize / 1024 / 1024}mb`,
// 'warning'
// );
// return;
// }
// const formData = new FormData();
// formData.append('file', file?.target?.files?.[0]);
// setLoading(true);
// const data = await (
// await fetch('/media', {
// method: 'POST',
// body: formData,
// })
// ).json();
// setLoading(false);
setListMedia([...mediaList, data]);
},
[mediaList]
@ -107,7 +133,7 @@ export const MediaBox: FC<{
[]
);
const { data } = useSWR('get-media', loadMedia);
const { data, mutate } = useSWR('get-media', loadMedia);
useEffect(() => {
if (data?.pages) {
@ -119,7 +145,7 @@ export const MediaBox: FC<{
}, [data]);
return (
<div className="fixed left-0 top-0 bg-black/80 z-[300] w-full h-full p-[60px] animate-fade">
<div className="fixed left-0 top-0 bg-black/80 z-[300] w-full min-h-full p-[60px] animate-fade">
<div className="w-full h-full bg-[#0B101B] border-tableBorder border-2 rounded-xl pb-[20px] px-[20px] relative">
<div className="flex">
<div className="flex-1">
@ -164,25 +190,16 @@ export const MediaBox: FC<{
}
onChange={uploadMedia}
/>
<button
className={`cursor-pointer font-[500] flex justify-center items-center gap-[4px] text-[12px] rounded-[4px] w-[107px] h-[25px] bg-[#0b0f1c] text-white ${interClass} border-[2px] border-[#506490]`}
>
<div>
<svg
xmlns="http://www.w3.org/2000/svg"
width="14"
height="15"
viewBox="0 0 14 15"
fill="none"
>
<path
d="M7 1.8125C5.87512 1.8125 4.7755 2.14607 3.8402 2.77102C2.90489 3.39597 2.17591 4.28423 1.74544 5.32349C1.31496 6.36274 1.20233 7.50631 1.42179 8.60958C1.64124 9.71284 2.18292 10.7263 2.97833 11.5217C3.77374 12.3171 4.78716 12.8588 5.89043 13.0782C6.99369 13.2977 8.13726 13.185 9.17651 12.7546C10.2158 12.3241 11.104 11.5951 11.729 10.6598C12.3539 9.7245 12.6875 8.62488 12.6875 7.5C12.6859 5.99207 12.0862 4.54636 11.0199 3.48009C9.95365 2.41382 8.50793 1.81409 7 1.8125ZM7 12.3125C6.04818 12.3125 5.11773 12.0303 4.32632 11.5014C3.53491 10.9726 2.91808 10.221 2.55383 9.34166C2.18959 8.46229 2.09428 7.49466 2.27997 6.56113C2.46566 5.62759 2.92401 4.77009 3.59705 4.09705C4.27009 3.42401 5.1276 2.96566 6.06113 2.77997C6.99466 2.59428 7.9623 2.68958 8.84167 3.05383C9.72104 3.41808 10.4726 4.03491 11.0015 4.82632C11.5303 5.61773 11.8125 6.54818 11.8125 7.5C11.8111 8.77591 11.3036 9.99915 10.4014 10.9014C9.49915 11.8036 8.27591 12.3111 7 12.3125ZM9.625 7.5C9.625 7.61603 9.57891 7.72731 9.49686 7.80936C9.41481 7.89141 9.30353 7.9375 9.1875 7.9375H7.4375V9.6875C7.4375 9.80353 7.39141 9.91481 7.30936 9.99686C7.22731 10.0789 7.11603 10.125 7 10.125C6.88397 10.125 6.77269 10.0789 6.69064 9.99686C6.6086 9.91481 6.5625 9.80353 6.5625 9.6875V7.9375H4.8125C4.69647 7.9375 4.58519 7.89141 4.50314 7.80936C4.4211 7.72731 4.375 7.61603 4.375 7.5C4.375 7.38397 4.4211 7.27269 4.50314 7.19064C4.58519 7.10859 4.69647 7.0625 4.8125 7.0625H6.5625V5.3125C6.5625 5.19647 6.6086 5.08519 6.69064 5.00314C6.77269 4.92109 6.88397 4.875 7 4.875C7.11603 4.875 7.22731 4.92109 7.30936 5.00314C7.39141 5.08519 7.4375 5.19647 7.4375 5.3125V7.0625H9.1875C9.30353 7.0625 9.41481 7.10859 9.49686 7.19064C9.57891 7.27269 9.625 7.38397 9.625 7.5Z"
fill="white"
/>
</svg>
</div>
<div>Upload</div>
</button>
<MultipartFileUploader
onUploadSuccess={mutate}
allowedFileTypes={
type === 'video'
? 'video/mp4'
: type === 'image'
? 'image/*'
: 'image/*,video/mp4'
}
/>
</div>
</button>
)}
@ -240,7 +257,7 @@ export const MediaBox: FC<{
.map((media) => (
<div
key={media.id}
className="w-[200px] h-[200px] flex border-tableBorder border-2 cursor-pointer"
className="w-[120px] h-[120px] flex border-tableBorder border-2 cursor-pointer"
onClick={setNewMedia(media)}
>
{media.path.indexOf('mp4') > -1 ? (

View File

@ -0,0 +1,109 @@
import React from 'react';
import Uppy, { type UploadResult } from '@uppy/core';
import { ProgressBar, FileInput, StatusBar } from '@uppy/react';
import { sha256 } from 'crypto-hash';
// @ts-ignore
import AwsS3Multipart from '@uppy/aws-s3-multipart';
// Uppy styles
import '@uppy/core/dist/style.min.css';
import '@uppy/dashboard/dist/style.min.css';
import { useFetch } from '@gitroom/helpers/utils/custom.fetch';
const fetchUploadApiEndpoint = async (
fetch: any,
endpoint: string,
data: any
) => {
const res = await fetch(`/media/${endpoint}`, {
method: 'POST',
body: JSON.stringify(data),
headers: {
accept: 'application/json',
'Content-Type': 'application/json',
},
});
return res.json();
};
export function MultipartFileUploader({
onUploadSuccess,
allowedFileTypes,
}: {
// @ts-ignore
onUploadSuccess: (result: UploadResult) => void;
allowedFileTypes: string;
}) {
const fetch = useFetch();
const uppy = React.useMemo(() => {
const uppy = new Uppy({
autoProceed: true,
restrictions: {
maxNumberOfFiles: 1,
allowedFileTypes: allowedFileTypes.split(','),
maxFileSize: 1000000,
},
}).use(AwsS3Multipart, {
// @ts-ignore
createMultipartUpload: async (file) => {
const arrayBuffer = await new Response(file.data).arrayBuffer();
const fileHash = await sha256(arrayBuffer);
const contentType = file.type;
return fetchUploadApiEndpoint(fetch, 'create-multipart-upload', {
file,
fileHash,
contentType,
});
},
// @ts-ignore
listParts: (file, props) =>
fetchUploadApiEndpoint(fetch, 'list-parts', { file, ...props }),
// @ts-ignore
signPart: (file, props) =>
fetchUploadApiEndpoint(fetch, 'sign-part', { file, ...props }),
// @ts-ignore
abortMultipartUpload: (file, props) =>
fetchUploadApiEndpoint(fetch, 'abort-multipart-upload', {
file,
...props,
}),
// @ts-ignore
completeMultipartUpload: (file, props) =>
fetchUploadApiEndpoint(fetch, 'complete-multipart-upload', {
file,
...props,
}),
});
return uppy;
}, []);
uppy.on('complete', (result) => {
onUploadSuccess(result);
});
uppy.on('upload-success', (file, response) => {
// @ts-ignore
uppy.setFileState(file.id, {
// @ts-ignore
progress: uppy.getState().files[file.id].progress,
// @ts-ignore
uploadURL: response.body.Location,
response: response,
isPaused: false,
});
});
return (
<>
<ProgressBar uppy={uppy} />
<FileInput
uppy={uppy}
pretty={true}
locale={{
strings: {
chooseFiles: 'Upload',
},
}}
/>
</>
);
}

View File

@ -38,8 +38,11 @@ export class MediaRepository {
id: org,
},
},
skip: pageNum * 10,
take: 10,
orderBy: {
createdAt: 'desc',
}
// skip: pageNum * 10,
// take: 10,
});
return {

View File

@ -0,0 +1,197 @@
import {
UploadPartCommand,
S3Client,
ListPartsCommand,
CreateMultipartUploadCommand,
CompleteMultipartUploadCommand,
AbortMultipartUploadCommand,
} from '@aws-sdk/client-s3';
import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
import { Request, Response } from 'express';
const { CLOUDFLARE_ACCOUNT_ID, CLOUDFLARE_ACCESS_KEY, CLOUDFLARE_SECRET_ACCESS_KEY, CLOUDFLARE_BUCKETNAME, CLOUDFLARE_BUCKET_URL } =
process.env;
const R2 = new S3Client({
region: 'auto',
endpoint: `https://${CLOUDFLARE_ACCOUNT_ID}.r2.cloudflarestorage.com`,
credentials: {
accessKeyId: CLOUDFLARE_ACCESS_KEY!,
secretAccessKey: CLOUDFLARE_SECRET_ACCESS_KEY!,
},
});
export default async function handleR2Upload(
endpoint: string,
req: Request,
res: Response
) {
switch (endpoint) {
case 'create-multipart-upload':
return createMultipartUpload(req, res);
case 'prepare-upload-parts':
return prepareUploadParts(req, res);
case 'complete-multipart-upload':
return completeMultipartUpload(req, res);
case 'list-parts':
return listParts(req, res);
case 'abort-multipart-upload':
return abortMultipartUpload(req, res);
case 'sign-part':
return signPart(req, res);
}
return res.status(404).end();
}
export async function createMultipartUpload(
req: Request,
res: Response
) {
const { file, fileHash, contentType } = req.body;
const filename = file.name;
try {
const params = {
Bucket: CLOUDFLARE_BUCKETNAME,
Key: `resources/${fileHash}/${filename}`,
ContentType: contentType,
Metadata: {
'x-amz-meta-file-hash': fileHash,
},
};
const command = new CreateMultipartUploadCommand({ ...params });
const response = await R2.send(command);
return res.status(200).json({
uploadId: response.UploadId,
key: response.Key,
});
} catch (err) {
console.log('Error', err);
return res.status(500).json({ source: { status: 500 } });
}
}
export async function prepareUploadParts(
req: Request,
res: Response
) {
const { partData } = req.body;
const parts = partData.parts;
const response = {
presignedUrls: {},
};
for (const part of parts) {
try {
const params = {
Bucket: CLOUDFLARE_BUCKETNAME,
Key: partData.key,
PartNumber: part.number,
UploadId: partData.uploadId,
};
const command = new UploadPartCommand({ ...params });
const url = await getSignedUrl(R2, command, { expiresIn: 3600 });
// @ts-ignore
response.presignedUrls[part.number] = url;
} catch (err) {
console.log('Error', err);
return res.status(500).json(err);
}
}
return res.status(200).json(response);
}
export async function listParts(req: Request, res: Response) {
const { key, uploadId } = req.body;
try {
const params = {
Bucket: CLOUDFLARE_BUCKETNAME,
Key: key,
UploadId: uploadId,
};
const command = new ListPartsCommand({ ...params });
const response = await R2.send(command);
return res.status(200).json(response['Parts']);
} catch (err) {
console.log('Error', err);
return res.status(500).json(err);
}
}
export async function completeMultipartUpload(
req: Request,
res: Response
) {
const { key, uploadId, parts } = req.body;
try {
const params = {
Bucket: CLOUDFLARE_BUCKETNAME,
Key: key,
UploadId: uploadId,
MultipartUpload: { Parts: parts },
};
const command = new CompleteMultipartUploadCommand({
Bucket: CLOUDFLARE_BUCKETNAME,
Key: key,
UploadId: uploadId,
MultipartUpload: { Parts: parts },
});
const response = await R2.send(command);
response.Location = process.env.CLOUDFLARE_BUCKET_URL + '/' + response?.Location?.split('/').at(-1);
return response;
} catch (err) {
console.log('Error', err);
return res.status(500).json(err);
}
}
export async function abortMultipartUpload(
req: Request,
res: Response
) {
const { key, uploadId } = req.body;
try {
const params = {
Bucket: CLOUDFLARE_BUCKETNAME,
Key: key,
UploadId: uploadId,
};
const command = new AbortMultipartUploadCommand({ ...params });
const response = await R2.send(command);
return res.status(200).json(response);
} catch (err) {
console.log('Error', err);
return res.status(500).json(err);
}
}
export async function signPart(req: Request, res: Response) {
const { key, uploadId } = req.body;
const partNumber = parseInt(req.body.partNumber);
const params = {
Bucket: CLOUDFLARE_BUCKETNAME,
Key: key,
PartNumber: partNumber,
UploadId: uploadId,
Expires: 3600
};
const command = new UploadPartCommand({ ...params });
const url = await getSignedUrl(R2, command, { expiresIn: 3600 });
return res.status(200).json({
url: url,
});
}

1591
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@ -16,7 +16,8 @@
},
"private": true,
"dependencies": {
"@aws-sdk/client-s3": "^3.529.1",
"@aws-sdk/client-s3": "^3.410.0",
"@aws-sdk/s3-request-presigner": "^3.410.0",
"@casl/ability": "^6.5.0",
"@copilotkit/backend": "0.37.0",
"@copilotkit/react-core": "0.37.0",
@ -51,6 +52,14 @@
"@types/yup": "^0.32.0",
"@uidotdev/usehooks": "^2.4.1",
"@uiw/react-md-editor": "^4.0.3",
"@uppy/aws-s3": "^3.3.0",
"@uppy/aws-s3-multipart": "^3.6.0",
"@uppy/core": "^3.5.0",
"@uppy/dashboard": "^3.5.2",
"@uppy/drag-drop": "^3.0.3",
"@uppy/file-input": "^3.0.3",
"@uppy/progress-bar": "^3.0.3",
"@uppy/react": "^3.1.3",
"@virtual-grid/react": "^2.0.2",
"array-move": "^4.0.0",
"axios": "1.6.7",
@ -65,6 +74,7 @@
"concat-stream": "^2.0.0",
"cookie-parser": "^1.4.6",
"copy-to-clipboard": "^3.3.3",
"crypto-hash": "^2.0.1",
"dayjs": "^1.11.10",
"google-auth-library": "^9.11.0",
"googleapis": "^137.1.0",