feat: fixes for uploading

apps/backend/src/api/api.module.ts

@@ -46,18 +46,6 @@ const authenticatedController = [
 @Module({
   imports: [
     UploadModule,
-    ...(!!process.env.UPLOAD_DIRECTORY &&
-    !!process.env.NEXT_PUBLIC_UPLOAD_STATIC_DIRECTORY
-      ? [
-          ServeStaticModule.forRoot({
-            rootPath: process.env.UPLOAD_DIRECTORY,
-            serveRoot: '/' + process.env.NEXT_PUBLIC_UPLOAD_STATIC_DIRECTORY,
-            serveStaticOptions: {
-              index: false,
-            },
-          }),
-        ]
-      : []),
   ],
   controllers: [
     RootController,

apps/backend/src/api/routes/media.controller.ts

@@ -50,17 +50,12 @@ export class MediaController {
 
   @Post('/upload-simple')
   @UseInterceptors(FileInterceptor('file'))
-  @UsePipes(new CustomFileValidationPipe())
   async uploadSimple(
     @GetOrgFromRequest() org: Organization,
-    @UploadedFile('file')
-    file: Express.Multer.File
+    @UploadedFile('file') file: Express.Multer.File
   ) {
-    const filePath =
-      file.path.indexOf('http') === 0
-        ? file.path
-        : file.path.replace(process.env.UPLOAD_DIRECTORY, '');
-    return this._mediaService.saveFile(org.id, file.originalname, filePath);
+    const getFile = await this.storage.uploadFile(file);
+    return this._mediaService.saveFile(org.id, getFile.originalname, getFile.path);
   }
 
   @Post('/:endpoint')

apps/frontend/next.config.js

@@ -15,12 +15,37 @@ const nextConfig = {
   transpilePackages: ['crypto-hash'],
   images: {
     remotePatterns: [
+      {
+        protocol: 'http',
+        hostname: '**',
+      },
       {
         protocol: 'https',
         hostname: '**',
       },
     ],
-  }
+  },
+  async redirects() {
+    return [
+      {
+        source: '/api/uploads/:path*',
+        destination:
+          process.env.STORAGE_PROVIDER === 'local' ? '/uploads/:path*' : '/404',
+        permanent: true,
+      },
+    ];
+  },
+  async rewrites() {
+    return [
+      {
+        source: '/uploads/:path*',
+        destination:
+          process.env.STORAGE_PROVIDER === 'local'
+            ? '/api/uploads/:path*'
+            : '/404',
+      },
+    ];
+  },
 };
 
 const plugins = [

apps/frontend/src/app/api/uploads/[[...path]]/route.ts

@@ -0,0 +1,47 @@
+import { NextRequest, NextResponse } from 'next/server';
+import { createReadStream, statSync } from 'fs';
+// @ts-ignore
+import mime from 'mime';
+
+async function* nodeStreamToIterator(stream: any) {
+  for await (const chunk of stream) {
+    yield chunk;
+  }
+}
+
+function iteratorToStream(iterator: any) {
+  return new ReadableStream({
+    async pull(controller) {
+      const { value, done } = await iterator.next();
+
+      if (done) {
+        controller.close();
+      } else {
+        controller.enqueue(new Uint8Array(value));
+      }
+    },
+  });
+}
+
+export const GET = (
+  request: NextRequest,
+  context: { params: { path: string[] } }
+) => {
+  const filePath =
+    process.env.UPLOAD_DIRECTORY + '/' + context.params.path.join('/');
+  const response = createReadStream(filePath);
+
+  const fileStats = statSync(filePath);
+  const contentType = mime.getType(filePath) || 'application/octet-stream';
+
+  const iterator = nodeStreamToIterator(response);
+  const webStream = iteratorToStream(iterator);
+  return new Response(webStream, {
+    headers: {
+      'Content-Type': contentType, // Set the appropriate content-type header
+      'Content-Length': fileStats.size.toString(), // Set the content-length header
+      'Last-Modified': fileStats.mtime.toUTCString(), // Set the last-modified header
+      'Cache-Control': 'public, max-age=31536000, immutable', // Example cache-control header
+    },
+  });
+};

apps/frontend/src/components/media/media.component.tsx

@@ -1,6 +1,6 @@
 'use client';
 
-import { ChangeEvent, FC, useCallback, useEffect, useState } from 'react';
+import { FC, useCallback, useEffect, useState } from 'react';
 import { Button } from '@gitroom/react/form/button';
 import useSWR from 'swr';
 import { useFetch } from '@gitroom/helpers/utils/custom.fetch';
@@ -11,7 +11,6 @@ import EventEmitter from 'events';
 import { TopTitle } from '@gitroom/frontend/components/launches/helpers/top.title.component';
 import clsx from 'clsx';
 import { VideoFrame } from '@gitroom/react/helpers/video.frame';
-import { useToaster } from '@gitroom/react/toaster/toaster';
 import { LoadingComponent } from '@gitroom/frontend/components/layout/loading';
 import { MultipartFileUploader } from '@gitroom/frontend/components/media/new.uploader';
 import dynamic from 'next/dynamic';

apps/frontend/src/middleware.ts

@@ -7,7 +7,9 @@ import { getCookieUrlFromDomain } from '@gitroom/helpers/subdomain/subdomain.man
 export async function middleware(request: NextRequest) {
   const nextUrl = request.nextUrl;
   const authCookie = request.cookies.get('auth');
-
+  if (nextUrl.pathname.startsWith('/uploads/')) {
+    return NextResponse.next();
+  }
   // If the URL is logout, delete the cookie and redirect to login
   if (nextUrl.href.indexOf('/auth/logout') > -1) {
     const response = NextResponse.redirect(

libraries/nestjs-libraries/src/database/prisma/integrations/integration.repository.ts

@@ -3,12 +3,12 @@ import { Injectable } from '@nestjs/common';
 import dayjs from 'dayjs';
 import { Integration } from '@prisma/client';
 import { makeId } from '@gitroom/nestjs-libraries/services/make.is';
-import { simpleUpload } from '@gitroom/nestjs-libraries/upload/r2.uploader';
-import axios from 'axios';
 import { IntegrationTimeDto } from '@gitroom/nestjs-libraries/dtos/integrations/integration.time.dto';
+import { UploadFactory } from '@gitroom/nestjs-libraries/upload/upload.factory';
 
 @Injectable()
 export class IntegrationRepository {
+  private storage = UploadFactory.createStorage();
   constructor(
     private _integration: PrismaRepository<'integration'>,
     private _posts: PrismaRepository<'post'>
@@ -32,16 +32,10 @@ export class IntegrationRepository {
   async updateIntegration(id: string, params: Partial<Integration>) {
     if (
       params.picture &&
-      params.picture.indexOf(process.env.CLOUDFLARE_BUCKET_URL!) === -1
+      (params.picture.indexOf(process.env.CLOUDFLARE_BUCKET_URL!) === -1 ||
+        params.picture.indexOf(process.env.FRONTEND_URL!) === -1)
     ) {
-      const picture = await axios.get(params.picture, {
-        responseType: 'arraybuffer',
-      });
-      params.picture = await simpleUpload(
-        picture.data,
-        `${makeId(10)}.png`,
-        'image/png'
-      );
+      params.picture = await this.storage.uploadSimple(params.picture);
     }
 
     return this._integration.model.integration.update({

libraries/nestjs-libraries/src/database/prisma/integrations/integration.service.ts

@@ -21,9 +21,11 @@ import { timer } from '@gitroom/helpers/utils/timer';
 import { ioRedis } from '@gitroom/nestjs-libraries/redis/redis.service';
 import { RefreshToken } from '@gitroom/nestjs-libraries/integrations/social.abstract';
 import { IntegrationTimeDto } from '@gitroom/nestjs-libraries/dtos/integrations/integration.time.dto';
+import { UploadFactory } from '@gitroom/nestjs-libraries/upload/upload.factory';
 
 @Injectable()
 export class IntegrationService {
+  private storage = UploadFactory.createStorage();
   constructor(
     private _integrationRepository: IntegrationRepository,
     private _integrationManager: IntegrationManager,
@@ -50,13 +52,7 @@ export class IntegrationService {
     timezone?: number,
     customInstanceDetails?: string
   ) {
-    const loadImage = await axios.get(picture, { responseType: 'arraybuffer' });
-    const uploadedPicture = await simpleUpload(
-      loadImage.data,
-      `${makeId(10)}.png`,
-      'image/png'
-    );
-
+    const uploadedPicture = await this.storage.uploadSimple(picture);
     return this._integrationRepository.createOrUpdateIntegration(
       org,
       name,

libraries/nestjs-libraries/src/integrations/social/bluesky.provider.ts

@@ -120,6 +120,8 @@ export class BlueskyProvider extends SocialAbstract implements SocialProvider {
     for (const post of postDetails) {
       const images = await Promise.all(
         post.media?.map(async (p) => {
+          const a = await fetch(p.url);
+          console.log(p.url);
           return await agent.uploadBlob(
             new Blob([
               await sharp(await (await fetch(p.url)).arrayBuffer())

libraries/nestjs-libraries/src/integrations/social/reddit.provider.ts

@@ -143,7 +143,7 @@ export class RedditProvider extends SocialAbstract implements SocialProvider {
           ? {
               url: `${
                 firstPostSettings.value.media[0].path.indexOf('http') === -1
-                  ? `${process.env.NEXT_PUBLIC_BACKEND_URL}/${process.env.NEXT_PUBLIC_UPLOAD_STATIC_DIRECTORY}`
+                  ? `${process.env.NEXT_PUBLIC_BACKEND_URL}/uploads`
                   : ``
               }${firstPostSettings.value.media[0].path}`,
             }

libraries/nestjs-libraries/src/upload/cloudflare.storage.ts

@@ -1,8 +1,11 @@
 import { S3Client, PutObjectCommand } from '@aws-sdk/client-s3';
 import 'multer';
-import {makeId} from "@gitroom/nestjs-libraries/services/make.is";
+import { makeId } from '@gitroom/nestjs-libraries/services/make.is';
 import mime from 'mime-types';
+// @ts-ignore
+import {getExtension} from 'mime';
 import { IUploadProvider } from './upload.interface';
+import axios from 'axios';
 
 class CloudflareStorage implements IUploadProvider {
   private _client: S3Client;
@@ -13,7 +16,7 @@ class CloudflareStorage implements IUploadProvider {
     secretKey: string,
     private region: string,
     private _bucketName: string,
-    private _uploadUrl: string,
+    private _uploadUrl: string
   ) {
     this._client = new S3Client({
       endpoint: `https://${accountID}.r2.cloudflarestorage.com`,
@@ -25,10 +28,29 @@ class CloudflareStorage implements IUploadProvider {
     });
   }
 
+  async uploadSimple(path: string) {
+    const loadImage = await axios.get(path, { responseType: 'arraybuffer' });
+    const contentType = loadImage?.headers?.['content-type'] || loadImage?.headers?.['Content-Type'];
+    const extension = getExtension(contentType)!;
+    const id = makeId(10);
+
+    const params = {
+      Bucket: this._bucketName,
+      Key: `${id}.${extension}`,
+      Body: loadImage.data,
+      ContentType: contentType,
+    };
+
+    const command = new PutObjectCommand({ ...params });
+    await this._client.send(command);
+
+    return `${this._uploadUrl}/${id}.${extension}`;
+  }
+
   async uploadFile(file: Express.Multer.File): Promise<any> {
     const id = makeId(10);
     const extension = mime.extension(file.mimetype) || '';
-    
+
     // Create the PutObjectCommand to upload the file to Cloudflare R2
     const command = new PutObjectCommand({
       Bucket: this._bucketName,
@@ -56,12 +78,10 @@ class CloudflareStorage implements IUploadProvider {
   // Implement the removeFile method from IUploadProvider
   async removeFile(filePath: string): Promise<void> {
     // const fileName = filePath.split('/').pop(); // Extract the filename from the path
-
     // const command = new DeleteObjectCommand({
     //   Bucket: this._bucketName,
     //   Key: fileName,
     // });
-
     // await this._client.send(command);
   }
 }

libraries/nestjs-libraries/src/upload/local.storage.ts

@@ -1,46 +1,81 @@
 import { IUploadProvider } from './upload.interface';
 import { mkdirSync, unlink, writeFileSync } from 'fs';
+// @ts-ignore
+import mime from 'mime';
 import { extname } from 'path';
+import axios from 'axios';
 
 export class LocalStorage implements IUploadProvider {
-    constructor(private uploadDirectory: string) {}
-    
-    async uploadFile(file: Express.Multer.File): Promise<any> {
-        const now = new Date();
-        const year = now.getFullYear();
-        const month = String(now.getMonth() + 1).padStart(2, '0');
-        const day = String(now.getDate()).padStart(2, '0');
+  constructor(private uploadDirectory: string) {}
 
-        const dir = `${this.uploadDirectory}/${year}/${month}/${day}`;
-        mkdirSync(dir, { recursive: true });
+  async uploadSimple(path: string) {
+    const loadImage = await axios.get(path, { responseType: 'arraybuffer' });
+    const contentType = loadImage?.headers?.['content-type'] || loadImage?.headers?.['Content-Type'];
+    const findExtension = mime.getExtension(contentType)!;
 
-        const randomName = Array(32)
-        .fill(null)
-        .map(() => Math.round(Math.random() * 16).toString(16))
-        .join('');
+    const now = new Date();
+    const year = now.getFullYear();
+    const month = String(now.getMonth() + 1).padStart(2, '0');
+    const day = String(now.getDate()).padStart(2, '0');
 
-        const filePath = `${dir}/${randomName}${extname(file.originalname)}`;
-        // Logic to save the file to the filesystem goes here
-        writeFileSync(filePath, file.buffer)
+    const innerPath = `/${year}/${month}/${day}`;
+    const dir = `${this.uploadDirectory}${innerPath}`;
+    mkdirSync(dir, { recursive: true });
 
-        return {
-            filename: `${randomName}${extname(file.originalname)}`,
-            path: filePath,
-            mimetype: file.mimetype,
-            originalname: file.originalname
-        };
-    }
+    const randomName = Array(32)
+      .fill(null)
+      .map(() => Math.round(Math.random() * 16).toString(16))
+      .join('');
 
-    async removeFile(filePath: string): Promise<void> {
-        // Logic to remove the file from the filesystem goes here
-        return new Promise((resolve, reject) => {
-            unlink(filePath, (err) => {
-            if (err) {
-                reject(err);
-            } else {
-                resolve();
-            }
-            });
-        });
-    }
+    const filePath = `${dir}/${randomName}.${findExtension}`;
+    const publicPath = `${innerPath}/${randomName}.${findExtension}`;
+    // Logic to save the file to the filesystem goes here
+    writeFileSync(filePath, loadImage.data);
+
+    return process.env.FRONTEND_URL + '/uploads' + publicPath;
+  }
+
+  async uploadFile(file: Express.Multer.File): Promise<any> {
+    const now = new Date();
+    const year = now.getFullYear();
+    const month = String(now.getMonth() + 1).padStart(2, '0');
+    const day = String(now.getDate()).padStart(2, '0');
+
+    const innerPath = `/${year}/${month}/${day}`;
+    const dir = `${this.uploadDirectory}${innerPath}`;
+    mkdirSync(dir, { recursive: true });
+
+    const randomName = Array(32)
+      .fill(null)
+      .map(() => Math.round(Math.random() * 16).toString(16))
+      .join('');
+
+    const filePath = `${dir}/${randomName}${extname(file.originalname)}`;
+    const publicPath = `${innerPath}/${randomName}${extname(
+      file.originalname
+    )}`;
+    console.log(filePath);
+    // Logic to save the file to the filesystem goes here
+    writeFileSync(filePath, file.buffer);
+
+    return {
+      filename: `${randomName}${extname(file.originalname)}`,
+      path: process.env.FRONTEND_URL + '/uploads' + publicPath,
+      mimetype: file.mimetype,
+      originalname: file.originalname,
+    };
+  }
+
+  async removeFile(filePath: string): Promise<void> {
+    // Logic to remove the file from the filesystem goes here
+    return new Promise((resolve, reject) => {
+      unlink(filePath, (err) => {
+        if (err) {
+          reject(err);
+        } else {
+          resolve();
+        }
+      });
+    });
+  }
 }

libraries/nestjs-libraries/src/upload/upload.interface.ts

@@ -1,4 +1,5 @@
 export interface IUploadProvider {
+    uploadSimple(path: string): Promise<string>;
     uploadFile(file: Express.Multer.File): Promise<any>;
     removeFile(filePath: string): Promise<void>;
 }

libraries/react-shared-libraries/src/helpers/use.media.directory.ts

@@ -1,15 +1,8 @@
 import {useCallback} from "react";
-import { useVariables } from './variable.context';
 
 export const useMediaDirectory = () => {
-    const {backendUrl, uploadDirectory} = useVariables();
     const set = useCallback((path: string) => {
-        if (path.indexOf('https') === 0) {
-            return path;
-        }
-
-        const urlWithoutPort = process.env.NEXT_PUBLIC_BACKEND_URL!.split(':').slice(0, 2).join(':');
-        return `${urlWithoutPort}/${path}`;
+        return path;
     }, []);
 
     return {

package.json

@@ -69,6 +69,7 @@
     "@types/jsonwebtoken": "^9.0.5",
     "@types/lodash": "^4.14.202",
     "@types/md5": "^2.3.5",
+    "@types/mime": "^4.0.0",
     "@types/mime-types": "^2.1.4",
     "@types/multer": "^1.4.11",
     "@types/nodemailer": "^6.4.16",
@@ -109,6 +110,7 @@
     "jsonwebtoken": "^9.0.2",
     "lodash": "^4.17.21",
     "md5": "^2.3.0",
+    "mime": "^3.0.0",
     "mime-types": "^2.1.35",
     "multer": "^1.4.5-lts.1",
     "nestjs-command": "^3.1.4",