jeffemmett
/
postiz
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: cloudflare r2

This commit is contained in:
Nevo David 2024-03-14 15:24:44 +07:00
parent 27e965c628
commit 7b128465a4
16 changed files with 1806 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
apps/backend/src/api/routes/media.controller.ts
View File

@ -1,23 +1,28 @@
import {
Controller, FileTypeValidator, Get, MaxFileSizeValidator, ParseFilePipe, Post, Query, UploadedFile, UseInterceptors,
Controller,
FileTypeValidator,
Get,
MaxFileSizeValidator,
ParseFilePipe,
Post,
Query,
UploadedFile,
UseInterceptors,
} from '@nestjs/common';
import { FileInterceptor } from '@nestjs/platform-express';
import { Express } from 'express';
import {GetOrgFromRequest} from "@gitroom/nestjs-libraries/user/org.from.request";
import {Organization} from "@prisma/client";
import {MediaService} from "@gitroom/nestjs-libraries/database/prisma/media/media.service";
import {ApiTags} from "@nestjs/swagger";
import { GetOrgFromRequest } from '@gitroom/nestjs-libraries/user/org.from.request';
import { Organization } from '@prisma/client';
import { MediaService } from '@gitroom/nestjs-libraries/database/prisma/media/media.service';
import { ApiTags } from '@nestjs/swagger';
@ApiTags('Media')
@Controller('/media')
export class MediaController {
constructor(
private _mediaService: MediaService
) {
}
constructor(private _mediaService: MediaService) {}
@Post('/')
@UseInterceptors(FileInterceptor('file'))
uploadFile(
async uploadFile(
@GetOrgFromRequest() org: Organization,
@UploadedFile(
'file',
@ -36,8 +41,8 @@ export class MediaController {
@Get('/')
getMedia(
@GetOrgFromRequest() org: Organization,
@Query('page') page: number,
@GetOrgFromRequest() org: Organization,
@Query('page') page: number
) {
return this._mediaService.getMedia(org.id, page);
}

3
apps/docs/introduction.mdx
View File

@ -34,7 +34,8 @@ When deploying to websites like [Railway](https://railway.app) or [Heroku](https
**It has four main components:**
- `frontend` - NextJS control panel serving as the admin dashboard for the project.
- `backend` - NestJS backend that serves as the API for the frontend.
- `cron` - NestJS cron jobs that run every X to update the database with new trendings,
- `cron` - NestJS cron jobs that run every X to update the database with new trending, refresh tokens and more.
- `workers` - NestJS workers that run every X to process scheduled posts, sync GitHub stars and more.
In the future there will also be a bot of Slack, Intercom, Telegram, and more.

5
apps/docs/mint.json
View File

@ -37,11 +37,6 @@
"url": "https://github.com/gitoomhq/gitroom"
},
"anchors": [
{
"name": "Documentation",
"icon": "book-open-cover",
"url": "https://docs.gitroom.com"
},
{
"name": "Community",
"icon": "discord",

13
apps/docs/quickstart.mdx
View File

@ -4,14 +4,11 @@ title: 'Quickstart'
## Prerequisites
To run the project you need to have multiple things:
- Node.js
- PostgresSQL
- Discord client id and token
- Vercel Blob Store (optional)
- Vercel Project ID, Team ID and Token (if you want to have multi-tenancy)
- Stripe account (optional)
- OpenAI API key (optional)
- Algolia account (optional)
- Node.js (version 18+)
- PostgresSQL (or any other SQL database)
- Redis
- Resend account
- Social Media Client and Secret (more details later)
### NodeJS
A complete guide of how to install NodeJS can be found [here](https://nodejs.org/en/download/).

14
apps/frontend/src/components/launches/helpers/new.image.component.tsx
View File

@ -49,7 +49,7 @@ export const newImage: ICommand = {
suffix: state.command.suffix,
});
return ;
return;
}
newSelectionRange = selectWord({
@ -68,7 +68,11 @@ export const newImage: ICommand = {
selectedText: state1.selectedText,
selection: state.selection,
prefix: '![',
suffix: `](${process.env.NEXT_PUBLIC_BACKEND_URL}/${process.env.NEXT_PUBLIC_UPLOAD_STATIC_DIRECTORY}${media.path})`,
suffix: `](${
media.path.indexOf('http') === -1
? `${process.env.NEXT_PUBLIC_BACKEND_URL}/${process.env.NEXT_PUBLIC_UPLOAD_STATIC_DIRECTORY}`
: ``
}${media.path})`,
});
return;
@ -79,7 +83,11 @@ export const newImage: ICommand = {
selectedText: state1.selectedText,
selection: state.selection,
prefix: '![image',
suffix: `](${process.env.NEXT_PUBLIC_BACKEND_URL}/${process.env.NEXT_PUBLIC_UPLOAD_STATIC_DIRECTORY}${media.path})`,
suffix: `](${
media.path.indexOf('http') === -1
? `${process.env.NEXT_PUBLIC_BACKEND_URL}/${process.env.NEXT_PUBLIC_UPLOAD_STATIC_DIRECTORY}`
: ``
}${media.path})`,
});
}
});

9
libraries/helpers/src/utils/read.or.fetch.ts Normal file
View File

@ -0,0 +1,9 @@
import {readFileSync} from "fs";
export const readOrFetch = async (path: string) => {
if (path.indexOf('https') === 0) {
return (await fetch(path, {})).arrayBuffer();
}
return readFileSync(path);
};

6
libraries/nestjs-libraries/src/integrations/article/dev.to.provider.ts
View File

@ -77,7 +77,11 @@ export class DevToProvider implements ArticleProvider {
title: settings.title,
body_markdown: content,
main_image: settings?.main_image?.path
? `${process.env.NEXT_PUBLIC_BACKEND_URL}/${process.env.NEXT_PUBLIC_UPLOAD_STATIC_DIRECTORY}${settings?.main_image?.path}`
? `${
settings?.main_image?.path.indexOf('http') === -1
? `${process.env.NEXT_PUBLIC_BACKEND_URL}/${process.env.NEXT_PUBLIC_UPLOAD_STATIC_DIRECTORY}`
: ``
}${settings?.main_image?.path}`
: undefined,
tags: settings?.tags?.map((t) => t.label),
organization_id: settings.organization,

57
libraries/nestjs-libraries/src/integrations/article/hashnode.provider.ts
View File

@ -95,35 +95,42 @@ export class HashnodeProvider implements ArticleProvider {
}
async post(token: string, content: string, settings: HashnodeSettingsDto) {
const query = jsonToGraphQLQuery({
mutation: {
publishPost: {
__args: {
input: {
title: settings.title,
publicationId: settings.publication,
...(settings.canonical
? { originalArticleURL: settings.canonical }
: {}),
contentMarkdown: content,
tags: settings.tags.map((tag) => ({ id: tag.value })),
...(settings.subtitle ? { subtitle: settings.subtitle } : {}),
...(settings.main_image
? {
coverImageOptions: {
coverImageURL: `${process.env.NEXT_PUBLIC_BACKEND_URL}/${process.env.NEXT_PUBLIC_UPLOAD_STATIC_DIRECTORY}${settings?.main_image?.path}`,
},
}
: {}),
const query = jsonToGraphQLQuery(
{
mutation: {
publishPost: {
__args: {
input: {
title: settings.title,
publicationId: settings.publication,
...(settings.canonical
? { originalArticleURL: settings.canonical }
: {}),
contentMarkdown: content,
tags: settings.tags.map((tag) => ({ id: tag.value })),
...(settings.subtitle ? { subtitle: settings.subtitle } : {}),
...(settings.main_image
? {
coverImageOptions: {
coverImageURL: `${
settings?.main_image?.path?.indexOf('http') === -1
? `${process.env.NEXT_PUBLIC_BACKEND_URL}/${process.env.NEXT_PUBLIC_UPLOAD_STATIC_DIRECTORY}`
: ``
}${settings?.main_image?.path}`,
},
}
: {}),
},
},
post: {
id: true,
url: true,
},
},
post: {
id: true,
url: true,
},
},
},
}, {pretty: true});
{ pretty: true }
);
const {
data: {

4
libraries/nestjs-libraries/src/integrations/social/linkedin.provider.ts
View File

@ -5,9 +5,9 @@ import {
SocialProvider,
} from '@gitroom/nestjs-libraries/integrations/social/social.integrations.interface';
import { makeId } from '@gitroom/nestjs-libraries/services/make.is';
import { readFileSync } from 'fs';
import sharp from 'sharp';
import { lookup } from 'mime-types';
import { readOrFetch } from '@gitroom/helpers/utils/read.or.fetch';
export class LinkedinProvider implements SocialProvider {
identifier = 'linkedin';
@ -168,7 +168,7 @@ export class LinkedinProvider implements SocialProvider {
id: await this.uploadPicture(
accessToken,
id,
await sharp(readFileSync(m.path), {
await sharp(await readOrFetch(m.path), {
animated: lookup(m.path) === 'image/gif',
})
.resize({

6
libraries/nestjs-libraries/src/integrations/social/reddit.provider.ts
View File

@ -145,7 +145,11 @@ export class RedditProvider implements SocialProvider {
: {}),
...(firstPostSettings.value.type === 'media'
? {
url: `${process.env.NEXT_PUBLIC_BACKEND_URL}/${process.env.NEXT_PUBLIC_UPLOAD_STATIC_DIRECTORY}${firstPostSettings.value.media[0].path}`,
url: `${
firstPostSettings.value.media[0].path.indexOf('http') === -1
? `${process.env.NEXT_PUBLIC_BACKEND_URL}/${process.env.NEXT_PUBLIC_UPLOAD_STATIC_DIRECTORY}`
: ``
}${firstPostSettings.value.media[0].path}`,
}
: {}),
text: post.message,

4
libraries/nestjs-libraries/src/integrations/social/x.provider.ts
View File

@ -5,9 +5,9 @@ import {
PostResponse,
SocialProvider,
} from '@gitroom/nestjs-libraries/integrations/social/social.integrations.interface';
import { readFileSync } from 'fs';
import { lookup } from 'mime-types';
import sharp from 'sharp';
import { readOrFetch } from '@gitroom/helpers/utils/read.or.fetch';
export class XProvider implements SocialProvider {
identifier = 'x';
@ -112,7 +112,7 @@ export class XProvider implements SocialProvider {
p?.media?.flatMap(async (m) => {
return {
id: await client.v1.uploadMedia(
await sharp(readFileSync(m.path), {
await sharp(await readOrFetch(m.path), {
animated: lookup(m.path) === 'image/gif',
})
.resize({

143
libraries/nestjs-libraries/src/upload/cloudflare.storage.ts Normal file
View File

@ -0,0 +1,143 @@
import { S3Client, PutObjectCommand } from '@aws-sdk/client-s3';
import concat from 'concat-stream';
import { StorageEngine } from 'multer';
import type { Request } from 'express';
import {makeId} from "@gitroom/nestjs-libraries/services/make.is";
import mime from 'mime-types';
export interface CloudflareCDNUploadResponse<T = string[]> {
success: boolean;
errors: {
code: number;
message: string;
}[];
result: {
id: string;
filename: string;
metadata: {
meta: string;
};
requireSignedURLs: boolean;
variants: T;
uploaded: string;
};
}
type CallbackFunction = (
error: Error | null,
info?: Partial<Express.Multer.File>
) => void;
class CloudflareStorage implements StorageEngine {
private _client: S3Client;
public constructor(
accountID: string,
accessKey: string,
secretKey: string,
private _bucketName: string,
private _uploadUrl: string
) {
this._client = new S3Client({
endpoint: `https://${accountID}.r2.cloudflarestorage.com`,
credentials: {
accessKeyId: accessKey,
secretAccessKey: secretKey,
},
});
}
public _handleFile(
_req: Request,
file: Express.Multer.File,
callback: CallbackFunction
): void {
file.stream.pipe(
concat({ encoding: 'buffer' }, async (data) => {
// @ts-ignore
callback(null, await this._uploadFile(data, data.length, file.mimetype, mime.extension(file.mimetype)));
})
);
}
public _removeFile(
_req: Request,
file: Express.Multer.File,
callback: (error: Error | null) => void
): void {
void this._deleteFile(file.destination, callback);
}
private async _uploadFile(data: Buffer, size: number, mime: string, extension: string): Promise<Express.Multer.File> {
const id = makeId(10);
const command = new PutObjectCommand({
Bucket: this._bucketName,
ACL: 'public-read',
Key: `${id}.${extension}`,
Body: data,
});
const upload = await this._client.send(command);
return {
filename: `${id}.${extension}`,
mimetype: mime,
size,
buffer: data,
originalname: `${id}.${extension}`,
fieldname: 'file',
path: `${this._uploadUrl}/${id}.${extension}`,
destination: `${this._uploadUrl}/${id}.${extension}`,
encoding: '7bit',
stream: data as any,
}
// const request = await fetch(this.destURL, {
// method: 'POST',
// headers: {
// Authorization: `Bearer ${this.accountToken}`,
// ...body.getHeaders(),
// },
// body,
// });
//
// const response: CloudflareCDNUploadResponse = await request.json();
// if (request.ok) {
// return callback(null, {
// path: response.result.variants[0],
// filename: response.result.filename,
// destination: response.result.id,
// });
// }
//
// console.log(response);
// return callback(
// new Error(
// 'There was an error in uploading an asset to Cloudflare Images.'
// )
// );
}
private async _deleteFile(
filedestination: string,
callback: CallbackFunction
) {
// const request = await fetch(`${this.destURL}/${filedestination}`, {
// method: 'DELETE',
// headers: {
// Authorization: `Bearer ${this.accountToken}`,
// },
// });
//
// if (request.ok) return callback(null);
// return callback(
// new Error(
// 'There was an error in deleting the asset from Cloudflare Images.'
// )
// );
}
}
export { CloudflareStorage };
export default CloudflareStorage;

54
libraries/nestjs-libraries/src/upload/upload.module.ts
View File

@ -3,30 +3,44 @@ import { MulterModule } from '@nestjs/platform-express';
import { diskStorage } from 'multer';
import { mkdirSync } from 'fs';
import { extname } from 'path';
import CloudflareStorage from '@gitroom/nestjs-libraries/upload/cloudflare.storage';
const storage = diskStorage({
destination: (req, file, cb) => {
const now = new Date();
const year = now.getFullYear();
const month = String(now.getMonth() + 1).padStart(2, '0'); // Month is zero-based, hence +1
const day = String(now.getDate()).padStart(2, '0');
const storage =
process.env.CLOUDFLARE_ACCOUNT_ID &&
process.env.CLOUDFLARE_ACCESS_KEY &&
process.env.CLOUDFLARE_SECRET_ACCESS_KEY &&
process.env.CLOUDFLARE_BUCKETNAME &&
process.env.CLOUDFLARE_BUCKET_URL
? new CloudflareStorage(
process.env.CLOUDFLARE_ACCOUNT_ID,
process.env.CLOUDFLARE_ACCESS_KEY,
process.env.CLOUDFLARE_SECRET_ACCESS_KEY,
process.env.CLOUDFLARE_BUCKETNAME,
process.env.CLOUDFLARE_BUCKET_URL
)
: diskStorage({
destination: (req, file, cb) => {
const now = new Date();
const year = now.getFullYear();
const month = String(now.getMonth() + 1).padStart(2, '0'); // Month is zero-based, hence +1
const day = String(now.getDate()).padStart(2, '0');
const dir = `${process.env.UPLOAD_DIRECTORY}/${year}/${month}/${day}`;
const dir = `${process.env.UPLOAD_DIRECTORY}/${year}/${month}/${day}`;
// Create the directory if it doesn't exist
mkdirSync(dir, { recursive: true });
// Create the directory if it doesn't exist
mkdirSync(dir, { recursive: true });
cb(null, dir);
},
filename: (req, file, cb) => {
// Generate a unique filename here if needed
const randomName = Array(32)
.fill(null)
.map(() => Math.round(Math.random() * 16).toString(16))
.join('');
cb(null, `${randomName}${extname(file.originalname)}`);
},
});
cb(null, dir);
},
filename: (req, file, cb) => {
// Generate a unique filename here if needed
const randomName = Array(32)
.fill(null)
.map(() => Math.round(Math.random() * 16).toString(16))
.join('');
cb(null, `${randomName}${extname(file.originalname)}`);
},
});
@Global()
@Module({

3
libraries/react-shared-libraries/src/helpers/use.media.directory.ts
View File

@ -2,6 +2,9 @@ import {useCallback} from "react";
export const useMediaDirectory = () => {
const set = useCallback((path: string) => {
if (path.indexOf('https') === 0) {
return path;
}
return `${process.env.NEXT_PUBLIC_BACKEND_URL}/${process.env.NEXT_PUBLIC_UPLOAD_STATIC_DIRECTORY}${path}`;
}, []);

1538
package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

3
package.json
View File

@ -16,6 +16,7 @@
},
"private": true,
"dependencies": {
"@aws-sdk/client-s3": "^3.529.1",
"@casl/ability": "^6.5.0",
"@hookform/resolvers": "^3.3.4",
"@mantine/core": "^5.10.5",
@ -35,6 +36,7 @@
"@sweetalert2/theme-dark": "^5.0.16",
"@tanstack/react-virtual": "^3.1.3",
"@types/bcrypt": "^5.0.2",
"@types/concat-stream": "^2.0.3",
"@types/jsonwebtoken": "^9.0.5",
"@types/lodash": "^4.14.202",
"@types/md5": "^2.3.5",
@ -54,6 +56,7 @@
"class-transformer": "^0.5.1",
"class-validator": "^0.14.1",
"clsx": "^2.1.0",
"concat-stream": "^2.0.0",
"cookie-parser": "^1.4.6",
"copy-to-clipboard": "^3.3.3",
"dayjs": "^1.11.10",