From 6785a9ec6dd29a88f84e5b5f1868f3573d3baeee Mon Sep 17 00:00:00 2001 From: Nevo David Date: Fri, 11 Jul 2025 17:53:28 +0700 Subject: [PATCH] feat: move throttle to reddit, so it can be used in multiple servers --- apps/workers/src/main.ts | 2 ++ .../helpers/src/utils/concurrency.service.ts | 36 +++++++++++++++++++ .../src/integrations/integration.manager.ts | 6 +++- .../src/integrations/social.abstract.ts | 13 ++----- package.json | 1 - pnpm-lock.yaml | 27 ++------------ 6 files changed, 48 insertions(+), 37 deletions(-) create mode 100644 libraries/helpers/src/utils/concurrency.service.ts diff --git a/apps/workers/src/main.ts b/apps/workers/src/main.ts index 42289c15..f93b7e4e 100644 --- a/apps/workers/src/main.ts +++ b/apps/workers/src/main.ts @@ -5,6 +5,8 @@ import { MicroserviceOptions } from '@nestjs/microservices'; import { BullMqServer } from '@gitroom/nestjs-libraries/bull-mq-transport-new/strategy'; async function bootstrap() { + process.env.IS_WORKER = 'true'; + // some comment again const app = await NestFactory.createMicroservice( AppModule, diff --git a/libraries/helpers/src/utils/concurrency.service.ts b/libraries/helpers/src/utils/concurrency.service.ts new file mode 100644 index 00000000..1e57ea57 --- /dev/null +++ b/libraries/helpers/src/utils/concurrency.service.ts @@ -0,0 +1,36 @@ +import { ioRedis } from '@gitroom/nestjs-libraries/redis/redis.service'; +import { timer } from '@gitroom/helpers/utils/timer'; + +export async function concurrencyService( + identifier: string, + func: (...args: any[]) => Promise +): Promise { + const key = `throttle:${identifier.split('-')[0]}`; + const expirationSeconds = 180; + + while (true) { + const setLock = await ioRedis.set( + key, + 'locked', + 'EX', + expirationSeconds, + 'NX' + ); + + if (setLock) { + break; + } + + // Wait before trying again + await timer(1000); + } + + let load: T; + try { + load = await func(); + } catch (err) {} + await timer(2000); + await ioRedis.del(key); + + return load; +} diff --git a/libraries/nestjs-libraries/src/integrations/integration.manager.ts b/libraries/nestjs-libraries/src/integrations/integration.manager.ts index 95f257af..29d18f0c 100644 --- a/libraries/nestjs-libraries/src/integrations/integration.manager.ts +++ b/libraries/nestjs-libraries/src/integrations/integration.manager.ts @@ -1,6 +1,10 @@ import 'reflect-metadata'; -import { Injectable } from '@nestjs/common'; +import { + Injectable, + OnModuleInit, + OnModuleDestroy, +} from '@nestjs/common'; import { XProvider } from '@gitroom/nestjs-libraries/integrations/social/x.provider'; import { SocialProvider } from '@gitroom/nestjs-libraries/integrations/social/social.integrations.interface'; import { LinkedinProvider } from '@gitroom/nestjs-libraries/integrations/social/linkedin.provider'; diff --git a/libraries/nestjs-libraries/src/integrations/social.abstract.ts b/libraries/nestjs-libraries/src/integrations/social.abstract.ts index 250197d6..3d4e9802 100644 --- a/libraries/nestjs-libraries/src/integrations/social.abstract.ts +++ b/libraries/nestjs-libraries/src/integrations/social.abstract.ts @@ -1,5 +1,5 @@ import { timer } from '@gitroom/helpers/utils/timer'; -import pThrottle from 'p-throttle'; +import { concurrencyService } from '@gitroom/helpers/utils/concurrency.service'; export class RefreshToken { constructor( @@ -22,15 +22,8 @@ export class NotEnoughScopes { constructor(public message = 'Not enough scopes') {} } -const pThrottleInstance = pThrottle({ - limit: 1, - interval: 5000, -}); - export abstract class SocialAbstract { - private fetchInstance = pThrottleInstance( - (url: RequestInfo, options?: RequestInit) => fetch(url, options) - ); + abstract identifier: string; public handleErrors( body: string @@ -44,7 +37,7 @@ export abstract class SocialAbstract { identifier = '', totalRetries = 0 ): Promise { - const request = await this.fetchInstance(url, options); + const request = await concurrencyService(this.identifier.split('-')[0], () => fetch(url, options)); if (request.status === 200 || request.status === 201) { return request; diff --git a/package.json b/package.json index afe74c91..0d9f3684 100644 --- a/package.json +++ b/package.json @@ -172,7 +172,6 @@ "nx": "19.7.2", "openai": "^4.47.1", "p-limit": "^3.1.0", - "p-throttle": "4.1.1", "polotno": "^2.10.5", "posthog-js": "^1.178.0", "react": "18.3.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 559da6f8..5b6bafb7 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -390,9 +390,6 @@ importers: p-limit: specifier: ^3.1.0 version: 3.1.0 - p-throttle: - specifier: 4.1.1 - version: 4.1.1 polotno: specifier: ^2.10.5 version: 2.25.1(@types/react@18.3.1)(@types/sortablejs@1.15.8)(react-dom@18.3.1(react@18.3.1))(react-native@0.80.1(@babel/core@7.28.0)(@types/react@18.3.1)(bufferutil@4.0.9)(react@18.3.1)(utf-8-validate@5.0.10))(react@18.3.1) @@ -12031,6 +12028,7 @@ packages: node-domexception@1.0.0: resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==} engines: {node: '>=10.5.0'} + deprecated: Use your platform's native DOMException instead node-emoji@1.11.0: resolution: {integrity: sha512-wo2DpQkQp7Sjm2A0cq+sN7EHKO6Sl0ctXeBdFZrL9T9+UywORbufTcTZxom8YqpLQt/FqNMUkOpkZrJVYSKD3A==} @@ -12401,10 +12399,6 @@ packages: resolution: {integrity: sha512-hEt02O4hUct5wtwg4H4KcWgDdm+l1bOaEy/hWzd8xtXB9BqxTWBBhb+2ImAtH4Cv4rPjV76xN3Zumqk3k3AhhQ==} engines: {node: '>=16.17'} - p-throttle@4.1.1: - resolution: {integrity: sha512-TuU8Ato+pRTPJoDzYD4s7ocJYcNSEZRvlxoq3hcPI2kZDZ49IQ1Wkj7/gDJc3X7XiEAAvRGtDzdXJI0tC3IL1g==} - engines: {node: '>=10'} - p-timeout@3.2.0: resolution: {integrity: sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==} engines: {node: '>=8'} @@ -20055,21 +20049,6 @@ snapshots: - typescript - verdaccio - '@nrwl/js@19.7.2(@babel/traverse@7.28.0)(@swc-node/register@1.9.2(@swc/core@1.5.7(@swc/helpers@0.5.13))(@swc/types@0.1.7)(typescript@5.5.4))(@swc/core@1.5.7(@swc/helpers@0.5.13))(@types/node@18.16.9)(nx@19.7.2(@swc-node/register@1.9.2(@swc/core@1.5.7(@swc/helpers@0.5.13))(@swc/types@0.1.7)(typescript@5.5.4))(@swc/core@1.5.7(@swc/helpers@0.5.13)))(typescript@5.5.4)': - dependencies: - '@nx/js': 19.7.2(@babel/traverse@7.28.0)(@swc-node/register@1.9.2(@swc/core@1.5.7(@swc/helpers@0.5.13))(@swc/types@0.1.7)(typescript@5.5.4))(@swc/core@1.5.7(@swc/helpers@0.5.13))(@types/node@18.16.9)(nx@19.7.2(@swc-node/register@1.9.2(@swc/core@1.5.7(@swc/helpers@0.5.13))(@swc/types@0.1.7)(typescript@5.5.4))(@swc/core@1.5.7(@swc/helpers@0.5.13)))(typescript@5.5.4) - transitivePeerDependencies: - - '@babel/traverse' - - '@swc-node/register' - - '@swc/core' - - '@swc/wasm' - - '@types/node' - - debug - - nx - - supports-color - - typescript - - verdaccio - '@nrwl/nest@19.7.2(@babel/traverse@7.28.0)(@swc-node/register@1.9.2(@swc/core@1.5.7(@swc/helpers@0.5.13))(@swc/types@0.1.7)(typescript@5.5.4))(@swc/core@1.5.7(@swc/helpers@0.5.13))(@types/node@18.16.9)(@zkochan/js-yaml@0.0.7)(babel-plugin-macros@3.1.0)(chokidar@3.5.3)(eslint@8.57.0)(nx@19.7.2(@swc-node/register@1.9.2(@swc/core@1.5.7(@swc/helpers@0.5.13))(@swc/types@0.1.7)(typescript@5.5.4))(@swc/core@1.5.7(@swc/helpers@0.5.13)))(ts-node@10.9.2(@swc/core@1.5.7(@swc/helpers@0.5.13))(@types/node@18.16.9)(typescript@5.5.4))(typescript@5.5.4)': dependencies: '@nx/nest': 19.7.2(@babel/traverse@7.28.0)(@swc-node/register@1.9.2(@swc/core@1.5.7(@swc/helpers@0.5.13))(@swc/types@0.1.7)(typescript@5.5.4))(@swc/core@1.5.7(@swc/helpers@0.5.13))(@types/node@18.16.9)(@zkochan/js-yaml@0.0.7)(babel-plugin-macros@3.1.0)(chokidar@3.5.3)(eslint@8.57.0)(nx@19.7.2(@swc-node/register@1.9.2(@swc/core@1.5.7(@swc/helpers@0.5.13))(@swc/types@0.1.7)(typescript@5.5.4))(@swc/core@1.5.7(@swc/helpers@0.5.13)))(ts-node@10.9.2(@swc/core@1.5.7(@swc/helpers@0.5.13))(@types/node@18.16.9)(typescript@5.5.4))(typescript@5.5.4) @@ -20416,7 +20395,7 @@ snapshots: '@babel/preset-env': 7.28.0(@babel/core@7.28.0) '@babel/preset-typescript': 7.27.1(@babel/core@7.28.0) '@babel/runtime': 7.27.6 - '@nrwl/js': 19.7.2(@babel/traverse@7.28.0)(@swc-node/register@1.9.2(@swc/core@1.5.7(@swc/helpers@0.5.13))(@swc/types@0.1.7)(typescript@5.5.4))(@swc/core@1.5.7(@swc/helpers@0.5.13))(@types/node@18.16.9)(nx@19.7.2(@swc-node/register@1.9.2(@swc/core@1.5.7(@swc/helpers@0.5.13))(@swc/types@0.1.7)(typescript@5.5.4))(@swc/core@1.5.7(@swc/helpers@0.5.13)))(typescript@5.5.4) + '@nrwl/js': 19.7.2(@babel/traverse@7.28.0)(@swc-node/register@1.9.2(@swc/core@1.5.7(@swc/helpers@0.5.13))(@swc/types@0.1.7)(typescript@5.5.4))(@swc/core@1.5.7(@swc/helpers@0.5.13))(@types/node@18.16.9)(nx@19.7.2(@swc-node/register@1.9.2(@swc/core@1.5.7(@swc/helpers@0.5.13))(@swc/types@0.1.7)(typescript@5.5.4))(@swc/core@1.5.7(@swc/helpers@0.5.13)))(typescript@5.4.5) '@nx/devkit': 19.7.2(nx@19.7.2(@swc-node/register@1.9.2(@swc/core@1.5.7(@swc/helpers@0.5.13))(@swc/types@0.1.7)(typescript@5.5.4))(@swc/core@1.5.7(@swc/helpers@0.5.13))) '@nx/workspace': 19.7.2(@swc-node/register@1.9.2(@swc/core@1.5.7(@swc/helpers@0.5.13))(@swc/types@0.1.7)(typescript@5.5.4))(@swc/core@1.5.7(@swc/helpers@0.5.13)) babel-plugin-const-enum: 1.2.0(@babel/core@7.28.0) @@ -32060,8 +32039,6 @@ snapshots: is-network-error: 1.1.0 retry: 0.13.1 - p-throttle@4.1.1: {} - p-timeout@3.2.0: dependencies: p-finally: 1.0.0