jeffemmett
/
postiz
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: better performance

This commit is contained in:
Nevo David 2024-03-04 13:53:57 +07:00
parent 515d6413b2
commit 9329f1fedd
13 changed files with 627 additions and 232 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
apps/backend/src/api/routes/posts.controller.ts
View File

@ -14,12 +14,14 @@ import { Organization } from '@prisma/client';
import { CreatePostDto } from '@gitroom/nestjs-libraries/dtos/posts/create.post.dto';
import { GetPostsDto } from '@gitroom/nestjs-libraries/dtos/posts/get.posts.dto';
import { CommentsService } from '@gitroom/nestjs-libraries/database/prisma/comments/comments.service';
import {StarsService} from "@gitroom/nestjs-libraries/database/prisma/stars/stars.service";
@Controller('/posts')
export class PostsController {
constructor(
private _postsService: PostsService,
private _commentsService: CommentsService
private _commentsService: CommentsService,
private _starsService: StarsService
) {}
@Get('/')
@ -42,6 +44,11 @@ export class PostsController {
};
}
@Get('/predict-trending')
predictTrending() {
return this._starsService.predictTrending();
}
@Get('/old')
oldPosts(
@GetOrgFromRequest() org: Organization,

36
apps/frontend/src/components/launches/add.edit.model.tsx
View File

@ -1,6 +1,6 @@
'use client';
import React, { FC, useCallback, useEffect, useState } from 'react';
import React, {FC, MouseEventHandler, useCallback, useEffect, useState} from 'react';
import dayjs from 'dayjs';
import { Integrations } from '@gitroom/frontend/components/launches/calendar.context';
import clsx from 'clsx';
@ -159,6 +159,13 @@ export const AddEditModal: FC<{
// sometimes it's easier to click escape to close
useKeypress('Escape', askClose);
const postNow = useCallback(((e) => {
e.stopPropagation();
e.preventDefault();
return schedule('now')();
}) as MouseEventHandler<HTMLDivElement>, []);
// function to send to the server and save
const schedule = useCallback(
(type: 'draft' | 'now' | 'schedule' | 'delete') => async () => {
@ -363,7 +370,7 @@ export const AddEditModal: FC<{
</div>
<div className="relative h-[68px] flex flex-col rounded-[4px] border border-[#172034] bg-[#0B101B]">
<div className="flex flex-1 gap-[10px] relative">
<div className="absolute w-full h-full flex gap-[10px] justify-end items-center right-[16px] overflow-hidden">
<div className="absolute w-full h-full flex gap-[10px] justify-end items-center right-[16px]">
{!!existingData.integration && (
<Button
onClick={schedule('delete')}
@ -384,10 +391,31 @@ export const AddEditModal: FC<{
<Button
onClick={schedule('schedule')}
className="rounded-[4px]"
className="rounded-[4px] relative"
disabled={selectedIntegrations.length === 0}
>
{!existingData.integration ? 'Add to calendar' : 'Update'}
<div className="flex justify-center items-center gap-[5px] h-full">
<div className="h-full flex items-center">
{!existingData.integration ? 'Add to calendar' : 'Update'}
</div>
<div className="group h-full flex items-center">
<svg
xmlns="http://www.w3.org/2000/svg"
width="18"
height="18"
viewBox="0 0 18 18"
fill="none"
>
<path
d="M15.0233 7.14804L9.39828 12.773C9.34604 12.8253 9.284 12.8668 9.21572 12.8951C9.14743 12.9234 9.07423 12.938 9.00031 12.938C8.92639 12.938 8.8532 12.9234 8.78491 12.8951C8.71662 12.8668 8.65458 12.8253 8.60234 12.773L2.97734 7.14804C2.8718 7.04249 2.8125 6.89934 2.8125 6.75007C2.8125 6.6008 2.8718 6.45765 2.97734 6.3521C3.08289 6.24655 3.22605 6.18726 3.37531 6.18726C3.52458 6.18726 3.66773 6.24655 3.77328 6.3521L9.00031 11.5798L14.2273 6.3521C14.2796 6.29984 14.3417 6.25838 14.4099 6.2301C14.4782 6.20181 14.5514 6.18726 14.6253 6.18726C14.6992 6.18726 14.7724 6.20181 14.8407 6.2301C14.909 6.25838 14.971 6.29984 15.0233 6.3521C15.0755 6.40436 15.117 6.46641 15.1453 6.53469C15.1736 6.60297 15.1881 6.67616 15.1881 6.75007C15.1881 6.82398 15.1736 6.89716 15.1453 6.96545C15.117 7.03373 15.0755 7.09578 15.0233 7.14804Z"
fill="white"
/>
</svg>
<div onClick={postNow} className="hidden group-hover:flex hover:flex flex-col justify-center absolute left-0 top-[100%] w-full h-[40px] bg-[#B91C1C] border border-tableBorder">
Post now
</div>
</div>
</div>
</Button>
</div>
</div>

22
apps/frontend/src/components/launches/calendar.context.tsx
View File

@ -28,6 +28,7 @@ const CalendarContext = createContext({
currentYear: dayjs().year(),
comments: [] as Array<{ date: string; total: number }>,
integrations: [] as Integrations[],
trendings: [] as string[],
posts: [] as Array<Post & { integration: Integration }>,
setFilters: (filters: { currentWeek: number; currentYear: number }) => {},
changeDate: (id: string, date: dayjs.Dayjs) => {},
@ -46,8 +47,15 @@ export const CalendarWeekProvider: FC<{
}> = ({ children, integrations }) => {
const fetch = useFetch();
const [internalData, setInternalData] = useState([] as any[]);
const [trendings, setTrendings] = useState<string[]>([]);
const { mutate } = useSWRConfig();
useEffect(() => {
(async () => {
setTrendings(await (await fetch('/posts/predict-trending')).json());
})();
}, []);
const [filters, setFilters] = useState({
currentWeek: dayjs().week(),
currentYear: dayjs().year(),
@ -72,12 +80,18 @@ export const CalendarWeekProvider: FC<{
const loadData = useCallback(
async (url: string) => {
return (await fetch(`${url}?${params}`)).json();
const data = (await fetch(`${url}?${params}`)).json();
return data;
},
[filters]
);
const swr = useSWR(`/posts`, loadData);
const swr = useSWR(`/posts`, loadData, {
refreshInterval: 3600000,
refreshWhenOffline: false,
refreshWhenHidden: false,
revalidateOnFocus: false,
});
const { isLoading } = swr;
const { posts, comments } = swr?.data || { posts: [], comments: [] };
@ -86,7 +100,7 @@ export const CalendarWeekProvider: FC<{
setInternalData((d) =>
d.map((post: Post) => {
if (post.id === id) {
return { ...post, publishDate: date.format('YYYY-MM-DDTHH:mm:ss') };
return { ...post, publishDate: date.utc().format('YYYY-MM-DDTHH:mm:ss') };
}
return post;
})
@ -101,9 +115,11 @@ export const CalendarWeekProvider: FC<{
}
}, [posts]);
return (
<CalendarContext.Provider
value={{
trendings,
...filters,
posts: isLoading ? [] : internalData,
integrations,

104
apps/frontend/src/components/launches/calendar.tsx
View File

@ -1,11 +1,14 @@
'use client';
import { FC, useCallback, useMemo } from 'react';
import { FC, useCallback, useEffect, useMemo, useState, JSX } from 'react';
import {
Integrations,
useCalendar,
} from '@gitroom/frontend/components/launches/calendar.context';
import dayjs from 'dayjs';
import isBetween from 'dayjs/plugin/isBetween';
dayjs.extend(isBetween);
import { openModal, useModals } from '@mantine/modals';
import { AddEditModal } from '@gitroom/frontend/components/launches/add.edit.model';
import clsx from 'clsx';
@ -17,8 +20,9 @@ import { Integration, Post } from '@prisma/client';
import { useAddProvider } from '@gitroom/frontend/components/launches/add.provider.component';
import { CommentComponent } from '@gitroom/frontend/components/launches/comments/comment.component';
import { useSWRConfig } from 'swr';
import { useIntersectionObserver } from '@uidotdev/usehooks';
const days = [
export const days = [
'',
'Monday',
'Tuesday',
@ -28,7 +32,7 @@ const days = [
'Saturday',
'Sunday',
];
const hours = [
export const hours = [
'00:00',
'01:00',
'02:00',
@ -139,8 +143,67 @@ export const Calendar = () => {
const CalendarColumn: FC<{ day: number; hour: string }> = (props) => {
const { day, hour } = props;
const { currentWeek, currentYear, integrations, posts, changeDate } =
useCalendar();
const { currentWeek, currentYear } = useCalendar();
const getDate = useMemo(() => {
const date =
dayjs()
.year(currentYear)
.isoWeek(currentWeek)
.isoWeekday(day)
.format('YYYY-MM-DD') +
'T' +
hour +
':00';
return dayjs(date);
}, [currentWeek]);
const isBeforeNow = useMemo(() => {
return getDate.isBefore(dayjs());
}, [getDate]);
const [ref, entry] = useIntersectionObserver({
threshold: 0.5,
root: null,
rootMargin: '0px',
});
return (
<div className="w-full h-full" ref={ref}>
{!entry?.isIntersecting ? (
<div
className={clsx(
'h-full flex justify-center items-center text-[12px]',
isBeforeNow && 'bg-secondary'
)}
>
{!isBeforeNow && (
<div
className={clsx(
'w-[20px] h-[20px] bg-forth rounded-full flex justify-center items-center hover:bg-seventh'
)}
>
+
</div>
)}
</div>
) : (
<CalendarColumnRender {...props} />
)}
</div>
);
};
const CalendarColumnRender: FC<{ day: number; hour: string }> = (props) => {
const { day, hour } = props;
const {
currentWeek,
currentYear,
integrations,
posts,
trendings,
changeDate,
} = useCalendar();
const modal = useModals();
const fetch = useFetch();
@ -159,10 +222,22 @@ const CalendarColumn: FC<{ day: number; hour: string }> = (props) => {
const postList = useMemo(() => {
return posts.filter((post) => {
return dayjs(post.publishDate).local().isSame(getDate);
return dayjs
.utc(post.publishDate)
.local()
.isBetween(getDate, getDate.add(10, 'minute'), 'minute', '[)');
});
}, [posts]);
const canBeTrending = useMemo(() => {
return !!trendings.find((trend) => {
return dayjs
.utc(trend)
.local()
.isBetween(getDate, getDate.add(10, 'minute'), 'minute', '[)');
});
}, [trendings]);
const isBeforeNow = useMemo(() => {
return getDate.isBefore(dayjs());
}, [getDate]);
@ -232,11 +307,18 @@ const CalendarColumn: FC<{ day: number; hour: string }> = (props) => {
<div className="relative w-full h-full">
<div className="absolute left-0 top-0 w-full h-full">
<div
{...(canBeTrending
? {
'data-tooltip-id': 'tooltip',
'data-tooltip-content': 'Predicted GitHub Trending Change',
}
: {})}
ref={drop}
className={clsx(
'h-[calc(216px/6)] gap-[2.5px] text-[12px] pointer w-full overflow-hidden justify-center overflow-x-auto flex scrollbar scrollbar-thumb-tableBorder scrollbar-track-secondary',
isBeforeNow && 'bg-secondary',
canDrop && 'bg-white/80'
canDrop && 'bg-white/80',
canBeTrending && 'bg-[#eaff00]'
)}
>
{postList.map((post) => (
@ -351,7 +433,13 @@ export const CommentBox: FC<{ totalComments: number; date: dayjs.Dayjs }> = (
}, [date]);
return (
<div className={totalComments === 0 ? 'transition-opacity opacity-0 group-hover:opacity-100' : ''}>
<div
className={
totalComments === 0
? 'transition-opacity opacity-0 group-hover:opacity-100'
: ''
}
>
<div
onClick={openCommentsModal}
data-tooltip-id="tooltip"

2
apps/frontend/src/components/launches/filters.tsx
View File

@ -25,7 +25,7 @@ export const Filters = () => {
}, [week.currentWeek, week.currentYear]);
return (
<div className="text-white h-[50px] flex gap-[8px] items-center select-none">
<div className="h-[20px] text-white flex gap-[8px] items-center select-none">
<div onClick={previousWeek}>
<svg
xmlns="http://www.w3.org/2000/svg"

4
apps/frontend/src/components/launches/launches.component.tsx
View File

@ -22,7 +22,9 @@ export const LaunchesComponent: FC<{
<CalendarWeekProvider integrations={sortedIntegrations}>
<div className="flex flex-1 flex-col">
<div className="flex flex-1 relative">
<div className="absolute w-full h-full flex flex-1 gap-[30px] overflow-hidden overflow-y-scroll scrollbar scrollbar-thumb-tableBorder scrollbar-track-secondary">
<div
className="absolute w-full h-full flex flex-1 gap-[30px] overflow-hidden overflow-y-scroll scrollbar scrollbar-thumb-tableBorder scrollbar-track-secondary"
>
<div className="w-[220px] bg-third p-[16px] flex flex-col gap-[24px] sticky top-0">
<h2 className="text-[20px]">Channels</h2>
<div className="gap-[16px] flex flex-col">

2
libraries/nestjs-libraries/src/database/prisma/posts/posts.repository.ts
View File

@ -154,7 +154,7 @@ export class PostsRepository {
}
async createOrUpdatePost(
state: 'draft' | 'schedule',
state: 'draft' | 'schedule' | 'now',
orgId: string,
date: string,
body: PostBody

6
libraries/nestjs-libraries/src/database/prisma/posts/posts.service.ts
View File

@ -163,7 +163,7 @@ export class PostsService {
await this._postRepository.createOrUpdatePost(
body.type,
orgId,
body.date,
body.type === 'now' ? dayjs().format('YYYY-MM-DDTHH:mm:00') : body.date,
post
);
@ -172,11 +172,11 @@ export class PostsService {
'post',
previousPost ? previousPost : posts?.[0]?.id
);
if (body.type === 'schedule' && dayjs(body.date).isAfter(dayjs())) {
if ((body.type === 'schedule' || body.type === 'now') && dayjs(body.date).isAfter(dayjs())) {
this._workerServiceProducer.emit('post', {
id: posts[0].id,
options: {
delay: 0, //dayjs(posts[0].publishDate).diff(dayjs(), 'millisecond'),
delay: body.type === 'now' ? 0 : dayjs(posts[0].publishDate).diff(dayjs(), 'millisecond'),
},
payload: {
id: posts[0].id,

558
libraries/nestjs-libraries/src/database/prisma/stars/stars.service.ts
View File

@ -1,231 +1,373 @@
import {Injectable} from "@nestjs/common";
import {StarsRepository} from "@gitroom/nestjs-libraries/database/prisma/stars/stars.repository";
import {chunk, groupBy} from "lodash";
import dayjs from "dayjs";
import {NotificationService} from "@gitroom/nestjs-libraries/notifications/notification.service";
import {StarsListDto} from "@gitroom/nestjs-libraries/dtos/analytics/stars.list.dto";
import {BullMqClient} from "@gitroom/nestjs-libraries/bull-mq-transport/client/bull-mq.client";
import { Injectable } from '@nestjs/common';
import { StarsRepository } from '@gitroom/nestjs-libraries/database/prisma/stars/stars.repository';
import { chunk, groupBy } from 'lodash';
import dayjs from 'dayjs';
import { NotificationService } from '@gitroom/nestjs-libraries/notifications/notification.service';
import { StarsListDto } from '@gitroom/nestjs-libraries/dtos/analytics/stars.list.dto';
import { BullMqClient } from '@gitroom/nestjs-libraries/bull-mq-transport/client/bull-mq.client';
import { mean } from 'simple-statistics';
enum Inform {
Removed,
New,
Changed
Removed,
New,
Changed,
}
@Injectable()
export class StarsService {
constructor(
private _starsRepository: StarsRepository,
private _notificationsService: NotificationService,
private _workerServiceProducer: BullMqClient
){}
constructor(
private _starsRepository: StarsRepository,
private _notificationsService: NotificationService,
private _workerServiceProducer: BullMqClient
) {}
getGitHubRepositoriesByOrgId(org: string) {
return this._starsRepository.getGitHubRepositoriesByOrgId(org);
getGitHubRepositoriesByOrgId(org: string) {
return this._starsRepository.getGitHubRepositoriesByOrgId(org);
}
getAllGitHubRepositories() {
return this._starsRepository.getAllGitHubRepositories();
}
getStarsByLogin(login: string) {
return this._starsRepository.getStarsByLogin(login);
}
getLastStarsByLogin(login: string) {
return this._starsRepository.getLastStarsByLogin(login);
}
createStars(
login: string,
totalNewsStars: number,
totalStars: number,
date: Date
) {
return this._starsRepository.createStars(
login,
totalNewsStars,
totalStars,
date
);
}
async sync(login: string) {
const loadAllStars = await this.syncProcess(login);
const sortedArray = Object.keys(loadAllStars).sort(
(a, b) => dayjs(a).unix() - dayjs(b).unix()
);
let addPreviousStars = 0;
for (const date of sortedArray) {
const dateObject = dayjs(date).toDate();
addPreviousStars += loadAllStars[date];
await this._starsRepository.createStars(
login,
loadAllStars[date],
addPreviousStars,
dateObject
);
}
}
async syncProcess(login: string, page = 1) {
const starsRequest = await fetch(
`https://api.github.com/repos/${login}/stargazers?page=${page}&per_page=100`,
{
headers: {
Accept: 'application/vnd.github.v3.star+json',
...(process.env.GITHUB_AUTH
? { Authorization: `token ${process.env.GITHUB_AUTH}` }
: {}),
},
}
);
const totalRemaining = +(
starsRequest.headers.get('x-ratelimit-remaining') ||
starsRequest.headers.get('X-RateLimit-Remaining') ||
0
);
const resetTime = +(
starsRequest.headers.get('x-ratelimit-reset') ||
starsRequest.headers.get('X-RateLimit-Reset') ||
0
);
if (totalRemaining < 10) {
console.log('waiting for the rate limit');
const delay = resetTime * 1000 - Date.now() + 1000;
await new Promise((resolve) => setTimeout(resolve, delay));
}
getAllGitHubRepositories() {
return this._starsRepository.getAllGitHubRepositories();
const data: Array<{ starred_at: string }> = await starsRequest.json();
const mapDataToDate = groupBy(data, (p) =>
dayjs(p.starred_at).format('YYYY-MM-DD')
);
// take all the stars from the page
const aggStars: { [key: string]: number } = Object.values(
mapDataToDate
).reduce(
(acc, value) => ({
...acc,
[value[0].starred_at]: value.length,
}),
{}
);
// if we have 100 stars, we need to fetch the next page and merge the results (recursively)
const nextOne: { [key: string]: number } =
data.length === 100 ? await this.syncProcess(login, page + 1) : {};
// merge the results
const allKeys = [
...new Set([...Object.keys(aggStars), ...Object.keys(nextOne)]),
];
return {
...allKeys.reduce(
(acc, key) => ({
...acc,
[key]: (aggStars[key] || 0) + (nextOne[key] || 0),
}),
{} as { [key: string]: number }
),
};
}
async updateTrending(
language: string,
hash: string,
arr: Array<{ name: string; position: number }>
) {
const currentTrending = await this._starsRepository.getTrendingByLanguage(
language
);
if (currentTrending?.hash === hash) {
return;
}
await this.newTrending(language);
if (currentTrending) {
const list: Array<{ name: string; position: number }> = JSON.parse(
currentTrending.trendingList
);
const removedFromTrending = list.filter(
(p) => !arr.find((a) => a.name === p.name)
);
const changedPosition = arr.filter((p) => {
const current = list.find((a) => a.name === p.name);
return current && current.position !== p.position;
});
if (removedFromTrending.length) {
// let people know they are not trending anymore
await this.inform(Inform.Removed, removedFromTrending, language);
}
if (changedPosition.length) {
// let people know they changed position
await this.inform(Inform.Changed, changedPosition, language);
}
}
getStarsByLogin(login: string) {
return this._starsRepository.getStarsByLogin(login);
}
const informNewPeople = arr.filter(
(p) => currentTrending?.trendingList?.indexOf(p.name) === -1
);
getLastStarsByLogin(login: string) {
return this._starsRepository.getLastStarsByLogin(login);
}
// let people know they are trending
await this.inform(Inform.New, informNewPeople, language);
await this.replaceOrAddTrending(language, hash, arr);
}
createStars(login: string, totalNewsStars: number, totalStars: number, date: Date) {
return this._starsRepository.createStars(login, totalNewsStars, totalStars, date);
}
async sync(login: string) {
const loadAllStars = await this.syncProcess(login);
const sortedArray = Object.keys(loadAllStars).sort((a, b) => dayjs(a).unix() - dayjs(b).unix());
let addPreviousStars = 0;
for (const date of sortedArray) {
const dateObject = dayjs(date).toDate();
addPreviousStars += loadAllStars[date];
await this._starsRepository.createStars(login, loadAllStars[date], addPreviousStars, dateObject);
async inform(
type: Inform,
removedFromTrending: Array<{ name: string; position: number }>,
language: string
) {
const names = await this._starsRepository.getGitHubsByNames(
removedFromTrending.map((p) => p.name)
);
const mapDbNamesToList = names.map(
(n) => removedFromTrending.find((p) => p.name === n.login)!
);
for (const person of mapDbNamesToList) {
const getOrganizationsByGitHubLogin =
await this._starsRepository.getOrganizationsByGitHubLogin(person.name);
for (const org of getOrganizationsByGitHubLogin) {
const topic = `organization:${org.organizationId}`;
switch (type) {
case Inform.Removed:
return this._notificationsService.sendNotificationToTopic(
'trending',
topic,
{ message: `You are not trending anymore in ${language}` }
);
case Inform.New:
return this._notificationsService.sendNotificationToTopic(
'trending',
topic,
{
message: `You are trending in ${
language || 'On the main feed'
} position #${person.position}`,
}
);
case Inform.Changed:
return this._notificationsService.sendNotificationToTopic(
'trending',
topic,
{
message: `You changed position in ${
language || 'On the main feed'
} position #${person.position}`,
}
);
}
}
}
}
async syncProcess(login: string, page = 1) {
const starsRequest = await fetch(`https://api.github.com/repos/${login}/stargazers?page=${page}&per_page=100`, {
headers: {
Accept: 'application/vnd.github.v3.star+json',
...process.env.GITHUB_AUTH ? {Authorization: `token ${process.env.GITHUB_AUTH}`} : {}
}
});
const totalRemaining = +(starsRequest.headers.get('x-ratelimit-remaining') || starsRequest.headers.get('X-RateLimit-Remaining') || 0);
const resetTime = +(starsRequest.headers.get('x-ratelimit-reset') || starsRequest.headers.get('X-RateLimit-Reset') || 0);
async replaceOrAddTrending(
language: string,
hash: string,
arr: Array<{ name: string; position: number }>
) {
return this._starsRepository.replaceOrAddTrending(language, hash, arr);
}
if (totalRemaining < 10) {
console.log('waiting for the rate limit');
const delay = (resetTime * 1000) - Date.now() + 1000;
await new Promise(resolve => setTimeout(resolve, delay));
}
async newTrending(language: string) {
return this._starsRepository.newTrending(language);
}
const data: Array<{starred_at: string}> = await starsRequest.json();
const mapDataToDate = groupBy(data, (p) => dayjs(p.starred_at).format('YYYY-MM-DD'));
async getStars(org: string) {
const getGitHubs = await this.getGitHubRepositoriesByOrgId(org);
const list = [];
for (const gitHub of getGitHubs) {
if (!gitHub.login) {
continue;
}
const stars = await this.getStarsByLogin(gitHub.login!);
const graphSize = stars.length < 10 ? stars.length : stars.length / 10;
// take all the stars from the page
const aggStars: {[key: string]: number} = Object.values(mapDataToDate).reduce((acc, value) => ({
list.push({
login: gitHub.login,
stars: chunk(stars, graphSize).reduce((acc, chunkedStars) => {
return [
...acc,
[value[0].starred_at]: value.length,
}), {});
// if we have 100 stars, we need to fetch the next page and merge the results (recursively)
const nextOne: {[key: string]: number} = (data.length === 100) ? await this.syncProcess(login, page + 1) : {};
// merge the results
const allKeys = [...new Set([...Object.keys(aggStars), ...Object.keys(nextOne)])];
return {
...allKeys.reduce((acc, key) => ({
...acc,
[key]: (aggStars[key] || 0) + (nextOne[key] || 0)
}), {} as {[key: string]: number})
};
}
async updateTrending(language: string, hash: string, arr: Array<{name: string, position: number}>) {
const currentTrending = await this._starsRepository.getTrendingByLanguage(language);
if (currentTrending?.hash === hash) {
return;
}
await this.newTrending(language);
if (currentTrending) {
const list: Array<{name: string, position: number}> = JSON.parse(currentTrending.trendingList);
const removedFromTrending = list.filter(p => !arr.find(a => a.name === p.name));
const changedPosition = arr.filter(p => {
const current = list.find(a => a.name === p.name);
return current && current.position !== p.position;
});
if (removedFromTrending.length) {
// let people know they are not trending anymore
await this.inform(Inform.Removed, removedFromTrending, language);
}
if (changedPosition.length) {
// let people know they changed position
await this.inform(Inform.Changed, changedPosition, language);
}
}
const informNewPeople = arr.filter(p => currentTrending?.trendingList?.indexOf(p.name) === -1);
// let people know they are trending
await this.inform(Inform.New, informNewPeople, language);
await this.replaceOrAddTrending(language, hash, arr);
}
async inform(type: Inform, removedFromTrending: Array<{name: string, position: number}>, language: string) {
const names = await this._starsRepository.getGitHubsByNames(removedFromTrending.map(p => p.name));
const mapDbNamesToList = names.map(n => removedFromTrending.find(p => p.name === n.login)!);
for (const person of mapDbNamesToList) {
const getOrganizationsByGitHubLogin = await this._starsRepository.getOrganizationsByGitHubLogin(person.name);
for (const org of getOrganizationsByGitHubLogin) {
const topic = `organization:${org.organizationId}`;
switch (type) {
case Inform.Removed:
return this._notificationsService.sendNotificationToTopic('trending', topic, {message: `You are not trending anymore in ${language}`});
case Inform.New:
return this._notificationsService.sendNotificationToTopic('trending', topic, {message: `You are trending in ${language || 'On the main feed'} position #${person.position}`});
case Inform.Changed:
return this._notificationsService.sendNotificationToTopic( 'trending', topic, {message: `You changed position in ${language || 'On the main feed'} position #${person.position}`});
}
}
}
}
async replaceOrAddTrending(language: string, hash: string, arr: Array<{name: string, position: number}>) {
return this._starsRepository.replaceOrAddTrending(language, hash, arr);
}
async newTrending(language: string) {
return this._starsRepository.newTrending(language);
}
async getStars(org: string) {
const getGitHubs = await this.getGitHubRepositoriesByOrgId(org);
const list = [];
for (const gitHub of getGitHubs) {
if (!gitHub.login) {
continue;
}
const stars = await this.getStarsByLogin(gitHub.login!);
const graphSize = stars.length < 10 ? stars.length : stars.length / 10;
list.push({
login: gitHub.login,
stars: chunk(stars, graphSize).reduce((acc, chunkedStars) => {
return [
...acc,
{
totalStars: chunkedStars[chunkedStars.length - 1].totalStars,
date: chunkedStars[chunkedStars.length - 1].date
}
]
}, [] as Array<{totalStars: number, date: Date}>)
});
}
return list;
}
async getTrending(language: string) {
return this._starsRepository.getLastTrending(language);
}
async getStarsFilter(orgId: string, starsFilter: StarsListDto) {
const getGitHubs = await this.getGitHubRepositoriesByOrgId(orgId);
if (getGitHubs.filter(f => f.login).length === 0) {
return [];
}
return this._starsRepository.getStarsFilter(getGitHubs.map(p => p.login) as string[], starsFilter);
}
async addGitHub(orgId: string, code: string) {
const {access_token} = await (await fetch('https://github.com/login/oauth/access_token', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
Accept: 'application/json'
{
totalStars: chunkedStars[chunkedStars.length - 1].totalStars,
date: chunkedStars[chunkedStars.length - 1].date,
},
body: JSON.stringify({
client_id: process.env.GITHUB_CLIENT_ID,
client_secret: process.env.GITHUB_CLIENT_SECRET,
code,
redirect_uri: `${process.env.FRONTEND_URL}/settings`
})
})).json();
return this._starsRepository.addGitHub(orgId, access_token);
];
}, [] as Array<{ totalStars: number; date: Date }>),
});
}
async getOrganizations(orgId: string, id: string) {
const getGitHub = await this._starsRepository.getGitHubById(orgId, id);
return (await fetch(`https://api.github.com/user/orgs`, {
headers: {
Authorization: `token ${getGitHub?.token!}`
}
})).json();
return list;
}
async getTrending(language: string) {
return this._starsRepository.getLastTrending(language);
}
async getStarsFilter(orgId: string, starsFilter: StarsListDto) {
const getGitHubs = await this.getGitHubRepositoriesByOrgId(orgId);
if (getGitHubs.filter((f) => f.login).length === 0) {
return [];
}
return this._starsRepository.getStarsFilter(
getGitHubs.map((p) => p.login) as string[],
starsFilter
);
}
async addGitHub(orgId: string, code: string) {
const { access_token } = await (
await fetch('https://github.com/login/oauth/access_token', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
Accept: 'application/json',
},
body: JSON.stringify({
client_id: process.env.GITHUB_CLIENT_ID,
client_secret: process.env.GITHUB_CLIENT_SECRET,
code,
redirect_uri: `${process.env.FRONTEND_URL}/settings`,
}),
})
).json();
return this._starsRepository.addGitHub(orgId, access_token);
}
async getOrganizations(orgId: string, id: string) {
const getGitHub = await this._starsRepository.getGitHubById(orgId, id);
return (
await fetch(`https://api.github.com/user/orgs`, {
headers: {
Authorization: `token ${getGitHub?.token!}`,
},
})
).json();
}
async getRepositoriesOfOrganization(
orgId: string,
id: string,
github: string
) {
const getGitHub = await this._starsRepository.getGitHubById(orgId, id);
return (
await fetch(`https://api.github.com/orgs/${github}/repos`, {
headers: {
Authorization: `token ${getGitHub?.token!}`,
},
})
).json();
}
async updateGitHubLogin(orgId: string, id: string, login: string) {
this._workerServiceProducer
.emit('sync_all_stars', { payload: { login } })
.subscribe();
return this._starsRepository.updateGitHubLogin(orgId, id, login);
}
async deleteRepository(orgId: string, id: string) {
return this._starsRepository.deleteRepository(orgId, id);
}
async predictTrending() {
const trendings = (await this.getTrending('')).reverse();
const dates = await this.predictTrendingLoop(trendings);
return dates.map(d => dayjs(d).format('YYYY-MM-DDTHH:mm:00'));
}
async predictTrendingLoop(trendings: Array<{ date: Date }>, current = 0): Promise<Date[]> {
const dates = trendings.map((result) => dayjs(result.date).toDate());
const intervals = dates
.slice(1)
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-expect-error
.map((date, i) => (date - dates[i]) / (1000 * 60 * 60 * 24));
const nextInterval = intervals.length === 0 ? null : mean(intervals);
const lastTrendingDate = dates[dates.length - 1];
const nextTrendingDate = !nextInterval
? false
: dayjs(
new Date(
lastTrendingDate.getTime() + nextInterval * 24 * 60 * 60 * 1000
)
).toDate();
if (!nextTrendingDate) {
return [];
}
async getRepositoriesOfOrganization(orgId: string, id: string, github: string) {
const getGitHub = await this._starsRepository.getGitHubById(orgId, id);
return (await fetch(`https://api.github.com/orgs/${github}/repos`, {
headers: {
Authorization: `token ${getGitHub?.token!}`
}
})).json();
}
async updateGitHubLogin(orgId: string, id: string, login: string) {
this._workerServiceProducer.emit('sync_all_stars', {payload: {login}}).subscribe();
return this._starsRepository.updateGitHubLogin(orgId, id, login);
}
async deleteRepository(orgId: string, id: string) {
return this._starsRepository.deleteRepository(orgId, id);
}
}
return [
nextTrendingDate,
...(current < 500
? await this.predictTrendingLoop(
[...trendings, { date: nextTrendingDate }],
current + 1
)
: []),
];
}
}

4
libraries/nestjs-libraries/src/dtos/posts/create.post.dto.ts
View File

@ -68,8 +68,8 @@ export class Post {
export class CreatePostDto {
@IsDefined()
@IsIn(['draft', 'schedule'])
type: 'draft' | 'schedule';
@IsIn(['draft', 'schedule', 'now'])
type: 'draft' | 'schedule' | 'now';
@IsDefined()
@IsDateString()

1
libraries/nestjs-libraries/src/services/trending.service.ts
View File

@ -23,6 +23,7 @@ export class TrendingService {
});
const hashedNames = md5(arr.map(p => p.name).join(''));
console.log(language, hashedNames);
await this._starsService.updateTrending(language.name, hashedNames, arr);
}
}

108
package-lock.json generated
View File

@ -27,6 +27,7 @@
"@prisma/client": "^5.8.1",
"@swc/helpers": "~0.5.2",
"@sweetalert2/theme-dark": "^5.0.16",
"@tanstack/react-virtual": "^3.1.3",
"@types/bcrypt": "^5.0.2",
"@types/jsonwebtoken": "^9.0.5",
"@types/lodash": "^4.14.202",
@ -34,7 +35,9 @@
"@types/mime-types": "^2.1.4",
"@types/remove-markdown": "^0.3.4",
"@types/stripe": "^8.0.417",
"@uidotdev/usehooks": "^2.4.1",
"@uiw/react-md-editor": "^4.0.3",
"@virtual-grid/react": "^2.0.2",
"axios": "^1.0.0",
"bcrypt": "^5.1.1",
"bullmq": "^5.1.5",
@ -4135,6 +4138,11 @@
"@jridgewell/sourcemap-codec": "^1.4.14"
}
},
"node_modules/@juggle/resize-observer": {
"version": "3.4.0",
"resolved": "https://registry.npmjs.org/@juggle/resize-observer/-/resize-observer-3.4.0.tgz",
"integrity": "sha512-dfLbk+PwWvFzSxwk3n5ySL0hfBog779o8h68wK/7/APo/7cgyWp5jcXockbxdk5kFRkbeXWm4Fbi9FrdN381sA=="
},
"node_modules/@kurkle/color": {
"version": "0.3.2",
"resolved": "https://registry.npmjs.org/@kurkle/color/-/color-0.3.2.tgz",
@ -6929,6 +6937,31 @@
}
}
},
"node_modules/@tanstack/react-virtual": {
"version": "3.1.3",
"resolved": "https://registry.npmjs.org/@tanstack/react-virtual/-/react-virtual-3.1.3.tgz",
"integrity": "sha512-YCzcbF/Ws/uZ0q3Z6fagH+JVhx4JLvbSflgldMgLsuvB8aXjZLLb3HvrEVxY480F9wFlBiXlvQxOyXb5ENPrNA==",
"dependencies": {
"@tanstack/virtual-core": "3.1.3"
},
"funding": {
"type": "github",
"url": "https://github.com/sponsors/tannerlinsley"
},
"peerDependencies": {
"react": "^16.8.0 || ^17.0.0 || ^18.0.0",
"react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0"
}
},
"node_modules/@tanstack/virtual-core": {
"version": "3.1.3",
"resolved": "https://registry.npmjs.org/@tanstack/virtual-core/-/virtual-core-3.1.3.tgz",
"integrity": "sha512-Y5B4EYyv1j9V8LzeAoOVeTg0LI7Fo5InYKgAjkY1Pu9GjtUwX/EKxNcU7ng3sKr99WEf+bPTcktAeybyMOYo+g==",
"funding": {
"type": "github",
"url": "https://github.com/sponsors/tannerlinsley"
}
},
"node_modules/@testing-library/dom": {
"version": "9.3.4",
"resolved": "https://registry.npmjs.org/@testing-library/dom/-/dom-9.3.4.tgz",
@ -7935,6 +7968,18 @@
"@ucast/mongo": "^2.4.0"
}
},
"node_modules/@uidotdev/usehooks": {
"version": "2.4.1",
"resolved": "https://registry.npmjs.org/@uidotdev/usehooks/-/usehooks-2.4.1.tgz",
"integrity": "sha512-1I+RwWyS+kdv3Mv0Vmc+p0dPYH0DTRAo04HLyXReYBL9AeseDWUJyi4THuksBJcu9F0Pih69Ak150VDnqbVnXg==",
"engines": {
"node": ">=16"
},
"peerDependencies": {
"react": ">=18.0.0",
"react-dom": ">=18.0.0"
}
},
"node_modules/@uiw/copy-to-clipboard": {
"version": "1.0.16",
"resolved": "https://registry.npmjs.org/@uiw/copy-to-clipboard/-/copy-to-clipboard-1.0.16.tgz",
@ -7989,6 +8034,32 @@
"resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.2.0.tgz",
"integrity": "sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ=="
},
"node_modules/@virtual-grid/core": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/@virtual-grid/core/-/core-2.0.1.tgz",
"integrity": "sha512-ZqhtuVGMVM89W/8PFljGqaWV0f8Jce+x68y4hWXpgCHlhSwpGrAKE5awgG1qHDsbTcM0kv1cL88KOFWknbnyWQ=="
},
"node_modules/@virtual-grid/react": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/@virtual-grid/react/-/react-2.0.2.tgz",
"integrity": "sha512-cwlMlTdJTX+BhlE1PhgLQgWzHkYCf722ZCwp/56caMg2sGjXloaQDWU+jUk6g2TFQ/mlteC1w7aYkPZC+ZwxWg==",
"dependencies": {
"@tanstack/react-virtual": "^3.0.1",
"@virtual-grid/core": "^2.0.1",
"@virtual-grid/shared": "^2.0.1",
"react-intersection-observer": "^9.5.2",
"use-deep-compare": "^1.2.1",
"use-resize-observer": "^9.1.0"
},
"peerDependencies": {
"react": "^17.0.2 || ^18.0.0"
}
},
"node_modules/@virtual-grid/shared": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/@virtual-grid/shared/-/shared-2.0.1.tgz",
"integrity": "sha512-E0krspmtVOGRg/qAgDKUjhTRV7VXmFp7Q05ljT87Llffh8g5JoXVsAUPV7JiRKnrSRFShpiVdCy9eq0VvVeifA=="
},
"node_modules/@vitejs/plugin-react": {
"version": "4.2.1",
"resolved": "https://registry.npmjs.org/@vitejs/plugin-react/-/plugin-react-4.2.1.tgz",
@ -20387,6 +20458,20 @@
"react": ">=16.0.0"
}
},
"node_modules/react-intersection-observer": {
"version": "9.8.1",
"resolved": "https://registry.npmjs.org/react-intersection-observer/-/react-intersection-observer-9.8.1.tgz",
"integrity": "sha512-QzOFdROX8D8MH3wE3OVKH0f3mLjKTtEN1VX/rkNuECCff+aKky0pIjulDhr3Ewqj5el/L+MhBkM3ef0Tbt+qUQ==",
"peerDependencies": {
"react": "^15.0.0 || ^16.0.0 || ^17.0.0 || ^18.0.0",
"react-dom": "^15.0.0 || ^16.0.0 || ^17.0.0 || ^18.0.0"
},
"peerDependenciesMeta": {
"react-dom": {
"optional": true
}
}
},
"node_modules/react-is": {
"version": "18.2.0",
"resolved": "https://registry.npmjs.org/react-is/-/react-is-18.2.0.tgz",
@ -23843,6 +23928,17 @@
"react": ">=16.8.0"
}
},
"node_modules/use-deep-compare": {
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/use-deep-compare/-/use-deep-compare-1.2.1.tgz",
"integrity": "sha512-JTnOZAr0fq1ix6CQ4XANoWIh03xAiMFlP/lVAYDdAOZwur6nqBSdATn1/Q9PLIGIW+C7xmFZBCcaA4KLDcQJtg==",
"dependencies": {
"dequal": "2.0.3"
},
"peerDependencies": {
"react": ">=16.8.0"
}
},
"node_modules/use-isomorphic-layout-effect": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/use-isomorphic-layout-effect/-/use-isomorphic-layout-effect-1.1.2.tgz",
@ -23872,6 +23968,18 @@
}
}
},
"node_modules/use-resize-observer": {
"version": "9.1.0",
"resolved": "https://registry.npmjs.org/use-resize-observer/-/use-resize-observer-9.1.0.tgz",
"integrity": "sha512-R25VqO9Wb3asSD4eqtcxk8sJalvIOYBqS8MNZlpDSQ4l4xMQxC/J7Id9HoTqPq8FwULIn0PVW+OAqF2dyYbjow==",
"dependencies": {
"@juggle/resize-observer": "^3.3.1"
},
"peerDependencies": {
"react": "16.8.0 - 18",
"react-dom": "16.8.0 - 18"
}
},
"node_modules/use-sync-external-store": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/use-sync-external-store/-/use-sync-external-store-1.2.0.tgz",

3
package.json
View File

@ -27,6 +27,7 @@
"@prisma/client": "^5.8.1",
"@swc/helpers": "~0.5.2",
"@sweetalert2/theme-dark": "^5.0.16",
"@tanstack/react-virtual": "^3.1.3",
"@types/bcrypt": "^5.0.2",
"@types/jsonwebtoken": "^9.0.5",
"@types/lodash": "^4.14.202",
@ -34,7 +35,9 @@
"@types/mime-types": "^2.1.4",
"@types/remove-markdown": "^0.3.4",
"@types/stripe": "^8.0.417",
"@uidotdev/usehooks": "^2.4.1",
"@uiw/react-md-editor": "^4.0.3",
"@virtual-grid/react": "^2.0.2",
"axios": "^1.0.0",
"bcrypt": "^5.1.1",
"bullmq": "^5.1.5",