jeffemmett
/
postiz
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: stars and forks

This commit is contained in:
Nevo David 2024-03-11 00:38:52 +07:00
parent d53bd606aa
commit 21a1d2a517
9 changed files with 488 additions and 307 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

128
apps/frontend/src/components/analytics/chart.tsx
View File

@ -1,64 +1,68 @@
"use client";
import {FC, useEffect, useMemo, useRef} from "react";
'use client';
import { FC, useEffect, useMemo, useRef } from 'react';
import DrawChart from 'chart.js/auto';
import {StarsList} from "@gitroom/frontend/components/analytics/stars.and.forks.interface";
import dayjs from "dayjs";
import {
ForksList,
StarsList,
} from '@gitroom/frontend/components/analytics/stars.and.forks.interface';
import dayjs from 'dayjs';
export const Chart: FC<{list: StarsList[]}> = (props) => {
const {list} = props;
const ref = useRef<any>(null);
const chart = useRef<null | DrawChart>(null);
useEffect(() => {
const gradient = ref.current.getContext('2d').createLinearGradient(0, 0, 0, ref.current.height);
gradient.addColorStop(0, 'rgba(114, 118, 137, 1)'); // Start color with some transparency
gradient.addColorStop(1, 'rgb(9, 11, 19, 1)');
chart.current = new DrawChart(
ref.current!,
{
type: 'line',
options: {
maintainAspectRatio: false,
responsive: true,
layout: {
padding: {
left: 0,
right: 0,
top: 0,
bottom: 0
}
},
scales: {
y: {
beginAtZero: true,
display: false
},
x: {
display: false
}
},
plugins: {
legend: {
display: false
},
}
},
data: {
labels: list.map(row => dayjs(row.date).format('DD/MM/YYYY')),
datasets: [
{
borderColor: '#fff',
label: 'Stars by date',
backgroundColor: gradient,
fill: true,
data: list.map(row => row.totalStars)
}
]
}
}
);
return () => {
chart?.current?.destroy();
}
}, []);
return <canvas className="w-full h-full" ref={ref} />
}
export const Chart: FC<{ list: StarsList[] | ForksList[] }> = (props) => {
const { list } = props;
const ref = useRef<any>(null);
const chart = useRef<null | DrawChart>(null);
useEffect(() => {
const gradient = ref.current
.getContext('2d')
.createLinearGradient(0, 0, 0, ref.current.height);
gradient.addColorStop(0, 'rgba(114, 118, 137, 1)'); // Start color with some transparency
gradient.addColorStop(1, 'rgb(9, 11, 19, 1)');
chart.current = new DrawChart(ref.current!, {
type: 'line',
options: {
maintainAspectRatio: false,
responsive: true,
layout: {
padding: {
left: 0,
right: 0,
top: 0,
bottom: 0,
},
},
scales: {
y: {
beginAtZero: true,
display: false,
},
x: {
display: false,
},
},
plugins: {
legend: {
display: false,
},
},
},
data: {
labels: list.map((row) => dayjs(row.date).format('DD/MM/YYYY')),
datasets: [
{
borderColor: '#fff',
// @ts-ignore
label: list?.[0]?.totalForks ? 'Forks by date' : 'Stars by date',
backgroundColor: gradient,
fill: true,
// @ts-ignore
data: list.map((row) => row.totalForks || row.totalStars),
},
],
},
});
return () => {
chart?.current?.destroy();
};
}, []);
return <canvas className="w-full h-full" ref={ref} />;
};

40
apps/frontend/src/components/analytics/stars.and.forks.interface.ts
View File

@ -1,22 +1,28 @@
export interface StarsList {
totalStars: number;
date: string;
totalStars: number;
date: string;
}
export interface ForksList {
totalForks: number;
date: string;
}
export interface Stars {
id: string,
stars: number,
totalStars: number,
login: string,
date: string,
id: string;
stars: number;
totalStars: number;
login: string;
date: string;
}
export interface StarsAndForksInterface {
list: Array<{
login: string;
stars: StarsList[]
}>;
trending: {
last: string;
predictions: string;
};
}
list: Array<{
login: string;
stars: StarsList[];
forks: ForksList[];
}>;
trending: {
last: string;
predictions: string;
};
}

6
apps/frontend/src/components/analytics/stars.and.forks.tsx
View File

@ -82,8 +82,8 @@ export const StarsAndForks: FC<StarsAndForksInterface> = (props) => {
</div>
<div className="flex-1 relative">
<div className="absolute w-full h-full left-0 top-0">
{item.stars.length ? (
<Chart list={item.stars} />
{item.forks.length ? (
<Chart list={item.forks} />
) : (
<div className="w-full h-full flex items-center justify-center text-3xl">
Processing stars...
@ -92,7 +92,7 @@ export const StarsAndForks: FC<StarsAndForksInterface> = (props) => {
</div>
</div>
<div className="text-[50px] leading-[60px]">
{item?.stars[item.stars.length - 1]?.totalStars}
{item?.forks[item.forks.length - 1]?.totalForks}
</div>
</div>
</div>

18
apps/frontend/src/components/analytics/stars.table.component.tsx
View File

@ -1,5 +1,10 @@
import {
FC, useCallback, useEffect, useMemo, useState, useTransition,
FC,
useCallback,
useEffect,
useMemo,
useState,
useTransition,
} from 'react';
import { UtcToLocalDateRender } from '@gitroom/react/helpers/utc.date.render';
import { Button } from '@gitroom/react/form/button';
@ -211,11 +216,17 @@ export const StarsTableComponent = () => {
<UpDown name="Date" param="date" />
</th>
<th>
<UpDown name="Total" param="totalStars" />
<UpDown name="Total Stars" param="totalStars" />
</th>
<th>
<UpDown name="Total Fork" param="totalForks" />
</th>
<th>
<UpDown name="Stars" param="stars" />
</th>
<th>
<UpDown name="Forks" param="forks" />
</th>
<th>Media</th>
</tr>
</thead>
@ -227,7 +238,10 @@ export const StarsTableComponent = () => {
<UtcToLocalDateRender date={p.date} format="DD/MM/YYYY" />
</td>
<td>{p.totalStars}</td>
<td>{p.totalForks}</td>
<td>{p.stars}</td>
<td>{p.forks}</td>
<td>
<Link href={renderMediaLink(p.date)}>
<Button>Check Launch</Button>

67
apps/workers/src/app/stars.controller.ts
View File

@ -1,50 +1,73 @@
import {Controller} from '@nestjs/common';
import {EventPattern, Transport} from '@nestjs/microservices';
import { JSDOM } from "jsdom";
import {StarsService} from "@gitroom/nestjs-libraries/database/prisma/stars/stars.service";
import {TrendingService} from "@gitroom/nestjs-libraries/services/trending.service";
import { Controller } from '@nestjs/common';
import { EventPattern, Transport } from '@nestjs/microservices';
import { JSDOM } from 'jsdom';
import { StarsService } from '@gitroom/nestjs-libraries/database/prisma/stars/stars.service';
import { TrendingService } from '@gitroom/nestjs-libraries/services/trending.service';
@Controller()
export class StarsController {
constructor(
private _starsService: StarsService,
private _trendingService: TrendingService
) {
}
private _starsService: StarsService,
private _trendingService: TrendingService
) {}
@EventPattern('check_stars', Transport.REDIS)
async checkStars(data: {login: string}) {
async checkStars(data: { login: string }) {
// no to be effected by the limit, we scrape the HTML instead of using the API
const loadedHtml = await (await fetch(`https://github.com/${data.login}`)).text();
const loadedHtml = await (
await fetch(`https://github.com/${data.login}`)
).text();
const dom = new JSDOM(loadedHtml);
const totalStars = +(
dom.window.document.querySelector('#repo-stars-counter-star')?.getAttribute('title')?.replace(/,/g, '')
) || 0;
const lastStarsValue = await this._starsService.getLastStarsByLogin(data.login);
const totalNewsStars = totalStars - (lastStarsValue?.totalStars || 0);
const totalStars =
+dom.window.document
.querySelector('#repo-stars-counter-star')
?.getAttribute('title')
?.replace(/,/g, '') || 0;
const totalForks = +dom.window.document
.querySelector('#repo-network-counter')
?.getAttribute('title')
?.replace(/,/g, '');
const lastValue = await this._starsService.getLastStarsByLogin(
data.login
);
const totalNewsStars = totalStars - (lastValue?.totalStars || 0);
const totalNewsForks = totalForks - (lastValue?.totalForks || 0);
// if there is no stars in the database, we need to sync the stars
if (!lastStarsValue?.totalStars) {
if (!lastValue?.totalStars) {
return;
}
// if there is stars in the database, sync the new stars
if (totalNewsStars > 0) {
return this._starsService.createStars(data.login, totalNewsStars, totalStars, new Date());
return this._starsService.createStars(
data.login,
totalNewsStars,
totalStars,
totalNewsForks,
totalForks,
new Date()
);
}
}
@EventPattern('sync_all_stars', Transport.REDIS, {concurrency: 1})
async syncAllStars(data: {login: string}) {
@EventPattern('sync_all_stars', Transport.REDIS, { concurrency: 1 })
async syncAllStars(data: { login: string }) {
// if there is a sync in progress, it's better not to touch it
if ((await this._starsService.getStarsByLogin(data.login)).length) {
return;
}
await this._starsService.sync(data.login);
const findValidToken = await this._starsService.findValidToken(data.login);
console.log(findValidToken?.token);
await this._starsService.sync(data.login, findValidToken?.token);
}
@EventPattern('sync_trending', Transport.REDIS, {concurrency: 1})
@EventPattern('sync_trending', Transport.REDIS, { concurrency: 1 })
async syncTrending() {
return this._trendingService.syncTrending();
}

2
libraries/nestjs-libraries/src/database/prisma/schema.prisma
View File

@ -98,6 +98,8 @@ model Star {
id String @id @default(uuid())
stars Int
totalStars Int
forks Int
totalForks Int
login String
date DateTime @default(now()) @db.Date
createdAt DateTime @default(now())

382
libraries/nestjs-libraries/src/database/prisma/stars/stars.repository.ts
View File

@ -1,196 +1,220 @@
import {PrismaRepository} from "@gitroom/nestjs-libraries/database/prisma/prisma.service";
import {Injectable} from "@nestjs/common";
import {StarsListDto} from "@gitroom/nestjs-libraries/dtos/analytics/stars.list.dto";
import { PrismaRepository } from '@gitroom/nestjs-libraries/database/prisma/prisma.service';
import { Injectable } from '@nestjs/common';
import { StarsListDto } from '@gitroom/nestjs-libraries/dtos/analytics/stars.list.dto';
@Injectable()
export class StarsRepository {
constructor(
private _github: PrismaRepository<'gitHub'>,
private _stars: PrismaRepository<'star'>,
private _trending: PrismaRepository<'trending'>,
private _trendingLog: PrismaRepository<'trendingLog'>,
) {
}
getGitHubRepositoriesByOrgId(org: string) {
return this._github.model.gitHub.findMany({
where: {
organizationId: org
}
});
}
replaceOrAddTrending(language: string, hashedNames: string, arr: { name: string; position: number }[]) {
return this._trending.model.trending.upsert({
create: {
language,
hash: hashedNames,
trendingList: JSON.stringify(arr),
date: new Date()
},
update: {
language,
hash: hashedNames,
trendingList: JSON.stringify(arr),
date: new Date()
},
where: {
language
}
});
}
constructor(
private _github: PrismaRepository<'gitHub'>,
private _stars: PrismaRepository<'star'>,
private _trending: PrismaRepository<'trending'>,
private _trendingLog: PrismaRepository<'trendingLog'>
) {}
getGitHubRepositoriesByOrgId(org: string) {
return this._github.model.gitHub.findMany({
where: {
organizationId: org,
},
});
}
replaceOrAddTrending(
language: string,
hashedNames: string,
arr: { name: string; position: number }[]
) {
return this._trending.model.trending.upsert({
create: {
language,
hash: hashedNames,
trendingList: JSON.stringify(arr),
date: new Date(),
},
update: {
language,
hash: hashedNames,
trendingList: JSON.stringify(arr),
date: new Date(),
},
where: {
language,
},
});
}
newTrending(language: string) {
return this._trendingLog.model.trendingLog.create({
data: {
date: new Date(),
language
}
});
}
newTrending(language: string) {
return this._trendingLog.model.trendingLog.create({
data: {
date: new Date(),
language,
},
});
}
getAllGitHubRepositories() {
return this._github.model.gitHub.findMany({
distinct: ['login'],
});
}
getAllGitHubRepositories() {
return this._github.model.gitHub.findMany({
distinct: ['login'],
});
}
async getLastStarsByLogin(login: string) {
return (await this._stars.model.star.findMany({
where: {
login,
},
orderBy: {
date: 'desc',
},
take: 1,
}))?.[0];
}
async getLastStarsByLogin(login: string) {
return (
await this._stars.model.star.findMany({
where: {
login,
},
orderBy: {
date: 'desc',
},
take: 1,
})
)?.[0];
}
async getStarsByLogin(login: string) {
return (await this._stars.model.star.findMany({
where: {
login,
},
orderBy: {
date: 'asc',
}
}));
}
async getStarsByLogin(login: string) {
return await this._stars.model.star.findMany({
where: {
login,
},
orderBy: {
date: 'asc',
},
});
}
async getGitHubsByNames(names: string[]) {
return this._github.model.gitHub.findMany({
where: {
login: {
in: names
}
}
});
}
async getGitHubsByNames(names: string[]) {
return this._github.model.gitHub.findMany({
where: {
login: {
in: names,
},
},
});
}
createStars(login: string, totalNewsStars: number, totalStars: number, date: Date) {
return this._stars.model.star.upsert({
create: {
login,
stars: totalNewsStars,
totalStars,
date
},
update: {
stars: totalNewsStars,
totalStars,
},
where: {
login_date: {
date,
login
}
}
});
}
findValidToken(login: string) {
return this._github.model.gitHub.findFirst({
where: {
login,
},
});
}
getTrendingByLanguage(language: string) {
return this._trending.model.trending.findUnique({
where: {
language
}
});
}
createStars(
login: string,
totalNewsStars: number,
totalStars: number,
totalNewForks: number,
totalForks: number,
date: Date
) {
return this._stars.model.star.upsert({
create: {
login,
stars: totalNewsStars,
forks: totalNewForks,
totalForks,
totalStars,
date,
},
update: {
stars: totalNewsStars,
totalStars,
forks: totalNewForks,
totalForks,
},
where: {
login_date: {
date,
login,
},
},
});
}
getLastTrending(language: string) {
return this._trendingLog.model.trendingLog.findMany({
where: {
language
},
orderBy: {
date: 'desc'
},
take: 100
});
}
getTrendingByLanguage(language: string) {
return this._trending.model.trending.findUnique({
where: {
language,
},
});
}
getStarsFilter(githubs: string[], starsFilter: StarsListDto) {
return this._stars.model.star.findMany({
orderBy: {
[starsFilter.key || 'date']: starsFilter.state || 'desc'
},
where: {
login: {
in: githubs.filter(f => f)
}
},
take: 20,
skip: (starsFilter.page - 1) * 10
});
}
getLastTrending(language: string) {
return this._trendingLog.model.trendingLog.findMany({
where: {
language,
},
orderBy: {
date: 'desc',
},
take: 100,
});
}
addGitHub(orgId: string, accessToken: string) {
return this._github.model.gitHub.create({
data: {
token: accessToken,
organizationId: orgId,
jobId: ''
}
});
}
getStarsFilter(githubs: string[], starsFilter: StarsListDto) {
return this._stars.model.star.findMany({
orderBy: {
[starsFilter.key || 'date']: starsFilter.state || 'desc',
},
where: {
login: {
in: githubs.filter((f) => f),
},
},
take: 20,
skip: (starsFilter.page - 1) * 10,
});
}
getGitHubById(orgId: string, id: string) {
return this._github.model.gitHub.findUnique({
where: {
organizationId: orgId,
id
}
});
}
addGitHub(orgId: string, accessToken: string) {
return this._github.model.gitHub.create({
data: {
token: accessToken,
organizationId: orgId,
jobId: '',
},
});
}
updateGitHubLogin(orgId: string, id: string, login: string) {
return this._github.model.gitHub.update({
where: {
organizationId: orgId,
id
},
data: {
login
}
});
}
getGitHubById(orgId: string, id: string) {
return this._github.model.gitHub.findUnique({
where: {
organizationId: orgId,
id,
},
});
}
deleteRepository(orgId: string, id: string) {
return this._github.model.gitHub.delete({
where: {
organizationId: orgId,
id
}
});
}
updateGitHubLogin(orgId: string, id: string, login: string) {
return this._github.model.gitHub.update({
where: {
organizationId: orgId,
id,
},
data: {
login,
},
});
}
getOrganizationsByGitHubLogin(login: string) {
return this._github.model.gitHub.findMany({
select: {
organizationId: true
},
where: {
login
},
distinct: ['organizationId']
});
}
}
deleteRepository(orgId: string, id: string) {
return this._github.model.gitHub.delete({
where: {
organizationId: orgId,
id,
},
});
}
getOrganizationsByGitHubLogin(login: string) {
return this._github.model.gitHub.findMany({
select: {
organizationId: true,
},
where: {
login,
},
distinct: ['organizationId'],
});
}
}

150
libraries/nestjs-libraries/src/database/prisma/stars/stars.service.ts
View File

@ -39,54 +39,86 @@ export class StarsService {
login: string,
totalNewsStars: number,
totalStars: number,
totalNewForks: number,
totalForks: number,
date: Date
) {
return this._starsRepository.createStars(
login,
totalNewsStars,
totalStars,
totalNewForks,
totalForks,
date
);
}
async sync(login: string) {
const loadAllStars = await this.syncProcess(login);
const sortedArray = Object.keys(loadAllStars).sort(
async sync(login: string, token?: string) {
const loadAllStars = await this.syncProcess(login, token);
const loadAllForks = await this.syncForksProcess(login, token);
const allDates = [
...new Set([...Object.keys(loadAllStars), ...Object.keys(loadAllForks)]),
];
console.log(allDates);
const sortedArray = allDates.sort(
(a, b) => dayjs(a).unix() - dayjs(b).unix()
);
let addPreviousStars = 0;
let addPreviousForks = 0;
for (const date of sortedArray) {
const dateObject = dayjs(date).toDate();
addPreviousStars += loadAllStars[date];
addPreviousStars += loadAllStars[date] || 0;
addPreviousForks += loadAllForks[date] || 0;
await this._starsRepository.createStars(
login,
loadAllStars[date],
loadAllStars[date] || 0,
addPreviousStars,
loadAllForks[date] || 0,
addPreviousForks,
dateObject
);
}
}
async syncProcess(login: string, page = 1) {
const starsRequest = await fetch(
`https://api.github.com/repos/${login}/stargazers?page=${page}&per_page=100`,
{
async findValidToken(login: string) {
return this._starsRepository.findValidToken(login);
}
async fetchWillFallback(url: string, userToken?: string): Promise<Response> {
if (userToken) {
const response = await fetch(url, {
headers: {
Accept: 'application/vnd.github.v3.star+json',
...(process.env.GITHUB_AUTH
? { Authorization: `token ${process.env.GITHUB_AUTH}` }
: {}),
Authorization: `Bearer ${userToken}`,
},
});
if (response.status === 200) {
return response;
}
);
}
const response2 = await fetch(url, {
headers: {
Accept: 'application/vnd.github.v3.star+json',
...(process.env.GITHUB_AUTH
? { Authorization: `token ${process.env.GITHUB_AUTH}` }
: {}),
},
});
const totalRemaining = +(
starsRequest.headers.get('x-ratelimit-remaining') ||
starsRequest.headers.get('X-RateLimit-Remaining') ||
response2.headers.get('x-ratelimit-remaining') ||
response2.headers.get('X-RateLimit-Remaining') ||
0
);
const resetTime = +(
starsRequest.headers.get('x-ratelimit-reset') ||
starsRequest.headers.get('X-RateLimit-Reset') ||
response2.headers.get('x-ratelimit-reset') ||
response2.headers.get('X-RateLimit-Reset') ||
0
);
@ -94,8 +126,65 @@ export class StarsService {
console.log('waiting for the rate limit');
const delay = resetTime * 1000 - Date.now() + 1000;
await new Promise((resolve) => setTimeout(resolve, delay));
return this.fetchWillFallback(url, userToken);
}
return response2;
}
async syncForksProcess(login: string, userToken?: string, page = 1) {
console.log('processing forks');
const starsRequest = await this.fetchWillFallback(
`https://api.github.com/repos/${login}/forks?page=${page}&per_page=100`,
userToken
);
const data: Array<{ created_at: string }> = await starsRequest.json();
const mapDataToDate = groupBy(data, (p) =>
dayjs(p.created_at).format('YYYY-MM-DD')
);
// take all the forks from the page
const aggForks: { [key: string]: number } = Object.values(
mapDataToDate
).reduce(
(acc, value) => ({
...acc,
[dayjs(value[0].created_at).format('YYYY-MM-DD')]: value.length,
}),
{}
);
// if we have 100 stars, we need to fetch the next page and merge the results (recursively)
const nextOne: { [key: string]: number } =
data.length === 100
? await this.syncForksProcess(login, userToken, page + 1)
: {};
// merge the results
const allKeys = [
...new Set([...Object.keys(aggForks), ...Object.keys(nextOne)]),
];
return {
...allKeys.reduce(
(acc, key) => ({
...acc,
[key]: (aggForks[key] || 0) + (nextOne[key] || 0),
}),
{} as { [key: string]: number }
),
};
}
async syncProcess(login: string, userToken?: string, page = 1) {
console.log('processing stars');
const starsRequest = await this.fetchWillFallback(
`https://api.github.com/repos/${login}/stargazers?page=${page}&per_page=100`,
userToken
);
const data: Array<{ starred_at: string }> = await starsRequest.json();
const mapDataToDate = groupBy(data, (p) =>
dayjs(p.starred_at).format('YYYY-MM-DD')
@ -107,14 +196,16 @@ export class StarsService {
).reduce(
(acc, value) => ({
...acc,
[value[0].starred_at]: value.length,
[dayjs(value[0].starred_at).format('YYYY-MM-DD')]: value.length,
}),
{}
);
// if we have 100 stars, we need to fetch the next page and merge the results (recursively)
const nextOne: { [key: string]: number } =
data.length === 100 ? await this.syncProcess(login, page + 1) : {};
data.length === 100
? await this.syncProcess(login, userToken, page + 1)
: {};
// merge the results
const allKeys = [
@ -168,7 +259,9 @@ export class StarsService {
}
const informNewPeople = arr.filter(
(p) => !currentTrending?.trendingList || currentTrending?.trendingList?.indexOf(p.name) === -1
(p) =>
!currentTrending?.trendingList ||
currentTrending?.trendingList?.indexOf(p.name) === -1
);
// let people know they are trending
@ -241,9 +334,15 @@ export class StarsService {
if (!gitHub.login) {
continue;
}
const stars = await this.getStarsByLogin(gitHub.login!);
const getAllByLogin = await this.getStarsByLogin(gitHub.login!);
const stars = getAllByLogin.filter((f) => f.stars);
const graphSize = stars.length < 10 ? stars.length : stars.length / 10;
const forks = getAllByLogin.filter((f) => f.forks);
const graphForkSize =
forks.length < 10 ? forks.length : forks.length / 10;
list.push({
login: gitHub.login,
stars: chunk(stars, graphSize).reduce((acc, chunkedStars) => {
@ -255,6 +354,15 @@ export class StarsService {
},
];
}, [] as Array<{ totalStars: number; date: Date }>),
forks: chunk(forks, graphForkSize).reduce((acc, chunkedForks) => {
return [
...acc,
{
totalForks: chunkedForks[chunkedForks.length - 1].totalForks,
date: chunkedForks[chunkedForks.length - 1].date,
},
];
}, [] as Array<{ totalForks: number; date: Date }>),
});
}

2
libraries/nestjs-libraries/src/dtos/analytics/stars.list.dto.ts
View File

@ -6,7 +6,7 @@ export class StarsListDto {
page: number;
@IsOptional()
@IsIn(['login', 'totalStars', 'stars', 'date'])
@IsIn(['login', 'totalStars', 'stars', 'date', 'forks', 'totalForks'])
key: 'login' | 'date' | 'stars' | 'totalStars';
@IsOptional()