From f4222a1dd4de58f62f95d4966b4b45b90e947086 Mon Sep 17 00:00:00 2001 From: Alex Selimov Date: Thu, 14 May 2026 22:27:32 -0400 Subject: [PATCH 1/2] Output flag option and no error on missing var - Add -o option to dictate where to output file to. - Update so that missing environment variables just skips that specific hub --- get_history/index.ts | 124 ++++++++++++++++++++++++++++++------------- 1 file changed, 87 insertions(+), 37 deletions(-) diff --git a/get_history/index.ts b/get_history/index.ts index fe2a0b3..cb335ff 100644 --- a/get_history/index.ts +++ b/get_history/index.ts @@ -1,19 +1,70 @@ -function requireEnv(key: string): string { - const value = process.env[key]; - if (!value) throw new Error(`Missing required env var: ${key}`); - return value; +import { mkdir } from "node:fs/promises"; +import { dirname } from "node:path"; + +function parseArgs(args: string[]): { outputPath: string } { + let outputPath = "../static/activity.json"; + + for (let i = 0; i < args.length; i++) { + const arg = args[i]; + + if (arg === "--help" || arg === "-h") { + console.log( + "Usage: bun run index.ts [--output ]\n\nOptions:\n -o, --output Where to write activity.json (default: ../static/activity.json)", + ); + process.exit(0); + } + + if (arg === "--output" || arg === "-o") { + const value = args[i + 1]; + if (!value) throw new Error(`${arg} requires a path`); + outputPath = value; + i++; + continue; + } + + if (arg?.startsWith("--output=")) { + outputPath = arg.slice("--output=".length); + continue; + } + + throw new Error(`Unknown argument: ${arg}`); + } + + return { outputPath }; } +const cli = parseArgs(Bun.argv.slice(2)); + const config = { - githubApiKey: requireEnv("GH_API_KEY"), - forgejoApiKey: requireEnv("FJ_API_KEY"), - forgejoUsername: requireEnv("FJ_USER"), - forgejoURL: requireEnv("FJ_URL"), - gitlabApiKey: process.env["GL_API_KEY"], - gitlabUsername: process.env["GL_USER"], + githubApiKey: process.env["GH_API_KEY"] ?? "", + forgejoApiKey: process.env["FJ_API_KEY"] ?? "", + forgejoUsername: process.env["FJ_USER"] ?? "", + forgejoURL: process.env["FJ_URL"] ?? "https://codeberg.org", + gitlabApiKey: process.env["GL_API_KEY"] ?? "", + gitlabUsername: process.env["GL_USER"] ?? "", gitlabURL: process.env["GL_URL"] ?? "https://gitlab.com", }; +function areGhVarsPresent() { + return ( + config.githubApiKey?.trim().length > 0 && + config.githubApiKey?.trim().length > 0 + ); +} + +function areFjVarsPresent() { + return ( + config.forgejoApiKey?.trim().length > 0 && + config.forgejoUsername?.trim().length > 0 + ); +} + +function areGlVarsPresent() { + return ( + config.gitlabApiKey?.trim().length > 0 && + config.gitlabUsername?.trim().length > 0 + ); +} const now = new Date(); const oneYearAgo = new Date(); oneYearAgo.setFullYear(oneYearAgo.getFullYear() - 1); @@ -142,7 +193,8 @@ function parseGitHubContributions(data: any): ActivityMap { const calendarMap: { [day: string]: number } = {}; for (const week of collection?.contributionCalendar?.weeks ?? []) { for (const day of week.contributionDays) { - if (day.contributionCount > 0) calendarMap[day.date] = day.contributionCount; + if (day.contributionCount > 0) + calendarMap[day.date] = day.contributionCount; } } @@ -221,7 +273,7 @@ async function getForgejoActivityMap(): Promise { ); } - const activities: any[] = await response.json(); + const activities = (await response.json()) as any[]; if (activities.length === 0) break; const pageMap = parseForgejoActivity(activities); @@ -312,7 +364,6 @@ function parseGitLabEvents(events: any[]): ActivityMap { async function getGitLabActivityMap(): Promise { if (!config.gitlabApiKey || !config.gitlabUsername) { - console.log("[GitLab] Skipping: GL_API_KEY or GL_USER not set"); return {}; } @@ -321,35 +372,23 @@ async function getGitLabActivityMap(): Promise { const map: ActivityMap = {}; const afterDate = oneYearAgo.toISOString().split("T")[0]!; - console.log(`[GitLab] Fetching events for user "${config.gitlabUsername}" from ${config.gitlabURL} after ${afterDate}`); - while (true) { const url = `${config.gitlabURL}/api/v4/users/${config.gitlabUsername}/events?action=pushed&after=${afterDate}&per_page=${pageLimit}&page=${page}`; - console.log(`[GitLab] GET ${url}`); const response = await fetch(url, { headers: { "PRIVATE-TOKEN": config.gitlabApiKey }, }); - console.log(`[GitLab] Response: ${response.status} ${response.statusText}`); - if (!response.ok) { - const body = await response.text(); - console.error(`[GitLab] Error body: ${body}`); - throw new Error(`GitLab API error: ${response.status} ${response.statusText}`); - } - - const events: any[] = await response.json(); - console.log(`[GitLab] Page ${page}: received ${events.length} events`); - - if (events.length > 0) { - console.log(`[GitLab] First event sample:`, JSON.stringify(events[0], null, 2)); + throw new Error( + `GitLab API error: ${response.status} ${response.statusText}`, + ); } + const events = (await response.json()) as any[]; if (events.length === 0) break; const parsed = parseGitLabEvents(events); - console.log(`[GitLab] Page ${page}: parsed ${Object.keys(parsed).length} days of activity`); mergeInto(map, parsed); if (events.length < pageLimit) break; @@ -360,10 +399,23 @@ async function getGitLabActivityMap(): Promise { } async function makeActivityMap(): Promise { + const getActivity = async ( + varsPresentCheck: () => boolean, + getActivityMap: () => Promise, + hubName: string, + ) => { + if (varsPresentCheck()) { + console.log(`Fetching ${hubName} activity`); + return getActivityMap(); + } else { + console.log(`${hubName} not configured, skipping...`); + return {}; + } + }; const [ghMap, fjMap, glMap] = await Promise.all([ - getGitHubActivityMap(), - getForgejoActivityMap(), - getGitLabActivityMap(), + getActivity(areGhVarsPresent, getGitHubActivityMap, "Github"), + getActivity(areFjVarsPresent, getForgejoActivityMap, "Forgejo"), + getActivity(areGlVarsPresent, getGitLabActivityMap, "Gitlab"), ]); const merged = deduplicateAcrossPlatforms(ghMap, fjMap); @@ -371,10 +423,8 @@ async function makeActivityMap(): Promise { } const activityMap = await makeActivityMap(); -await Bun.write( - "../static/activity.json", - JSON.stringify(activityMap, null, 2), -); +await mkdir(dirname(cli.outputPath), { recursive: true }); +await Bun.write(cli.outputPath, JSON.stringify(activityMap, null, 2)); console.log( - `Wrote ${Object.keys(activityMap).length} days of activity to out/activity.json`, + `Wrote ${Object.keys(activityMap).length} days of activity to ${cli.outputPath}`, ); From e1941ac77d8c0f567d64557f9df71ad365ef7556 Mon Sep 17 00:00:00 2001 From: Alex Selimov Date: Thu, 14 May 2026 22:55:17 -0400 Subject: [PATCH 2/2] Port script to go --- get_history/.env.example | 1 + get_history/.gitignore | 21 +- get_history/bun.lock | 26 -- get_history/go.mod | 3 + get_history/index.ts | 430 --------------------- get_history/main.go | 773 ++++++++++++++++++++++++++++++++++++++ get_history/package.json | 12 - get_history/tsconfig.json | 30 -- 8 files changed, 778 insertions(+), 518 deletions(-) delete mode 100644 get_history/bun.lock create mode 100644 get_history/go.mod delete mode 100644 get_history/index.ts create mode 100644 get_history/main.go delete mode 100644 get_history/package.json delete mode 100644 get_history/tsconfig.json diff --git a/get_history/.env.example b/get_history/.env.example index eb06bba..d92e706 100644 --- a/get_history/.env.example +++ b/get_history/.env.example @@ -4,3 +4,4 @@ FJ_USER= FJ_URL= GL_API_KEY= GL_USER= +GL_URL= diff --git a/get_history/.gitignore b/get_history/.gitignore index a14702c..743a741 100644 --- a/get_history/.gitignore +++ b/get_history/.gitignore @@ -1,32 +1,13 @@ -# dependencies (bun install) -node_modules - # output out dist +get_history *.tgz -# code coverage -coverage -*.lcov - -# logs -logs -_.log -report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json - # dotenv environment variable files .env -.env.development.local -.env.test.local -.env.production.local .env.local -# caches -.eslintcache -.cache -*.tsbuildinfo - # IntelliJ based IDEs .idea diff --git a/get_history/bun.lock b/get_history/bun.lock deleted file mode 100644 index 978a13a..0000000 --- a/get_history/bun.lock +++ /dev/null @@ -1,26 +0,0 @@ -{ - "lockfileVersion": 1, - "configVersion": 1, - "workspaces": { - "": { - "name": "get_history", - "devDependencies": { - "@types/bun": "latest", - }, - "peerDependencies": { - "typescript": "^5", - }, - }, - }, - "packages": { - "@types/bun": ["@types/bun@1.3.13", "", { "dependencies": { "bun-types": "1.3.13" } }, "sha512-9fqXWk5YIHGGnUau9TEi+qdlTYDAnOj+xLCmSTwXfAIqXr2x4tytJb43E9uCvt09zJURKXwAtkoH4nLQfzeTXw=="], - - "@types/node": ["@types/node@25.6.2", "", { "dependencies": { "undici-types": "~7.19.0" } }, "sha512-sokuT28dxf9JT5Kady1fsXOvI4HVpjZa95NKT5y9PNTIrs2AsobR4GFAA90ZG8M+nxVRLysCXsVj6eGC7Vbrlw=="], - - "bun-types": ["bun-types@1.3.13", "", { "dependencies": { "@types/node": "*" } }, "sha512-QXKeHLlOLqQX9LgYaHJfzdBaV21T63HhFJnvuRCcjZiaUDpbs5ED1MgxbMra71CsryN/1dAoXuJJJwIv/2drVA=="], - - "typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="], - - "undici-types": ["undici-types@7.19.2", "", {}, "sha512-qYVnV5OEm2AW8cJMCpdV20CDyaN3g0AjDlOGf1OW4iaDEx8MwdtChUp4zu4H0VP3nDRF/8RKWH+IPp9uW0YGZg=="], - } -} diff --git a/get_history/go.mod b/get_history/go.mod new file mode 100644 index 0000000..2387dd8 --- /dev/null +++ b/get_history/go.mod @@ -0,0 +1,3 @@ +module github.com/alexselimov/hugo-unified-git-activity/get_history + +go 1.22 diff --git a/get_history/index.ts b/get_history/index.ts deleted file mode 100644 index cb335ff..0000000 --- a/get_history/index.ts +++ /dev/null @@ -1,430 +0,0 @@ -import { mkdir } from "node:fs/promises"; -import { dirname } from "node:path"; - -function parseArgs(args: string[]): { outputPath: string } { - let outputPath = "../static/activity.json"; - - for (let i = 0; i < args.length; i++) { - const arg = args[i]; - - if (arg === "--help" || arg === "-h") { - console.log( - "Usage: bun run index.ts [--output ]\n\nOptions:\n -o, --output Where to write activity.json (default: ../static/activity.json)", - ); - process.exit(0); - } - - if (arg === "--output" || arg === "-o") { - const value = args[i + 1]; - if (!value) throw new Error(`${arg} requires a path`); - outputPath = value; - i++; - continue; - } - - if (arg?.startsWith("--output=")) { - outputPath = arg.slice("--output=".length); - continue; - } - - throw new Error(`Unknown argument: ${arg}`); - } - - return { outputPath }; -} - -const cli = parseArgs(Bun.argv.slice(2)); - -const config = { - githubApiKey: process.env["GH_API_KEY"] ?? "", - forgejoApiKey: process.env["FJ_API_KEY"] ?? "", - forgejoUsername: process.env["FJ_USER"] ?? "", - forgejoURL: process.env["FJ_URL"] ?? "https://codeberg.org", - gitlabApiKey: process.env["GL_API_KEY"] ?? "", - gitlabUsername: process.env["GL_USER"] ?? "", - gitlabURL: process.env["GL_URL"] ?? "https://gitlab.com", -}; - -function areGhVarsPresent() { - return ( - config.githubApiKey?.trim().length > 0 && - config.githubApiKey?.trim().length > 0 - ); -} - -function areFjVarsPresent() { - return ( - config.forgejoApiKey?.trim().length > 0 && - config.forgejoUsername?.trim().length > 0 - ); -} - -function areGlVarsPresent() { - return ( - config.gitlabApiKey?.trim().length > 0 && - config.gitlabUsername?.trim().length > 0 - ); -} -const now = new Date(); -const oneYearAgo = new Date(); -oneYearAgo.setFullYear(oneYearAgo.getFullYear() - 1); -const formattedOneYearAgo = oneYearAgo.toISOString().replace("Z", "+00:00"); - -function buildGhQuery(from: string, to: string): string { - return `{ - viewer { - contributionsCollection(from: "${from}", to: "${to}") { - contributionCalendar { - weeks { - contributionDays { - date - contributionCount - } - } - } - commitContributionsByRepository(maxRepositories: 100) { - repository { - nameWithOwner - isMirror - } - contributions(first: 100) { - nodes { - occurredAt - commitCount - } - } - } - } - } - }`; -} - -type DayContributions = { - [repo: string]: number; -}; - -type ActivityMap = { - [day: string]: DayContributions; -}; - -function mergeInto(target: ActivityMap, source: ActivityMap): void { - for (const [day, repos] of Object.entries(source)) { - if (!target[day]) target[day] = {}; - for (const [repo, count] of Object.entries(repos)) { - target[day]![repo] = (target[day]![repo] ?? 0) + count; - } - } -} - -function repoBaseName(fullName: string): string { - return fullName.split("/")[1]?.toLowerCase() ?? fullName.toLowerCase(); -} - -function deduplicateAcrossPlatforms( - ghMap: ActivityMap, - fjMap: ActivityMap, -): ActivityMap { - const ghBaseNames = new Set(); - for (const repos of Object.values(ghMap)) { - for (const repo of Object.keys(repos)) { - if (repo !== "private") ghBaseNames.add(repoBaseName(repo)); - } - } - - const sharedBaseNames = new Set(); - for (const repos of Object.values(fjMap)) { - for (const repo of Object.keys(repos)) { - if (repo !== "private" && ghBaseNames.has(repoBaseName(repo))) { - sharedBaseNames.add(repoBaseName(repo)); - } - } - } - - const result: ActivityMap = {}; - const allDays = new Set([...Object.keys(ghMap), ...Object.keys(fjMap)]); - - for (const day of allDays) { - result[day] = {}; - const ghDay = ghMap[day] ?? {}; - const fjDay = fjMap[day] ?? {}; - - for (const [repo, count] of Object.entries(ghDay)) { - const base = repoBaseName(repo); - if (repo !== "private" && sharedBaseNames.has(base)) { - const fjRepo = Object.keys(fjDay).find((r) => repoBaseName(r) === base); - const fjCount = fjRepo ? (fjDay[fjRepo] ?? 0) : 0; - if (count >= fjCount) { - result[day]![repo] = count; - } - // else: Forgejo wins, added below - } else { - result[day]![repo] = (result[day]![repo] ?? 0) + count; - } - } - - for (const [repo, count] of Object.entries(fjDay)) { - const base = repoBaseName(repo); - if (repo !== "private" && sharedBaseNames.has(base)) { - const ghRepo = Object.keys(ghDay).find((r) => repoBaseName(r) === base); - const ghCount = ghRepo ? (ghDay[ghRepo] ?? 0) : 0; - if (count > ghCount) { - if (ghRepo) delete result[day]![ghRepo]; - result[day]![repo] = count; - } - // else: GitHub already won - } else { - result[day]![repo] = (result[day]![repo] ?? 0) + count; - } - } - - if (Object.keys(result[day]!).length === 0) { - delete result[day]; - } - } - - return result; -} - -function parseGitHubContributions(data: any): ActivityMap { - const result: ActivityMap = {}; - - const collection = data?.data?.viewer?.contributionsCollection; - - const calendarMap: { [day: string]: number } = {}; - for (const week of collection?.contributionCalendar?.weeks ?? []) { - for (const day of week.contributionDays) { - if (day.contributionCount > 0) - calendarMap[day.date] = day.contributionCount; - } - } - - for (const repoEntry of collection?.commitContributionsByRepository ?? []) { - const repoName: string = repoEntry.repository.nameWithOwner; - if (repoEntry.repository.isMirror) continue; - - for (const contribution of repoEntry.contributions.nodes) { - const day: string = contribution.occurredAt.split("T")[0]; - const count: number = contribution.commitCount; - if (!result[day]) result[day] = {}; - result[day]![repoName] = (result[day]![repoName] ?? 0) + count; - } - } - - for (const [day, total] of Object.entries(calendarMap)) { - const known = Object.values(result[day] ?? {}).reduce((a, b) => a + b, 0); - const hidden = total - known; - if (hidden > 0) { - if (!result[day]) result[day] = {}; - result[day]!["private"] = (result[day]!["private"] ?? 0) + hidden; - } - } - - return result; -} - -function parseForgejoActivity(activities: any[]): ActivityMap { - const result: ActivityMap = {}; - - for (const activity of activities) { - if (activity.repo?.mirror) continue; - if (activity.op_type !== "commit_repo") continue; - - const isPrivate: boolean = activity.repo?.private ?? false; - const repoFullName: string = activity.repo?.full_name ?? "unknown"; - const displayName = isPrivate ? "private" : repoFullName; - - let content: { Commits?: { Timestamp: string | number }[] }; - try { - content = JSON.parse(activity.content); - } catch { - continue; - } - - for (const commit of content.Commits ?? []) { - let day: string; - if (typeof commit.Timestamp === "number") { - day = new Date(commit.Timestamp * 1000).toISOString().split("T")[0]!; - } else { - day = commit.Timestamp.split("T")[0]!; - } - - if (!result[day]) result[day] = {}; - result[day]![displayName] = (result[day]![displayName] ?? 0) + 1; - } - } - - return result; -} - -async function getForgejoActivityMap(): Promise { - const pageLimit = 50; - let page = 1; - const map: ActivityMap = {}; - - while (true) { - const response = await fetch( - `${config.forgejoURL}/api/v1/users/${config.forgejoUsername}/activities/feeds?limit=${pageLimit}&page=${page}&after=${formattedOneYearAgo}`, - { headers: { Authorization: `Bearer ${config.forgejoApiKey}` } }, - ); - - if (!response.ok) { - throw new Error( - `Forgejo API error: ${response.status} ${response.statusText}`, - ); - } - - const activities = (await response.json()) as any[]; - if (activities.length === 0) break; - - const pageMap = parseForgejoActivity(activities); - mergeInto(map, pageMap); - - if (activities.length < pageLimit) break; - page++; - } - - return map; -} - -async function ghFetch(query: string): Promise { - const response = await fetch("https://api.github.com/graphql", { - method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${config.githubApiKey}`, - }, - body: JSON.stringify({ query }), - }); - if (!response.ok) { - throw new Error( - `GitHub API error: ${response.status} ${response.statusText}`, - ); - } - return response.json(); -} - -async function getGitHubActivityMap(): Promise { - const ranges = Array.from({ length: 12 }, (_, i) => { - const to = new Date(now); - to.setMonth(to.getMonth() - i); - const from = new Date(to); - from.setMonth(from.getMonth() - 1); - return { from: from.toISOString(), to: to.toISOString() }; - }); - - const results = await Promise.all( - ranges.map(({ from, to }) => ghFetch(buildGhQuery(from, to))), - ); - - const map: ActivityMap = {}; - for (const data of results) { - mergeInto(map, parseGitHubContributions(data)); - } - - return map; -} - -function parseGitLabEvents(events: any[]): ActivityMap { - const result: ActivityMap = {}; - - for (const event of events) { - const pushData = event.push_data; - if (!pushData) continue; - // "deleted" is a branch deletion — no commits to count - if (pushData.action === "deleted") continue; - - // New branch pushes report commit_count=0 even though commits exist; - // fall back to 1 when there's a known head commit. - const commitCount: number = - pushData.commit_count > 0 - ? pushData.commit_count - : pushData.commit_to - ? 1 - : 0; - if (commitCount === 0) continue; - - // GitLab doesn't include a nested project object in the events API response; - // construct a name from the fields that are always present. - const repoFullName: string = - event.project?.path_with_namespace ?? - (event.author_username && event.target_title - ? `${event.author_username}/${event.target_title}` - : "unknown"); - const isPrivate: boolean = event.project?.visibility === "private"; - const displayName = isPrivate ? "private" : repoFullName; - - const day: string = event.created_at.split("T")[0]!; - - if (!result[day]) result[day] = {}; - result[day]![displayName] = (result[day]![displayName] ?? 0) + commitCount; - } - - return result; -} - -async function getGitLabActivityMap(): Promise { - if (!config.gitlabApiKey || !config.gitlabUsername) { - return {}; - } - - const pageLimit = 100; - let page = 1; - const map: ActivityMap = {}; - const afterDate = oneYearAgo.toISOString().split("T")[0]!; - - while (true) { - const url = `${config.gitlabURL}/api/v4/users/${config.gitlabUsername}/events?action=pushed&after=${afterDate}&per_page=${pageLimit}&page=${page}`; - - const response = await fetch(url, { - headers: { "PRIVATE-TOKEN": config.gitlabApiKey }, - }); - - if (!response.ok) { - throw new Error( - `GitLab API error: ${response.status} ${response.statusText}`, - ); - } - - const events = (await response.json()) as any[]; - if (events.length === 0) break; - - const parsed = parseGitLabEvents(events); - mergeInto(map, parsed); - - if (events.length < pageLimit) break; - page++; - } - - return map; -} - -async function makeActivityMap(): Promise { - const getActivity = async ( - varsPresentCheck: () => boolean, - getActivityMap: () => Promise, - hubName: string, - ) => { - if (varsPresentCheck()) { - console.log(`Fetching ${hubName} activity`); - return getActivityMap(); - } else { - console.log(`${hubName} not configured, skipping...`); - return {}; - } - }; - const [ghMap, fjMap, glMap] = await Promise.all([ - getActivity(areGhVarsPresent, getGitHubActivityMap, "Github"), - getActivity(areFjVarsPresent, getForgejoActivityMap, "Forgejo"), - getActivity(areGlVarsPresent, getGitLabActivityMap, "Gitlab"), - ]); - - const merged = deduplicateAcrossPlatforms(ghMap, fjMap); - return deduplicateAcrossPlatforms(merged, glMap); -} - -const activityMap = await makeActivityMap(); -await mkdir(dirname(cli.outputPath), { recursive: true }); -await Bun.write(cli.outputPath, JSON.stringify(activityMap, null, 2)); -console.log( - `Wrote ${Object.keys(activityMap).length} days of activity to ${cli.outputPath}`, -); diff --git a/get_history/main.go b/get_history/main.go new file mode 100644 index 0000000..3dacfc0 --- /dev/null +++ b/get_history/main.go @@ -0,0 +1,773 @@ +package main + +import ( + "bytes" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "time" +) + +type dayContributions map[string]int +type activityMap map[string]dayContributions + +type config struct { + githubAPIKey string + forgejoAPIKey string + forgejoUser string + forgejoURL string + gitlabAPIKey string + gitlabUser string + gitlabURL string +} + +type app struct { + config config + client *http.Client + now time.Time + oneYearAgo time.Time + formattedOneYearAgo string +} + +func main() { + outputPath, err := parseArgs(os.Args[1:]) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(2) + } + + if err := loadDotEnv(); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + + now := time.Now() + oneYearAgo := now.AddDate(-1, 0, 0) + a := app{ + config: loadConfig(), + client: &http.Client{Timeout: 30 * time.Second}, + now: now, + oneYearAgo: oneYearAgo, + formattedOneYearAgo: formatGitHubTime(oneYearAgo), + } + + activity, err := a.makeActivityMap() + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + + if err := os.MkdirAll(filepath.Dir(outputPath), 0o755); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + + data, err := json.MarshalIndent(activity, "", " ") + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + data = append(data, '\n') + + if err := os.WriteFile(outputPath, data, 0o644); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + + fmt.Printf("Wrote %d days of activity to %s\n", len(activity), outputPath) +} + +func parseArgs(args []string) (string, error) { + fs := flag.NewFlagSet("get_history", flag.ContinueOnError) + fs.SetOutput(os.Stdout) + fs.Usage = func() { + fmt.Fprintln(fs.Output(), "Usage: go run . [--output ]") + fmt.Fprintln(fs.Output(), "\nOptions:") + fmt.Fprintln(fs.Output(), " -o, --output Where to write activity.json (default: ../static/activity.json)") + } + + var output string + fs.StringVar(&output, "output", "../static/activity.json", "Where to write activity.json") + fs.StringVar(&output, "o", "../static/activity.json", "Where to write activity.json") + + if err := fs.Parse(args); err != nil { + if errors.Is(err, flag.ErrHelp) { + os.Exit(0) + } + return "", err + } + if fs.NArg() > 0 { + return "", fmt.Errorf("unknown argument: %s", fs.Arg(0)) + } + return output, nil +} + +func loadConfig() config { + return config{ + githubAPIKey: os.Getenv("GH_API_KEY"), + forgejoAPIKey: os.Getenv("FJ_API_KEY"), + forgejoUser: os.Getenv("FJ_USER"), + forgejoURL: envDefault("FJ_URL", "https://codeberg.org"), + gitlabAPIKey: os.Getenv("GL_API_KEY"), + gitlabUser: os.Getenv("GL_USER"), + gitlabURL: envDefault("GL_URL", "https://gitlab.com"), + } +} + +func envDefault(key, fallback string) string { + if value := os.Getenv(key); value != "" { + return value + } + return fallback +} + +func loadDotEnv() error { + paths := []string{".env"} + if info, err := os.Stat("get_history"); err == nil && info.IsDir() { + paths = append(paths, filepath.Join("get_history", ".env")) + } + loaded := map[string]bool{} + + for _, path := range paths { + if loaded[path] { + continue + } + loaded[path] = true + + data, err := os.ReadFile(path) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + continue + } + return fmt.Errorf("read %s: %w", path, err) + } + if err := parseDotEnv(string(data)); err != nil { + return fmt.Errorf("parse %s: %w", path, err) + } + } + + return nil +} + +func parseDotEnv(contents string) error { + for lineNumber, rawLine := range strings.Split(contents, "\n") { + line := strings.TrimSpace(strings.TrimSuffix(rawLine, "\r")) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + line = strings.TrimSpace(strings.TrimPrefix(line, "export ")) + + key, value, ok := strings.Cut(line, "=") + if !ok { + return fmt.Errorf("line %d: expected KEY=VALUE", lineNumber+1) + } + + key = strings.TrimSpace(key) + if key == "" { + return fmt.Errorf("line %d: empty key", lineNumber+1) + } + + value = strings.TrimSpace(value) + if len(value) >= 2 { + quote := value[0] + if (quote == '\'' || quote == '"') && value[len(value)-1] == quote { + value = value[1 : len(value)-1] + } + } + + if _, exists := os.LookupEnv(key); !exists { + if err := os.Setenv(key, value); err != nil { + return fmt.Errorf("line %d: %w", lineNumber+1, err) + } + } + } + + return nil +} + +func (a app) areGHVarsPresent() bool { + return strings.TrimSpace(a.config.githubAPIKey) != "" +} + +func (a app) areFJVarsPresent() bool { + return strings.TrimSpace(a.config.forgejoAPIKey) != "" && strings.TrimSpace(a.config.forgejoUser) != "" +} + +func (a app) areGLVarsPresent() bool { + return strings.TrimSpace(a.config.gitlabAPIKey) != "" && strings.TrimSpace(a.config.gitlabUser) != "" +} + +func formatGitHubTime(t time.Time) string { + return t.UTC().Format("2006-01-02T15:04:05.000+00:00") +} + +func buildGHQuery(from, to string) string { + return fmt.Sprintf(`{ + viewer { + contributionsCollection(from: %q, to: %q) { + contributionCalendar { + weeks { + contributionDays { + date + contributionCount + } + } + } + commitContributionsByRepository(maxRepositories: 100) { + repository { + nameWithOwner + isMirror + } + contributions(first: 100) { + nodes { + occurredAt + commitCount + } + } + } + } + } + }`, from, to) +} + +func mergeInto(target, source activityMap) { + for day, repos := range source { + if _, ok := target[day]; !ok { + target[day] = dayContributions{} + } + for repo, count := range repos { + target[day][repo] += count + } + } +} + +func repoBaseName(fullName string) string { + parts := strings.Split(fullName, "/") + if len(parts) > 1 { + return strings.ToLower(parts[1]) + } + return strings.ToLower(fullName) +} + +func deduplicateAcrossPlatforms(left, right activityMap) activityMap { + leftBaseNames := map[string]bool{} + for _, repos := range left { + for repo := range repos { + if repo != "private" { + leftBaseNames[repoBaseName(repo)] = true + } + } + } + + sharedBaseNames := map[string]bool{} + for _, repos := range right { + for repo := range repos { + base := repoBaseName(repo) + if repo != "private" && leftBaseNames[base] { + sharedBaseNames[base] = true + } + } + } + + result := activityMap{} + days := map[string]bool{} + for day := range left { + days[day] = true + } + for day := range right { + days[day] = true + } + + for day := range days { + result[day] = dayContributions{} + leftDay := left[day] + rightDay := right[day] + + for repo, count := range leftDay { + base := repoBaseName(repo) + if repo != "private" && sharedBaseNames[base] { + rightRepo := findRepoByBase(rightDay, base) + rightCount := 0 + if rightRepo != "" { + rightCount = rightDay[rightRepo] + } + if count >= rightCount { + result[day][repo] = count + } + } else { + result[day][repo] += count + } + } + + for repo, count := range rightDay { + base := repoBaseName(repo) + if repo != "private" && sharedBaseNames[base] { + leftRepo := findRepoByBase(leftDay, base) + leftCount := 0 + if leftRepo != "" { + leftCount = leftDay[leftRepo] + } + if count > leftCount { + if leftRepo != "" { + delete(result[day], leftRepo) + } + result[day][repo] = count + } + } else { + result[day][repo] += count + } + } + + if len(result[day]) == 0 { + delete(result, day) + } + } + + return result +} + +func findRepoByBase(repos dayContributions, base string) string { + for repo := range repos { + if repoBaseName(repo) == base { + return repo + } + } + return "" +} + +func (a app) parseGitHubContributions(data []byte) (activityMap, error) { + var response struct { + Data struct { + Viewer struct { + ContributionsCollection struct { + ContributionCalendar struct { + Weeks []struct { + ContributionDays []struct { + Date string `json:"date"` + ContributionCount int `json:"contributionCount"` + } `json:"contributionDays"` + } `json:"weeks"` + } `json:"contributionCalendar"` + CommitContributionsByRepository []struct { + Repository struct { + NameWithOwner string `json:"nameWithOwner"` + IsMirror bool `json:"isMirror"` + } `json:"repository"` + Contributions struct { + Nodes []struct { + OccurredAt string `json:"occurredAt"` + CommitCount int `json:"commitCount"` + } `json:"nodes"` + } `json:"contributions"` + } `json:"commitContributionsByRepository"` + } `json:"contributionsCollection"` + } `json:"viewer"` + } `json:"data"` + } + if err := json.Unmarshal(data, &response); err != nil { + return nil, err + } + + result := activityMap{} + calendarMap := map[string]int{} + collection := response.Data.Viewer.ContributionsCollection + for _, week := range collection.ContributionCalendar.Weeks { + for _, day := range week.ContributionDays { + if day.ContributionCount > 0 { + calendarMap[day.Date] = day.ContributionCount + } + } + } + + for _, repoEntry := range collection.CommitContributionsByRepository { + if repoEntry.Repository.IsMirror { + continue + } + repoName := repoEntry.Repository.NameWithOwner + for _, contribution := range repoEntry.Contributions.Nodes { + day := strings.Split(contribution.OccurredAt, "T")[0] + if _, ok := result[day]; !ok { + result[day] = dayContributions{} + } + result[day][repoName] += contribution.CommitCount + } + } + + for day, total := range calendarMap { + known := 0 + for _, count := range result[day] { + known += count + } + hidden := total - known + if hidden > 0 { + if _, ok := result[day]; !ok { + result[day] = dayContributions{} + } + result[day]["private"] += hidden + } + } + + return result, nil +} + +func (a app) parseForgejoActivity(activities []forgejoActivity) activityMap { + result := activityMap{} + + for _, activity := range activities { + if activity.Repo.Mirror || activity.OpType != "commit_repo" { + continue + } + + displayName := activity.Repo.FullName + if activity.Repo.Private { + displayName = "private" + } + if displayName == "" { + displayName = "unknown" + } + + var content struct { + Commits []struct { + Timestamp any `json:"Timestamp"` + } `json:"Commits"` + } + if err := json.Unmarshal([]byte(activity.Content), &content); err != nil { + continue + } + + for _, commit := range content.Commits { + day, ok := forgejoCommitDay(commit.Timestamp) + if !ok { + continue + } + if _, exists := result[day]; !exists { + result[day] = dayContributions{} + } + result[day][displayName]++ + } + } + + return result +} + +type forgejoActivity struct { + OpType string `json:"op_type"` + Content string `json:"content"` + Repo struct { + Mirror bool `json:"mirror"` + Private bool `json:"private"` + FullName string `json:"full_name"` + } `json:"repo"` +} + +func forgejoCommitDay(timestamp any) (string, bool) { + switch value := timestamp.(type) { + case float64: + return time.Unix(int64(value), 0).UTC().Format("2006-01-02"), true + case string: + if value == "" { + return "", false + } + return strings.Split(value, "T")[0], true + default: + return "", false + } +} + +func (a app) getForgejoActivityMap() (activityMap, error) { + const pageLimit = 50 + page := 1 + result := activityMap{} + + for { + url := fmt.Sprintf("%s/api/v1/users/%s/activities/feeds?limit=%d&page=%d&after=%s", strings.TrimRight(a.config.forgejoURL, "/"), a.config.forgejoUser, pageLimit, page, a.formattedOneYearAgo) + body, err := a.doJSONRequest(http.MethodGet, url, nil, map[string]string{"Authorization": "Bearer " + a.config.forgejoAPIKey}, "Forgejo") + if err != nil { + return nil, err + } + + var activities []forgejoActivity + if err := json.Unmarshal(body, &activities); err != nil { + return nil, err + } + if len(activities) == 0 { + break + } + + mergeInto(result, a.parseForgejoActivity(activities)) + if len(activities) < pageLimit { + break + } + page++ + } + + return result, nil +} + +func (a app) ghFetch(query string) ([]byte, error) { + payload, err := json.Marshal(map[string]string{"query": query}) + if err != nil { + return nil, err + } + return a.doJSONRequest(http.MethodPost, "https://api.github.com/graphql", payload, map[string]string{ + "Content-Type": "application/json", + "Authorization": "Bearer " + a.config.githubAPIKey, + }, "GitHub") +} + +func (a app) getGitHubActivityMap() (activityMap, error) { + type dateRange struct{ from, to string } + ranges := make([]dateRange, 12) + for i := range ranges { + to := a.now.AddDate(0, -i, 0) + from := to.AddDate(0, -1, 0) + ranges[i] = dateRange{from: toISO(from), to: toISO(to)} + } + + result := activityMap{} + var mu sync.Mutex + var wg sync.WaitGroup + errCh := make(chan error, len(ranges)) + + for _, r := range ranges { + r := r + wg.Add(1) + go func() { + defer wg.Done() + body, err := a.ghFetch(buildGHQuery(r.from, r.to)) + if err != nil { + errCh <- err + return + } + parsed, err := a.parseGitHubContributions(body) + if err != nil { + errCh <- err + return + } + mu.Lock() + mergeInto(result, parsed) + mu.Unlock() + }() + } + + wg.Wait() + close(errCh) + for err := range errCh { + if err != nil { + return nil, err + } + } + + return result, nil +} + +func toISO(t time.Time) string { + return t.UTC().Format("2006-01-02T15:04:05.000Z") +} + +type gitlabEvent struct { + CreatedAt string `json:"created_at"` + AuthorUsername string `json:"author_username"` + TargetTitle string `json:"target_title"` + PushData *struct { + Action string `json:"action"` + CommitCount int `json:"commit_count"` + CommitTo string `json:"commit_to"` + } `json:"push_data"` + Project *struct { + PathWithNamespace string `json:"path_with_namespace"` + Visibility string `json:"visibility"` + } `json:"project"` +} + +func parseGitLabEvents(events []gitlabEvent) activityMap { + result := activityMap{} + + for _, event := range events { + if event.PushData == nil || event.PushData.Action == "deleted" { + continue + } + + commitCount := 0 + if event.PushData.CommitCount > 0 { + commitCount = event.PushData.CommitCount + } else if event.PushData.CommitTo != "" { + commitCount = 1 + } + if commitCount == 0 { + continue + } + + repoFullName := "unknown" + isPrivate := false + if event.Project != nil { + if event.Project.PathWithNamespace != "" { + repoFullName = event.Project.PathWithNamespace + } + isPrivate = event.Project.Visibility == "private" + } else if event.AuthorUsername != "" && event.TargetTitle != "" { + repoFullName = event.AuthorUsername + "/" + event.TargetTitle + } + + displayName := repoFullName + if isPrivate { + displayName = "private" + } + day := strings.Split(event.CreatedAt, "T")[0] + if _, ok := result[day]; !ok { + result[day] = dayContributions{} + } + result[day][displayName] += commitCount + } + + return result +} + +func (a app) getGitLabActivityMap() (activityMap, error) { + const pageLimit = 100 + page := 1 + result := activityMap{} + afterDate := a.oneYearAgo.UTC().Format("2006-01-02") + + for { + url := fmt.Sprintf("%s/api/v4/users/%s/events?action=pushed&after=%s&per_page=%d&page=%d", strings.TrimRight(a.config.gitlabURL, "/"), a.config.gitlabUser, afterDate, pageLimit, page) + body, err := a.doJSONRequest(http.MethodGet, url, nil, map[string]string{"PRIVATE-TOKEN": a.config.gitlabAPIKey}, "GitLab") + if err != nil { + return nil, err + } + + var events []gitlabEvent + if err := json.Unmarshal(body, &events); err != nil { + return nil, err + } + if len(events) == 0 { + break + } + + mergeInto(result, parseGitLabEvents(events)) + if len(events) < pageLimit { + break + } + page++ + } + + return result, nil +} + +func (a app) makeActivityMap() (activityMap, error) { + type source struct { + name string + configured bool + fetch func() (activityMap, error) + } + + sources := []source{ + {name: "Github", configured: a.areGHVarsPresent(), fetch: a.getGitHubActivityMap}, + {name: "Forgejo", configured: a.areFJVarsPresent(), fetch: a.getForgejoActivityMap}, + {name: "Gitlab", configured: a.areGLVarsPresent(), fetch: a.getGitLabActivityMap}, + } + + results := make([]activityMap, len(sources)) + var wg sync.WaitGroup + errCh := make(chan error, len(sources)) + + for i, source := range sources { + i, source := i, source + if !source.configured { + fmt.Printf("%s not configured, skipping...\n", source.name) + results[i] = activityMap{} + continue + } + + fmt.Printf("Fetching %s activity\n", source.name) + wg.Add(1) + go func() { + defer wg.Done() + result, err := source.fetch() + if err != nil { + errCh <- err + return + } + results[i] = result + }() + } + + wg.Wait() + close(errCh) + for err := range errCh { + if err != nil { + return nil, err + } + } + + merged := deduplicateAcrossPlatforms(results[0], results[1]) + return deduplicateAcrossPlatforms(merged, results[2]), nil +} + +func (a app) doJSONRequest(method, url string, body []byte, headers map[string]string, apiName string) ([]byte, error) { + var reader io.Reader + if body != nil { + reader = bytes.NewReader(body) + } + + req, err := http.NewRequest(method, url, reader) + if err != nil { + return nil, err + } + for key, value := range headers { + req.Header.Set(key, value) + } + + resp, err := a.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return nil, fmt.Errorf("%s API error: %d %s", apiName, resp.StatusCode, resp.Status) + } + return respBody, nil +} + +func (m activityMap) MarshalJSON() ([]byte, error) { + type day struct { + key string + val dayContributions + } + days := make([]day, 0, len(m)) + for key, val := range m { + days = append(days, day{key: key, val: val}) + } + sort.Slice(days, func(i, j int) bool { return days[i].key < days[j].key }) + + var buf bytes.Buffer + buf.WriteByte('{') + for i, day := range days { + if i > 0 { + buf.WriteByte(',') + } + key, _ := json.Marshal(day.key) + val, err := json.Marshal(day.val) + if err != nil { + return nil, err + } + buf.Write(key) + buf.WriteByte(':') + buf.Write(val) + } + buf.WriteByte('}') + return buf.Bytes(), nil +} diff --git a/get_history/package.json b/get_history/package.json deleted file mode 100644 index 3d6e7ac..0000000 --- a/get_history/package.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "name": "get_history", - "module": "index.ts", - "type": "module", - "private": true, - "devDependencies": { - "@types/bun": "latest" - }, - "peerDependencies": { - "typescript": "^5" - } -} diff --git a/get_history/tsconfig.json b/get_history/tsconfig.json deleted file mode 100644 index b2e7497..0000000 --- a/get_history/tsconfig.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "compilerOptions": { - // Environment setup & latest features - "lib": ["ESNext"], - "target": "ESNext", - "module": "Preserve", - "moduleDetection": "force", - "jsx": "react-jsx", - "allowJs": true, - "types": ["bun"], - - // Bundler mode - "moduleResolution": "bundler", - "allowImportingTsExtensions": true, - "verbatimModuleSyntax": true, - "noEmit": true, - - // Best practices - "strict": true, - "skipLibCheck": true, - "noFallthroughCasesInSwitch": true, - "noUncheckedIndexedAccess": true, - "noImplicitOverride": true, - - // Some stricter flags (disabled by default) - "noUnusedLocals": false, - "noUnusedParameters": false, - "noPropertyAccessFromIndexSignature": false - } -}