diff --git a/get_history/.env.example b/get_history/.env.example index d92e706..eb06bba 100644 --- a/get_history/.env.example +++ b/get_history/.env.example @@ -4,4 +4,3 @@ FJ_USER= FJ_URL= GL_API_KEY= GL_USER= -GL_URL= diff --git a/get_history/.gitignore b/get_history/.gitignore index 743a741..a14702c 100644 --- a/get_history/.gitignore +++ b/get_history/.gitignore @@ -1,13 +1,32 @@ +# dependencies (bun install) +node_modules + # output out dist -get_history *.tgz +# code coverage +coverage +*.lcov + +# logs +logs +_.log +report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json + # dotenv environment variable files .env +.env.development.local +.env.test.local +.env.production.local .env.local +# caches +.eslintcache +.cache +*.tsbuildinfo + # IntelliJ based IDEs .idea diff --git a/get_history/bun.lock b/get_history/bun.lock new file mode 100644 index 0000000..978a13a --- /dev/null +++ b/get_history/bun.lock @@ -0,0 +1,26 @@ +{ + "lockfileVersion": 1, + "configVersion": 1, + "workspaces": { + "": { + "name": "get_history", + "devDependencies": { + "@types/bun": "latest", + }, + "peerDependencies": { + "typescript": "^5", + }, + }, + }, + "packages": { + "@types/bun": ["@types/bun@1.3.13", "", { "dependencies": { "bun-types": "1.3.13" } }, "sha512-9fqXWk5YIHGGnUau9TEi+qdlTYDAnOj+xLCmSTwXfAIqXr2x4tytJb43E9uCvt09zJURKXwAtkoH4nLQfzeTXw=="], + + "@types/node": ["@types/node@25.6.2", "", { "dependencies": { "undici-types": "~7.19.0" } }, "sha512-sokuT28dxf9JT5Kady1fsXOvI4HVpjZa95NKT5y9PNTIrs2AsobR4GFAA90ZG8M+nxVRLysCXsVj6eGC7Vbrlw=="], + + "bun-types": ["bun-types@1.3.13", "", { "dependencies": { "@types/node": "*" } }, "sha512-QXKeHLlOLqQX9LgYaHJfzdBaV21T63HhFJnvuRCcjZiaUDpbs5ED1MgxbMra71CsryN/1dAoXuJJJwIv/2drVA=="], + + "typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="], + + "undici-types": ["undici-types@7.19.2", "", {}, "sha512-qYVnV5OEm2AW8cJMCpdV20CDyaN3g0AjDlOGf1OW4iaDEx8MwdtChUp4zu4H0VP3nDRF/8RKWH+IPp9uW0YGZg=="], + } +} diff --git a/get_history/go.mod b/get_history/go.mod deleted file mode 100644 index 2387dd8..0000000 --- a/get_history/go.mod +++ /dev/null @@ -1,3 +0,0 @@ -module github.com/alexselimov/hugo-unified-git-activity/get_history - -go 1.22 diff --git a/get_history/index.ts b/get_history/index.ts new file mode 100644 index 0000000..fe2a0b3 --- /dev/null +++ b/get_history/index.ts @@ -0,0 +1,380 @@ +function requireEnv(key: string): string { + const value = process.env[key]; + if (!value) throw new Error(`Missing required env var: ${key}`); + return value; +} + +const config = { + githubApiKey: requireEnv("GH_API_KEY"), + forgejoApiKey: requireEnv("FJ_API_KEY"), + forgejoUsername: requireEnv("FJ_USER"), + forgejoURL: requireEnv("FJ_URL"), + gitlabApiKey: process.env["GL_API_KEY"], + gitlabUsername: process.env["GL_USER"], + gitlabURL: process.env["GL_URL"] ?? "https://gitlab.com", +}; + +const now = new Date(); +const oneYearAgo = new Date(); +oneYearAgo.setFullYear(oneYearAgo.getFullYear() - 1); +const formattedOneYearAgo = oneYearAgo.toISOString().replace("Z", "+00:00"); + +function buildGhQuery(from: string, to: string): string { + return `{ + viewer { + contributionsCollection(from: "${from}", to: "${to}") { + contributionCalendar { + weeks { + contributionDays { + date + contributionCount + } + } + } + commitContributionsByRepository(maxRepositories: 100) { + repository { + nameWithOwner + isMirror + } + contributions(first: 100) { + nodes { + occurredAt + commitCount + } + } + } + } + } + }`; +} + +type DayContributions = { + [repo: string]: number; +}; + +type ActivityMap = { + [day: string]: DayContributions; +}; + +function mergeInto(target: ActivityMap, source: ActivityMap): void { + for (const [day, repos] of Object.entries(source)) { + if (!target[day]) target[day] = {}; + for (const [repo, count] of Object.entries(repos)) { + target[day]![repo] = (target[day]![repo] ?? 0) + count; + } + } +} + +function repoBaseName(fullName: string): string { + return fullName.split("/")[1]?.toLowerCase() ?? fullName.toLowerCase(); +} + +function deduplicateAcrossPlatforms( + ghMap: ActivityMap, + fjMap: ActivityMap, +): ActivityMap { + const ghBaseNames = new Set(); + for (const repos of Object.values(ghMap)) { + for (const repo of Object.keys(repos)) { + if (repo !== "private") ghBaseNames.add(repoBaseName(repo)); + } + } + + const sharedBaseNames = new Set(); + for (const repos of Object.values(fjMap)) { + for (const repo of Object.keys(repos)) { + if (repo !== "private" && ghBaseNames.has(repoBaseName(repo))) { + sharedBaseNames.add(repoBaseName(repo)); + } + } + } + + const result: ActivityMap = {}; + const allDays = new Set([...Object.keys(ghMap), ...Object.keys(fjMap)]); + + for (const day of allDays) { + result[day] = {}; + const ghDay = ghMap[day] ?? {}; + const fjDay = fjMap[day] ?? {}; + + for (const [repo, count] of Object.entries(ghDay)) { + const base = repoBaseName(repo); + if (repo !== "private" && sharedBaseNames.has(base)) { + const fjRepo = Object.keys(fjDay).find((r) => repoBaseName(r) === base); + const fjCount = fjRepo ? (fjDay[fjRepo] ?? 0) : 0; + if (count >= fjCount) { + result[day]![repo] = count; + } + // else: Forgejo wins, added below + } else { + result[day]![repo] = (result[day]![repo] ?? 0) + count; + } + } + + for (const [repo, count] of Object.entries(fjDay)) { + const base = repoBaseName(repo); + if (repo !== "private" && sharedBaseNames.has(base)) { + const ghRepo = Object.keys(ghDay).find((r) => repoBaseName(r) === base); + const ghCount = ghRepo ? (ghDay[ghRepo] ?? 0) : 0; + if (count > ghCount) { + if (ghRepo) delete result[day]![ghRepo]; + result[day]![repo] = count; + } + // else: GitHub already won + } else { + result[day]![repo] = (result[day]![repo] ?? 0) + count; + } + } + + if (Object.keys(result[day]!).length === 0) { + delete result[day]; + } + } + + return result; +} + +function parseGitHubContributions(data: any): ActivityMap { + const result: ActivityMap = {}; + + const collection = data?.data?.viewer?.contributionsCollection; + + const calendarMap: { [day: string]: number } = {}; + for (const week of collection?.contributionCalendar?.weeks ?? []) { + for (const day of week.contributionDays) { + if (day.contributionCount > 0) calendarMap[day.date] = day.contributionCount; + } + } + + for (const repoEntry of collection?.commitContributionsByRepository ?? []) { + const repoName: string = repoEntry.repository.nameWithOwner; + if (repoEntry.repository.isMirror) continue; + + for (const contribution of repoEntry.contributions.nodes) { + const day: string = contribution.occurredAt.split("T")[0]; + const count: number = contribution.commitCount; + if (!result[day]) result[day] = {}; + result[day]![repoName] = (result[day]![repoName] ?? 0) + count; + } + } + + for (const [day, total] of Object.entries(calendarMap)) { + const known = Object.values(result[day] ?? {}).reduce((a, b) => a + b, 0); + const hidden = total - known; + if (hidden > 0) { + if (!result[day]) result[day] = {}; + result[day]!["private"] = (result[day]!["private"] ?? 0) + hidden; + } + } + + return result; +} + +function parseForgejoActivity(activities: any[]): ActivityMap { + const result: ActivityMap = {}; + + for (const activity of activities) { + if (activity.repo?.mirror) continue; + if (activity.op_type !== "commit_repo") continue; + + const isPrivate: boolean = activity.repo?.private ?? false; + const repoFullName: string = activity.repo?.full_name ?? "unknown"; + const displayName = isPrivate ? "private" : repoFullName; + + let content: { Commits?: { Timestamp: string | number }[] }; + try { + content = JSON.parse(activity.content); + } catch { + continue; + } + + for (const commit of content.Commits ?? []) { + let day: string; + if (typeof commit.Timestamp === "number") { + day = new Date(commit.Timestamp * 1000).toISOString().split("T")[0]!; + } else { + day = commit.Timestamp.split("T")[0]!; + } + + if (!result[day]) result[day] = {}; + result[day]![displayName] = (result[day]![displayName] ?? 0) + 1; + } + } + + return result; +} + +async function getForgejoActivityMap(): Promise { + const pageLimit = 50; + let page = 1; + const map: ActivityMap = {}; + + while (true) { + const response = await fetch( + `${config.forgejoURL}/api/v1/users/${config.forgejoUsername}/activities/feeds?limit=${pageLimit}&page=${page}&after=${formattedOneYearAgo}`, + { headers: { Authorization: `Bearer ${config.forgejoApiKey}` } }, + ); + + if (!response.ok) { + throw new Error( + `Forgejo API error: ${response.status} ${response.statusText}`, + ); + } + + const activities: any[] = await response.json(); + if (activities.length === 0) break; + + const pageMap = parseForgejoActivity(activities); + mergeInto(map, pageMap); + + if (activities.length < pageLimit) break; + page++; + } + + return map; +} + +async function ghFetch(query: string): Promise { + const response = await fetch("https://api.github.com/graphql", { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${config.githubApiKey}`, + }, + body: JSON.stringify({ query }), + }); + if (!response.ok) { + throw new Error( + `GitHub API error: ${response.status} ${response.statusText}`, + ); + } + return response.json(); +} + +async function getGitHubActivityMap(): Promise { + const ranges = Array.from({ length: 12 }, (_, i) => { + const to = new Date(now); + to.setMonth(to.getMonth() - i); + const from = new Date(to); + from.setMonth(from.getMonth() - 1); + return { from: from.toISOString(), to: to.toISOString() }; + }); + + const results = await Promise.all( + ranges.map(({ from, to }) => ghFetch(buildGhQuery(from, to))), + ); + + const map: ActivityMap = {}; + for (const data of results) { + mergeInto(map, parseGitHubContributions(data)); + } + + return map; +} + +function parseGitLabEvents(events: any[]): ActivityMap { + const result: ActivityMap = {}; + + for (const event of events) { + const pushData = event.push_data; + if (!pushData) continue; + // "deleted" is a branch deletion — no commits to count + if (pushData.action === "deleted") continue; + + // New branch pushes report commit_count=0 even though commits exist; + // fall back to 1 when there's a known head commit. + const commitCount: number = + pushData.commit_count > 0 + ? pushData.commit_count + : pushData.commit_to + ? 1 + : 0; + if (commitCount === 0) continue; + + // GitLab doesn't include a nested project object in the events API response; + // construct a name from the fields that are always present. + const repoFullName: string = + event.project?.path_with_namespace ?? + (event.author_username && event.target_title + ? `${event.author_username}/${event.target_title}` + : "unknown"); + const isPrivate: boolean = event.project?.visibility === "private"; + const displayName = isPrivate ? "private" : repoFullName; + + const day: string = event.created_at.split("T")[0]!; + + if (!result[day]) result[day] = {}; + result[day]![displayName] = (result[day]![displayName] ?? 0) + commitCount; + } + + return result; +} + +async function getGitLabActivityMap(): Promise { + if (!config.gitlabApiKey || !config.gitlabUsername) { + console.log("[GitLab] Skipping: GL_API_KEY or GL_USER not set"); + return {}; + } + + const pageLimit = 100; + let page = 1; + const map: ActivityMap = {}; + const afterDate = oneYearAgo.toISOString().split("T")[0]!; + + console.log(`[GitLab] Fetching events for user "${config.gitlabUsername}" from ${config.gitlabURL} after ${afterDate}`); + + while (true) { + const url = `${config.gitlabURL}/api/v4/users/${config.gitlabUsername}/events?action=pushed&after=${afterDate}&per_page=${pageLimit}&page=${page}`; + console.log(`[GitLab] GET ${url}`); + + const response = await fetch(url, { + headers: { "PRIVATE-TOKEN": config.gitlabApiKey }, + }); + + console.log(`[GitLab] Response: ${response.status} ${response.statusText}`); + + if (!response.ok) { + const body = await response.text(); + console.error(`[GitLab] Error body: ${body}`); + throw new Error(`GitLab API error: ${response.status} ${response.statusText}`); + } + + const events: any[] = await response.json(); + console.log(`[GitLab] Page ${page}: received ${events.length} events`); + + if (events.length > 0) { + console.log(`[GitLab] First event sample:`, JSON.stringify(events[0], null, 2)); + } + + if (events.length === 0) break; + + const parsed = parseGitLabEvents(events); + console.log(`[GitLab] Page ${page}: parsed ${Object.keys(parsed).length} days of activity`); + mergeInto(map, parsed); + + if (events.length < pageLimit) break; + page++; + } + + return map; +} + +async function makeActivityMap(): Promise { + const [ghMap, fjMap, glMap] = await Promise.all([ + getGitHubActivityMap(), + getForgejoActivityMap(), + getGitLabActivityMap(), + ]); + + const merged = deduplicateAcrossPlatforms(ghMap, fjMap); + return deduplicateAcrossPlatforms(merged, glMap); +} + +const activityMap = await makeActivityMap(); +await Bun.write( + "../static/activity.json", + JSON.stringify(activityMap, null, 2), +); +console.log( + `Wrote ${Object.keys(activityMap).length} days of activity to out/activity.json`, +); diff --git a/get_history/main.go b/get_history/main.go deleted file mode 100644 index 3dacfc0..0000000 --- a/get_history/main.go +++ /dev/null @@ -1,773 +0,0 @@ -package main - -import ( - "bytes" - "encoding/json" - "errors" - "flag" - "fmt" - "io" - "net/http" - "os" - "path/filepath" - "sort" - "strings" - "sync" - "time" -) - -type dayContributions map[string]int -type activityMap map[string]dayContributions - -type config struct { - githubAPIKey string - forgejoAPIKey string - forgejoUser string - forgejoURL string - gitlabAPIKey string - gitlabUser string - gitlabURL string -} - -type app struct { - config config - client *http.Client - now time.Time - oneYearAgo time.Time - formattedOneYearAgo string -} - -func main() { - outputPath, err := parseArgs(os.Args[1:]) - if err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(2) - } - - if err := loadDotEnv(); err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) - } - - now := time.Now() - oneYearAgo := now.AddDate(-1, 0, 0) - a := app{ - config: loadConfig(), - client: &http.Client{Timeout: 30 * time.Second}, - now: now, - oneYearAgo: oneYearAgo, - formattedOneYearAgo: formatGitHubTime(oneYearAgo), - } - - activity, err := a.makeActivityMap() - if err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) - } - - if err := os.MkdirAll(filepath.Dir(outputPath), 0o755); err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) - } - - data, err := json.MarshalIndent(activity, "", " ") - if err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) - } - data = append(data, '\n') - - if err := os.WriteFile(outputPath, data, 0o644); err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) - } - - fmt.Printf("Wrote %d days of activity to %s\n", len(activity), outputPath) -} - -func parseArgs(args []string) (string, error) { - fs := flag.NewFlagSet("get_history", flag.ContinueOnError) - fs.SetOutput(os.Stdout) - fs.Usage = func() { - fmt.Fprintln(fs.Output(), "Usage: go run . [--output ]") - fmt.Fprintln(fs.Output(), "\nOptions:") - fmt.Fprintln(fs.Output(), " -o, --output Where to write activity.json (default: ../static/activity.json)") - } - - var output string - fs.StringVar(&output, "output", "../static/activity.json", "Where to write activity.json") - fs.StringVar(&output, "o", "../static/activity.json", "Where to write activity.json") - - if err := fs.Parse(args); err != nil { - if errors.Is(err, flag.ErrHelp) { - os.Exit(0) - } - return "", err - } - if fs.NArg() > 0 { - return "", fmt.Errorf("unknown argument: %s", fs.Arg(0)) - } - return output, nil -} - -func loadConfig() config { - return config{ - githubAPIKey: os.Getenv("GH_API_KEY"), - forgejoAPIKey: os.Getenv("FJ_API_KEY"), - forgejoUser: os.Getenv("FJ_USER"), - forgejoURL: envDefault("FJ_URL", "https://codeberg.org"), - gitlabAPIKey: os.Getenv("GL_API_KEY"), - gitlabUser: os.Getenv("GL_USER"), - gitlabURL: envDefault("GL_URL", "https://gitlab.com"), - } -} - -func envDefault(key, fallback string) string { - if value := os.Getenv(key); value != "" { - return value - } - return fallback -} - -func loadDotEnv() error { - paths := []string{".env"} - if info, err := os.Stat("get_history"); err == nil && info.IsDir() { - paths = append(paths, filepath.Join("get_history", ".env")) - } - loaded := map[string]bool{} - - for _, path := range paths { - if loaded[path] { - continue - } - loaded[path] = true - - data, err := os.ReadFile(path) - if err != nil { - if errors.Is(err, os.ErrNotExist) { - continue - } - return fmt.Errorf("read %s: %w", path, err) - } - if err := parseDotEnv(string(data)); err != nil { - return fmt.Errorf("parse %s: %w", path, err) - } - } - - return nil -} - -func parseDotEnv(contents string) error { - for lineNumber, rawLine := range strings.Split(contents, "\n") { - line := strings.TrimSpace(strings.TrimSuffix(rawLine, "\r")) - if line == "" || strings.HasPrefix(line, "#") { - continue - } - line = strings.TrimSpace(strings.TrimPrefix(line, "export ")) - - key, value, ok := strings.Cut(line, "=") - if !ok { - return fmt.Errorf("line %d: expected KEY=VALUE", lineNumber+1) - } - - key = strings.TrimSpace(key) - if key == "" { - return fmt.Errorf("line %d: empty key", lineNumber+1) - } - - value = strings.TrimSpace(value) - if len(value) >= 2 { - quote := value[0] - if (quote == '\'' || quote == '"') && value[len(value)-1] == quote { - value = value[1 : len(value)-1] - } - } - - if _, exists := os.LookupEnv(key); !exists { - if err := os.Setenv(key, value); err != nil { - return fmt.Errorf("line %d: %w", lineNumber+1, err) - } - } - } - - return nil -} - -func (a app) areGHVarsPresent() bool { - return strings.TrimSpace(a.config.githubAPIKey) != "" -} - -func (a app) areFJVarsPresent() bool { - return strings.TrimSpace(a.config.forgejoAPIKey) != "" && strings.TrimSpace(a.config.forgejoUser) != "" -} - -func (a app) areGLVarsPresent() bool { - return strings.TrimSpace(a.config.gitlabAPIKey) != "" && strings.TrimSpace(a.config.gitlabUser) != "" -} - -func formatGitHubTime(t time.Time) string { - return t.UTC().Format("2006-01-02T15:04:05.000+00:00") -} - -func buildGHQuery(from, to string) string { - return fmt.Sprintf(`{ - viewer { - contributionsCollection(from: %q, to: %q) { - contributionCalendar { - weeks { - contributionDays { - date - contributionCount - } - } - } - commitContributionsByRepository(maxRepositories: 100) { - repository { - nameWithOwner - isMirror - } - contributions(first: 100) { - nodes { - occurredAt - commitCount - } - } - } - } - } - }`, from, to) -} - -func mergeInto(target, source activityMap) { - for day, repos := range source { - if _, ok := target[day]; !ok { - target[day] = dayContributions{} - } - for repo, count := range repos { - target[day][repo] += count - } - } -} - -func repoBaseName(fullName string) string { - parts := strings.Split(fullName, "/") - if len(parts) > 1 { - return strings.ToLower(parts[1]) - } - return strings.ToLower(fullName) -} - -func deduplicateAcrossPlatforms(left, right activityMap) activityMap { - leftBaseNames := map[string]bool{} - for _, repos := range left { - for repo := range repos { - if repo != "private" { - leftBaseNames[repoBaseName(repo)] = true - } - } - } - - sharedBaseNames := map[string]bool{} - for _, repos := range right { - for repo := range repos { - base := repoBaseName(repo) - if repo != "private" && leftBaseNames[base] { - sharedBaseNames[base] = true - } - } - } - - result := activityMap{} - days := map[string]bool{} - for day := range left { - days[day] = true - } - for day := range right { - days[day] = true - } - - for day := range days { - result[day] = dayContributions{} - leftDay := left[day] - rightDay := right[day] - - for repo, count := range leftDay { - base := repoBaseName(repo) - if repo != "private" && sharedBaseNames[base] { - rightRepo := findRepoByBase(rightDay, base) - rightCount := 0 - if rightRepo != "" { - rightCount = rightDay[rightRepo] - } - if count >= rightCount { - result[day][repo] = count - } - } else { - result[day][repo] += count - } - } - - for repo, count := range rightDay { - base := repoBaseName(repo) - if repo != "private" && sharedBaseNames[base] { - leftRepo := findRepoByBase(leftDay, base) - leftCount := 0 - if leftRepo != "" { - leftCount = leftDay[leftRepo] - } - if count > leftCount { - if leftRepo != "" { - delete(result[day], leftRepo) - } - result[day][repo] = count - } - } else { - result[day][repo] += count - } - } - - if len(result[day]) == 0 { - delete(result, day) - } - } - - return result -} - -func findRepoByBase(repos dayContributions, base string) string { - for repo := range repos { - if repoBaseName(repo) == base { - return repo - } - } - return "" -} - -func (a app) parseGitHubContributions(data []byte) (activityMap, error) { - var response struct { - Data struct { - Viewer struct { - ContributionsCollection struct { - ContributionCalendar struct { - Weeks []struct { - ContributionDays []struct { - Date string `json:"date"` - ContributionCount int `json:"contributionCount"` - } `json:"contributionDays"` - } `json:"weeks"` - } `json:"contributionCalendar"` - CommitContributionsByRepository []struct { - Repository struct { - NameWithOwner string `json:"nameWithOwner"` - IsMirror bool `json:"isMirror"` - } `json:"repository"` - Contributions struct { - Nodes []struct { - OccurredAt string `json:"occurredAt"` - CommitCount int `json:"commitCount"` - } `json:"nodes"` - } `json:"contributions"` - } `json:"commitContributionsByRepository"` - } `json:"contributionsCollection"` - } `json:"viewer"` - } `json:"data"` - } - if err := json.Unmarshal(data, &response); err != nil { - return nil, err - } - - result := activityMap{} - calendarMap := map[string]int{} - collection := response.Data.Viewer.ContributionsCollection - for _, week := range collection.ContributionCalendar.Weeks { - for _, day := range week.ContributionDays { - if day.ContributionCount > 0 { - calendarMap[day.Date] = day.ContributionCount - } - } - } - - for _, repoEntry := range collection.CommitContributionsByRepository { - if repoEntry.Repository.IsMirror { - continue - } - repoName := repoEntry.Repository.NameWithOwner - for _, contribution := range repoEntry.Contributions.Nodes { - day := strings.Split(contribution.OccurredAt, "T")[0] - if _, ok := result[day]; !ok { - result[day] = dayContributions{} - } - result[day][repoName] += contribution.CommitCount - } - } - - for day, total := range calendarMap { - known := 0 - for _, count := range result[day] { - known += count - } - hidden := total - known - if hidden > 0 { - if _, ok := result[day]; !ok { - result[day] = dayContributions{} - } - result[day]["private"] += hidden - } - } - - return result, nil -} - -func (a app) parseForgejoActivity(activities []forgejoActivity) activityMap { - result := activityMap{} - - for _, activity := range activities { - if activity.Repo.Mirror || activity.OpType != "commit_repo" { - continue - } - - displayName := activity.Repo.FullName - if activity.Repo.Private { - displayName = "private" - } - if displayName == "" { - displayName = "unknown" - } - - var content struct { - Commits []struct { - Timestamp any `json:"Timestamp"` - } `json:"Commits"` - } - if err := json.Unmarshal([]byte(activity.Content), &content); err != nil { - continue - } - - for _, commit := range content.Commits { - day, ok := forgejoCommitDay(commit.Timestamp) - if !ok { - continue - } - if _, exists := result[day]; !exists { - result[day] = dayContributions{} - } - result[day][displayName]++ - } - } - - return result -} - -type forgejoActivity struct { - OpType string `json:"op_type"` - Content string `json:"content"` - Repo struct { - Mirror bool `json:"mirror"` - Private bool `json:"private"` - FullName string `json:"full_name"` - } `json:"repo"` -} - -func forgejoCommitDay(timestamp any) (string, bool) { - switch value := timestamp.(type) { - case float64: - return time.Unix(int64(value), 0).UTC().Format("2006-01-02"), true - case string: - if value == "" { - return "", false - } - return strings.Split(value, "T")[0], true - default: - return "", false - } -} - -func (a app) getForgejoActivityMap() (activityMap, error) { - const pageLimit = 50 - page := 1 - result := activityMap{} - - for { - url := fmt.Sprintf("%s/api/v1/users/%s/activities/feeds?limit=%d&page=%d&after=%s", strings.TrimRight(a.config.forgejoURL, "/"), a.config.forgejoUser, pageLimit, page, a.formattedOneYearAgo) - body, err := a.doJSONRequest(http.MethodGet, url, nil, map[string]string{"Authorization": "Bearer " + a.config.forgejoAPIKey}, "Forgejo") - if err != nil { - return nil, err - } - - var activities []forgejoActivity - if err := json.Unmarshal(body, &activities); err != nil { - return nil, err - } - if len(activities) == 0 { - break - } - - mergeInto(result, a.parseForgejoActivity(activities)) - if len(activities) < pageLimit { - break - } - page++ - } - - return result, nil -} - -func (a app) ghFetch(query string) ([]byte, error) { - payload, err := json.Marshal(map[string]string{"query": query}) - if err != nil { - return nil, err - } - return a.doJSONRequest(http.MethodPost, "https://api.github.com/graphql", payload, map[string]string{ - "Content-Type": "application/json", - "Authorization": "Bearer " + a.config.githubAPIKey, - }, "GitHub") -} - -func (a app) getGitHubActivityMap() (activityMap, error) { - type dateRange struct{ from, to string } - ranges := make([]dateRange, 12) - for i := range ranges { - to := a.now.AddDate(0, -i, 0) - from := to.AddDate(0, -1, 0) - ranges[i] = dateRange{from: toISO(from), to: toISO(to)} - } - - result := activityMap{} - var mu sync.Mutex - var wg sync.WaitGroup - errCh := make(chan error, len(ranges)) - - for _, r := range ranges { - r := r - wg.Add(1) - go func() { - defer wg.Done() - body, err := a.ghFetch(buildGHQuery(r.from, r.to)) - if err != nil { - errCh <- err - return - } - parsed, err := a.parseGitHubContributions(body) - if err != nil { - errCh <- err - return - } - mu.Lock() - mergeInto(result, parsed) - mu.Unlock() - }() - } - - wg.Wait() - close(errCh) - for err := range errCh { - if err != nil { - return nil, err - } - } - - return result, nil -} - -func toISO(t time.Time) string { - return t.UTC().Format("2006-01-02T15:04:05.000Z") -} - -type gitlabEvent struct { - CreatedAt string `json:"created_at"` - AuthorUsername string `json:"author_username"` - TargetTitle string `json:"target_title"` - PushData *struct { - Action string `json:"action"` - CommitCount int `json:"commit_count"` - CommitTo string `json:"commit_to"` - } `json:"push_data"` - Project *struct { - PathWithNamespace string `json:"path_with_namespace"` - Visibility string `json:"visibility"` - } `json:"project"` -} - -func parseGitLabEvents(events []gitlabEvent) activityMap { - result := activityMap{} - - for _, event := range events { - if event.PushData == nil || event.PushData.Action == "deleted" { - continue - } - - commitCount := 0 - if event.PushData.CommitCount > 0 { - commitCount = event.PushData.CommitCount - } else if event.PushData.CommitTo != "" { - commitCount = 1 - } - if commitCount == 0 { - continue - } - - repoFullName := "unknown" - isPrivate := false - if event.Project != nil { - if event.Project.PathWithNamespace != "" { - repoFullName = event.Project.PathWithNamespace - } - isPrivate = event.Project.Visibility == "private" - } else if event.AuthorUsername != "" && event.TargetTitle != "" { - repoFullName = event.AuthorUsername + "/" + event.TargetTitle - } - - displayName := repoFullName - if isPrivate { - displayName = "private" - } - day := strings.Split(event.CreatedAt, "T")[0] - if _, ok := result[day]; !ok { - result[day] = dayContributions{} - } - result[day][displayName] += commitCount - } - - return result -} - -func (a app) getGitLabActivityMap() (activityMap, error) { - const pageLimit = 100 - page := 1 - result := activityMap{} - afterDate := a.oneYearAgo.UTC().Format("2006-01-02") - - for { - url := fmt.Sprintf("%s/api/v4/users/%s/events?action=pushed&after=%s&per_page=%d&page=%d", strings.TrimRight(a.config.gitlabURL, "/"), a.config.gitlabUser, afterDate, pageLimit, page) - body, err := a.doJSONRequest(http.MethodGet, url, nil, map[string]string{"PRIVATE-TOKEN": a.config.gitlabAPIKey}, "GitLab") - if err != nil { - return nil, err - } - - var events []gitlabEvent - if err := json.Unmarshal(body, &events); err != nil { - return nil, err - } - if len(events) == 0 { - break - } - - mergeInto(result, parseGitLabEvents(events)) - if len(events) < pageLimit { - break - } - page++ - } - - return result, nil -} - -func (a app) makeActivityMap() (activityMap, error) { - type source struct { - name string - configured bool - fetch func() (activityMap, error) - } - - sources := []source{ - {name: "Github", configured: a.areGHVarsPresent(), fetch: a.getGitHubActivityMap}, - {name: "Forgejo", configured: a.areFJVarsPresent(), fetch: a.getForgejoActivityMap}, - {name: "Gitlab", configured: a.areGLVarsPresent(), fetch: a.getGitLabActivityMap}, - } - - results := make([]activityMap, len(sources)) - var wg sync.WaitGroup - errCh := make(chan error, len(sources)) - - for i, source := range sources { - i, source := i, source - if !source.configured { - fmt.Printf("%s not configured, skipping...\n", source.name) - results[i] = activityMap{} - continue - } - - fmt.Printf("Fetching %s activity\n", source.name) - wg.Add(1) - go func() { - defer wg.Done() - result, err := source.fetch() - if err != nil { - errCh <- err - return - } - results[i] = result - }() - } - - wg.Wait() - close(errCh) - for err := range errCh { - if err != nil { - return nil, err - } - } - - merged := deduplicateAcrossPlatforms(results[0], results[1]) - return deduplicateAcrossPlatforms(merged, results[2]), nil -} - -func (a app) doJSONRequest(method, url string, body []byte, headers map[string]string, apiName string) ([]byte, error) { - var reader io.Reader - if body != nil { - reader = bytes.NewReader(body) - } - - req, err := http.NewRequest(method, url, reader) - if err != nil { - return nil, err - } - for key, value := range headers { - req.Header.Set(key, value) - } - - resp, err := a.client.Do(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - respBody, err := io.ReadAll(resp.Body) - if err != nil { - return nil, err - } - if resp.StatusCode < 200 || resp.StatusCode >= 300 { - return nil, fmt.Errorf("%s API error: %d %s", apiName, resp.StatusCode, resp.Status) - } - return respBody, nil -} - -func (m activityMap) MarshalJSON() ([]byte, error) { - type day struct { - key string - val dayContributions - } - days := make([]day, 0, len(m)) - for key, val := range m { - days = append(days, day{key: key, val: val}) - } - sort.Slice(days, func(i, j int) bool { return days[i].key < days[j].key }) - - var buf bytes.Buffer - buf.WriteByte('{') - for i, day := range days { - if i > 0 { - buf.WriteByte(',') - } - key, _ := json.Marshal(day.key) - val, err := json.Marshal(day.val) - if err != nil { - return nil, err - } - buf.Write(key) - buf.WriteByte(':') - buf.Write(val) - } - buf.WriteByte('}') - return buf.Bytes(), nil -} diff --git a/get_history/package.json b/get_history/package.json new file mode 100644 index 0000000..3d6e7ac --- /dev/null +++ b/get_history/package.json @@ -0,0 +1,12 @@ +{ + "name": "get_history", + "module": "index.ts", + "type": "module", + "private": true, + "devDependencies": { + "@types/bun": "latest" + }, + "peerDependencies": { + "typescript": "^5" + } +} diff --git a/get_history/tsconfig.json b/get_history/tsconfig.json new file mode 100644 index 0000000..b2e7497 --- /dev/null +++ b/get_history/tsconfig.json @@ -0,0 +1,30 @@ +{ + "compilerOptions": { + // Environment setup & latest features + "lib": ["ESNext"], + "target": "ESNext", + "module": "Preserve", + "moduleDetection": "force", + "jsx": "react-jsx", + "allowJs": true, + "types": ["bun"], + + // Bundler mode + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "verbatimModuleSyntax": true, + "noEmit": true, + + // Best practices + "strict": true, + "skipLibCheck": true, + "noFallthroughCasesInSwitch": true, + "noUncheckedIndexedAccess": true, + "noImplicitOverride": true, + + // Some stricter flags (disabled by default) + "noUnusedLocals": false, + "noUnusedParameters": false, + "noPropertyAccessFromIndexSignature": false + } +}