package main import ( "bytes" "encoding/json" "errors" "flag" "fmt" "io" "net/http" "os" "path/filepath" "sort" "strings" "sync" "time" ) type dayContributions map[string]int type activityMap map[string]dayContributions type config struct { githubAPIKey string forgejoAPIKey string forgejoUser string forgejoURL string gitlabAPIKey string gitlabUser string gitlabURL string } type app struct { config config client *http.Client now time.Time oneYearAgo time.Time formattedOneYearAgo string } func main() { outputPath, err := parseArgs(os.Args[1:]) if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(2) } if err := loadDotEnv(); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } now := time.Now() oneYearAgo := now.AddDate(-1, 0, 0) a := app{ config: loadConfig(), client: &http.Client{Timeout: 30 * time.Second}, now: now, oneYearAgo: oneYearAgo, formattedOneYearAgo: formatGitHubTime(oneYearAgo), } activity, err := a.makeActivityMap() if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } if err := os.MkdirAll(filepath.Dir(outputPath), 0o755); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } data, err := json.MarshalIndent(activity, "", " ") if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } data = append(data, '\n') if err := os.WriteFile(outputPath, data, 0o644); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } fmt.Printf("Wrote %d days of activity to %s\n", len(activity), outputPath) } func parseArgs(args []string) (string, error) { fs := flag.NewFlagSet("get_history", flag.ContinueOnError) fs.SetOutput(os.Stdout) fs.Usage = func() { fmt.Fprintln(fs.Output(), "Usage: go run . [--output ]") fmt.Fprintln(fs.Output(), "\nOptions:") fmt.Fprintln(fs.Output(), " -o, --output Where to write activity.json (default: ../static/activity.json)") } var output string fs.StringVar(&output, "output", "../static/activity.json", "Where to write activity.json") fs.StringVar(&output, "o", "../static/activity.json", "Where to write activity.json") if err := fs.Parse(args); err != nil { if errors.Is(err, flag.ErrHelp) { os.Exit(0) } return "", err } if fs.NArg() > 0 { return "", fmt.Errorf("unknown argument: %s", fs.Arg(0)) } return output, nil } func loadConfig() config { return config{ githubAPIKey: os.Getenv("GH_API_KEY"), forgejoAPIKey: os.Getenv("FJ_API_KEY"), forgejoUser: os.Getenv("FJ_USER"), forgejoURL: envDefault("FJ_URL", "https://codeberg.org"), gitlabAPIKey: os.Getenv("GL_API_KEY"), gitlabUser: os.Getenv("GL_USER"), gitlabURL: envDefault("GL_URL", "https://gitlab.com"), } } func envDefault(key, fallback string) string { if value := os.Getenv(key); value != "" { return value } return fallback } func loadDotEnv() error { paths := []string{".env"} if info, err := os.Stat("get_history"); err == nil && info.IsDir() { paths = append(paths, filepath.Join("get_history", ".env")) } loaded := map[string]bool{} for _, path := range paths { if loaded[path] { continue } loaded[path] = true data, err := os.ReadFile(path) if err != nil { if errors.Is(err, os.ErrNotExist) { continue } return fmt.Errorf("read %s: %w", path, err) } if err := parseDotEnv(string(data)); err != nil { return fmt.Errorf("parse %s: %w", path, err) } } return nil } func parseDotEnv(contents string) error { for lineNumber, rawLine := range strings.Split(contents, "\n") { line := strings.TrimSpace(strings.TrimSuffix(rawLine, "\r")) if line == "" || strings.HasPrefix(line, "#") { continue } line = strings.TrimSpace(strings.TrimPrefix(line, "export ")) key, value, ok := strings.Cut(line, "=") if !ok { return fmt.Errorf("line %d: expected KEY=VALUE", lineNumber+1) } key = strings.TrimSpace(key) if key == "" { return fmt.Errorf("line %d: empty key", lineNumber+1) } value = strings.TrimSpace(value) if len(value) >= 2 { quote := value[0] if (quote == '\'' || quote == '"') && value[len(value)-1] == quote { value = value[1 : len(value)-1] } } if _, exists := os.LookupEnv(key); !exists { if err := os.Setenv(key, value); err != nil { return fmt.Errorf("line %d: %w", lineNumber+1, err) } } } return nil } func (a app) areGHVarsPresent() bool { return strings.TrimSpace(a.config.githubAPIKey) != "" } func (a app) areFJVarsPresent() bool { return strings.TrimSpace(a.config.forgejoAPIKey) != "" && strings.TrimSpace(a.config.forgejoUser) != "" } func (a app) areGLVarsPresent() bool { return strings.TrimSpace(a.config.gitlabAPIKey) != "" && strings.TrimSpace(a.config.gitlabUser) != "" } func formatGitHubTime(t time.Time) string { return t.UTC().Format("2006-01-02T15:04:05.000+00:00") } func buildGHQuery(from, to string) string { return fmt.Sprintf(`{ viewer { contributionsCollection(from: %q, to: %q) { contributionCalendar { weeks { contributionDays { date contributionCount } } } commitContributionsByRepository(maxRepositories: 100) { repository { nameWithOwner isMirror } contributions(first: 100) { nodes { occurredAt commitCount } } } } } }`, from, to) } func mergeInto(target, source activityMap) { for day, repos := range source { if _, ok := target[day]; !ok { target[day] = dayContributions{} } for repo, count := range repos { target[day][repo] += count } } } func repoBaseName(fullName string) string { parts := strings.Split(fullName, "/") if len(parts) > 1 { return strings.ToLower(parts[1]) } return strings.ToLower(fullName) } func deduplicateAcrossPlatforms(left, right activityMap) activityMap { leftBaseNames := map[string]bool{} for _, repos := range left { for repo := range repos { if repo != "private" { leftBaseNames[repoBaseName(repo)] = true } } } sharedBaseNames := map[string]bool{} for _, repos := range right { for repo := range repos { base := repoBaseName(repo) if repo != "private" && leftBaseNames[base] { sharedBaseNames[base] = true } } } result := activityMap{} days := map[string]bool{} for day := range left { days[day] = true } for day := range right { days[day] = true } for day := range days { result[day] = dayContributions{} leftDay := left[day] rightDay := right[day] for repo, count := range leftDay { base := repoBaseName(repo) if repo != "private" && sharedBaseNames[base] { rightRepo := findRepoByBase(rightDay, base) rightCount := 0 if rightRepo != "" { rightCount = rightDay[rightRepo] } if count >= rightCount { result[day][repo] = count } } else { result[day][repo] += count } } for repo, count := range rightDay { base := repoBaseName(repo) if repo != "private" && sharedBaseNames[base] { leftRepo := findRepoByBase(leftDay, base) leftCount := 0 if leftRepo != "" { leftCount = leftDay[leftRepo] } if count > leftCount { if leftRepo != "" { delete(result[day], leftRepo) } result[day][repo] = count } } else { result[day][repo] += count } } if len(result[day]) == 0 { delete(result, day) } } return result } func findRepoByBase(repos dayContributions, base string) string { for repo := range repos { if repoBaseName(repo) == base { return repo } } return "" } func (a app) parseGitHubContributions(data []byte) (activityMap, error) { var response struct { Data struct { Viewer struct { ContributionsCollection struct { ContributionCalendar struct { Weeks []struct { ContributionDays []struct { Date string `json:"date"` ContributionCount int `json:"contributionCount"` } `json:"contributionDays"` } `json:"weeks"` } `json:"contributionCalendar"` CommitContributionsByRepository []struct { Repository struct { NameWithOwner string `json:"nameWithOwner"` IsMirror bool `json:"isMirror"` } `json:"repository"` Contributions struct { Nodes []struct { OccurredAt string `json:"occurredAt"` CommitCount int `json:"commitCount"` } `json:"nodes"` } `json:"contributions"` } `json:"commitContributionsByRepository"` } `json:"contributionsCollection"` } `json:"viewer"` } `json:"data"` } if err := json.Unmarshal(data, &response); err != nil { return nil, err } result := activityMap{} calendarMap := map[string]int{} collection := response.Data.Viewer.ContributionsCollection for _, week := range collection.ContributionCalendar.Weeks { for _, day := range week.ContributionDays { if day.ContributionCount > 0 { calendarMap[day.Date] = day.ContributionCount } } } for _, repoEntry := range collection.CommitContributionsByRepository { if repoEntry.Repository.IsMirror { continue } repoName := repoEntry.Repository.NameWithOwner for _, contribution := range repoEntry.Contributions.Nodes { day := strings.Split(contribution.OccurredAt, "T")[0] if _, ok := result[day]; !ok { result[day] = dayContributions{} } result[day][repoName] += contribution.CommitCount } } for day, total := range calendarMap { known := 0 for _, count := range result[day] { known += count } hidden := total - known if hidden > 0 { if _, ok := result[day]; !ok { result[day] = dayContributions{} } result[day]["private"] += hidden } } return result, nil } func (a app) parseForgejoActivity(activities []forgejoActivity) activityMap { result := activityMap{} for _, activity := range activities { if activity.Repo.Mirror || activity.OpType != "commit_repo" { continue } displayName := activity.Repo.FullName if activity.Repo.Private { displayName = "private" } if displayName == "" { displayName = "unknown" } var content struct { Commits []struct { Timestamp any `json:"Timestamp"` } `json:"Commits"` } if err := json.Unmarshal([]byte(activity.Content), &content); err != nil { continue } for _, commit := range content.Commits { day, ok := forgejoCommitDay(commit.Timestamp) if !ok { continue } if _, exists := result[day]; !exists { result[day] = dayContributions{} } result[day][displayName]++ } } return result } type forgejoActivity struct { OpType string `json:"op_type"` Content string `json:"content"` Repo struct { Mirror bool `json:"mirror"` Private bool `json:"private"` FullName string `json:"full_name"` } `json:"repo"` } func forgejoCommitDay(timestamp any) (string, bool) { switch value := timestamp.(type) { case float64: return time.Unix(int64(value), 0).UTC().Format("2006-01-02"), true case string: if value == "" { return "", false } return strings.Split(value, "T")[0], true default: return "", false } } func (a app) getForgejoActivityMap() (activityMap, error) { const pageLimit = 50 page := 1 result := activityMap{} for { url := fmt.Sprintf("%s/api/v1/users/%s/activities/feeds?limit=%d&page=%d&after=%s", strings.TrimRight(a.config.forgejoURL, "/"), a.config.forgejoUser, pageLimit, page, a.formattedOneYearAgo) body, err := a.doJSONRequest(http.MethodGet, url, nil, map[string]string{"Authorization": "Bearer " + a.config.forgejoAPIKey}, "Forgejo") if err != nil { return nil, err } var activities []forgejoActivity if err := json.Unmarshal(body, &activities); err != nil { return nil, err } if len(activities) == 0 { break } mergeInto(result, a.parseForgejoActivity(activities)) if len(activities) < pageLimit { break } page++ } return result, nil } func (a app) ghFetch(query string) ([]byte, error) { payload, err := json.Marshal(map[string]string{"query": query}) if err != nil { return nil, err } return a.doJSONRequest(http.MethodPost, "https://api.github.com/graphql", payload, map[string]string{ "Content-Type": "application/json", "Authorization": "Bearer " + a.config.githubAPIKey, }, "GitHub") } func (a app) getGitHubActivityMap() (activityMap, error) { type dateRange struct{ from, to string } ranges := make([]dateRange, 12) for i := range ranges { to := a.now.AddDate(0, -i, 0) from := to.AddDate(0, -1, 0) ranges[i] = dateRange{from: toISO(from), to: toISO(to)} } result := activityMap{} var mu sync.Mutex var wg sync.WaitGroup errCh := make(chan error, len(ranges)) for _, r := range ranges { r := r wg.Add(1) go func() { defer wg.Done() body, err := a.ghFetch(buildGHQuery(r.from, r.to)) if err != nil { errCh <- err return } parsed, err := a.parseGitHubContributions(body) if err != nil { errCh <- err return } mu.Lock() mergeInto(result, parsed) mu.Unlock() }() } wg.Wait() close(errCh) for err := range errCh { if err != nil { return nil, err } } return result, nil } func toISO(t time.Time) string { return t.UTC().Format("2006-01-02T15:04:05.000Z") } type gitlabEvent struct { CreatedAt string `json:"created_at"` AuthorUsername string `json:"author_username"` TargetTitle string `json:"target_title"` PushData *struct { Action string `json:"action"` CommitCount int `json:"commit_count"` CommitTo string `json:"commit_to"` } `json:"push_data"` Project *struct { PathWithNamespace string `json:"path_with_namespace"` Visibility string `json:"visibility"` } `json:"project"` } func parseGitLabEvents(events []gitlabEvent) activityMap { result := activityMap{} for _, event := range events { if event.PushData == nil || event.PushData.Action == "deleted" { continue } commitCount := 0 if event.PushData.CommitCount > 0 { commitCount = event.PushData.CommitCount } else if event.PushData.CommitTo != "" { commitCount = 1 } if commitCount == 0 { continue } repoFullName := "unknown" isPrivate := false if event.Project != nil { if event.Project.PathWithNamespace != "" { repoFullName = event.Project.PathWithNamespace } isPrivate = event.Project.Visibility == "private" } else if event.AuthorUsername != "" && event.TargetTitle != "" { repoFullName = event.AuthorUsername + "/" + event.TargetTitle } displayName := repoFullName if isPrivate { displayName = "private" } day := strings.Split(event.CreatedAt, "T")[0] if _, ok := result[day]; !ok { result[day] = dayContributions{} } result[day][displayName] += commitCount } return result } func (a app) getGitLabActivityMap() (activityMap, error) { const pageLimit = 100 page := 1 result := activityMap{} afterDate := a.oneYearAgo.UTC().Format("2006-01-02") for { url := fmt.Sprintf("%s/api/v4/users/%s/events?action=pushed&after=%s&per_page=%d&page=%d", strings.TrimRight(a.config.gitlabURL, "/"), a.config.gitlabUser, afterDate, pageLimit, page) body, err := a.doJSONRequest(http.MethodGet, url, nil, map[string]string{"PRIVATE-TOKEN": a.config.gitlabAPIKey}, "GitLab") if err != nil { return nil, err } var events []gitlabEvent if err := json.Unmarshal(body, &events); err != nil { return nil, err } if len(events) == 0 { break } mergeInto(result, parseGitLabEvents(events)) if len(events) < pageLimit { break } page++ } return result, nil } func (a app) makeActivityMap() (activityMap, error) { type source struct { name string configured bool fetch func() (activityMap, error) } sources := []source{ {name: "Github", configured: a.areGHVarsPresent(), fetch: a.getGitHubActivityMap}, {name: "Forgejo", configured: a.areFJVarsPresent(), fetch: a.getForgejoActivityMap}, {name: "Gitlab", configured: a.areGLVarsPresent(), fetch: a.getGitLabActivityMap}, } results := make([]activityMap, len(sources)) var wg sync.WaitGroup errCh := make(chan error, len(sources)) for i, source := range sources { i, source := i, source if !source.configured { fmt.Printf("%s not configured, skipping...\n", source.name) results[i] = activityMap{} continue } fmt.Printf("Fetching %s activity\n", source.name) wg.Add(1) go func() { defer wg.Done() result, err := source.fetch() if err != nil { errCh <- err return } results[i] = result }() } wg.Wait() close(errCh) for err := range errCh { if err != nil { return nil, err } } merged := deduplicateAcrossPlatforms(results[0], results[1]) return deduplicateAcrossPlatforms(merged, results[2]), nil } func (a app) doJSONRequest(method, url string, body []byte, headers map[string]string, apiName string) ([]byte, error) { var reader io.Reader if body != nil { reader = bytes.NewReader(body) } req, err := http.NewRequest(method, url, reader) if err != nil { return nil, err } for key, value := range headers { req.Header.Set(key, value) } resp, err := a.client.Do(req) if err != nil { return nil, err } defer resp.Body.Close() respBody, err := io.ReadAll(resp.Body) if err != nil { return nil, err } if resp.StatusCode < 200 || resp.StatusCode >= 300 { return nil, fmt.Errorf("%s API error: %d %s", apiName, resp.StatusCode, resp.Status) } return respBody, nil } func (m activityMap) MarshalJSON() ([]byte, error) { type day struct { key string val dayContributions } days := make([]day, 0, len(m)) for key, val := range m { days = append(days, day{key: key, val: val}) } sort.Slice(days, func(i, j int) bool { return days[i].key < days[j].key }) var buf bytes.Buffer buf.WriteByte('{') for i, day := range days { if i > 0 { buf.WriteByte(',') } key, _ := json.Marshal(day.key) val, err := json.Marshal(day.val) if err != nil { return nil, err } buf.Write(key) buf.WriteByte(':') buf.Write(val) } buf.WriteByte('}') return buf.Bytes(), nil }