import { db } from "../data/db"; import { fetchListMembership, fetchStarLists, fetchStarredRepos, detectStarListApi } from "../services/githubStarLists"; import type { GitHubConfig } from "../services/githubClient"; import { runWithConcurrency } from "./queues"; export type SyncProgress = { stage: string; current: number; total: number; failed: number; }; type ListMembershipResult = { listId: string; repoIds: string[]; failed?: boolean; }; export type SyncResult = { lists: number; repos: number; failedListIds: string[]; }; type RetryResult = { failedListIds: string[]; }; export async function syncFromGitHub( config: GitHubConfig, onProgress?: (progress: SyncProgress) => void ): Promise { const listApi = await detectStarListApi(config); onProgress?.({ stage: "fetching_star_lists", current: 0, total: 1, failed: 0 }); const lists = await fetchStarLists(config, listApi); onProgress?.({ stage: "fetching_star_lists", current: 1, total: 1, failed: 0 }); onProgress?.({ stage: "fetching_starred_repos", current: 0, total: 1, failed: 0 }); const repos = await fetchStarredRepos(config); onProgress?.({ stage: "fetching_starred_repos", current: 1, total: 1, failed: 0 }); onProgress?.({ stage: "scanning_list_membership", current: 0, total: lists.length || 1, failed: 0 }); let processed = 0; let failed = 0; const membershipTasks = lists.map((list) => async (): Promise => { try { const membership = await fetchListMembership(config, list.id); processed += 1; onProgress?.({ stage: "scanning_list_membership", current: processed, total: lists.length || 1, failed, }); return { ...membership, failed: false }; } catch { processed += 1; failed += 1; onProgress?.({ stage: "scanning_list_membership", current: processed, total: lists.length || 1, failed, }); return { listId: list.id, repoIds: [], failed: true }; } }); const memberships = await runWithConcurrency(membershipTasks, 2); const failedListIds = memberships.filter((membership) => membership.failed).map((m) => m.listId); const repoListMap = new Map>(); for (const membership of memberships) { for (const repoId of membership.repoIds) { const existing = repoListMap.get(repoId) ?? new Set(); existing.add(membership.listId); repoListMap.set(repoId, existing); } } await db.transaction("rw", db.lists, db.repos, db.repoLists, async () => { await db.lists.clear(); await db.repos.clear(); await db.repoLists.clear(); await db.lists.bulkPut( lists.map((list) => ({ id: list.id, name: list.name, description: list.description ?? "", isPrivate: false, })) ); await db.repos.bulkPut( repos.map((repo) => ({ id: repo.id, fullName: repo.fullName, description: repo.description ?? "", repoUrl: repo.url, topics: repo.topics, language: repo.language, updatedAt: repo.updatedAt, readmeExcerpt: "", stargazerCount: repo.stargazerCount, })) ); await db.repoLists.bulkPut( Array.from(repoListMap.entries()).map(([repoId, listIds]) => ({ repoId, listIds: Array.from(listIds), })) ); }); return { lists: lists.length, repos: repos.length, failedListIds }; } export async function retryListMembership( config: GitHubConfig, listIds: string[], onProgress?: (progress: SyncProgress) => void ): Promise { if (listIds.length === 0) return { failedListIds: [] }; let processed = 0; let failed = 0; onProgress?.({ stage: "retrying_list_membership", current: 0, total: listIds.length, failed: 0 }); const tasks = listIds.map((listId) => async () => { try { const membership = await fetchListMembership(config, listId); processed += 1; onProgress?.({ stage: "retrying_list_membership", current: processed, total: listIds.length, failed }); return { listId, repoIds: membership.repoIds, failed: false }; } catch { processed += 1; failed += 1; onProgress?.({ stage: "retrying_list_membership", current: processed, total: listIds.length, failed }); return { listId, repoIds: [], failed: true }; } }); const results = await runWithConcurrency(tasks, 2); const failedListIds = results.filter((item) => item.failed).map((item) => item.listId); await db.transaction("rw", db.repoLists, async () => { for (const membership of results) { for (const repoId of membership.repoIds) { const existing = await db.repoLists.get(repoId); const listIdsForRepo = existing?.listIds ?? []; const next = Array.from(new Set([...listIdsForRepo, membership.listId])); await db.repoLists.put({ repoId, listIds: next }); } } }); return { failedListIds }; }