Hugging Face's logo

Spaces:
alexliap
/
greek-cc-progress
Running

App Files Community
alexliap commited on
Commit
f42555e
·
verified ·
1 Parent(s): 29618dc

greek-cc progress tracker

Browse files
Files changed (5) hide show
  1. README.md +11 -4
  2. index.html +88 -17
  3. render.js +76 -0
  4. style.css +70 -20
  5. tracker.js +136 -0
README.md CHANGED
@@ -1,10 +1,17 @@
1
  ---
2
  title: Greek Cc Progress
3
- emoji: ⚡
4
- colorFrom: pink
5
- colorTo: red
6
  sdk: static
7
  pinned: false
 
 
 
 
8
  ---
9
 
10
- Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
 
 
 
 
1
  ---
2
  title: Greek Cc Progress
3
+ emoji: 🏃
4
+ colorFrom: yellow
5
+ colorTo: yellow
6
  sdk: static
7
  pinned: false
8
+ short_description: Progress app on Greek-CC extraction
9
+ hf_oauth: true
10
+ hf_oauth_scopes:
11
+ - read-repos
12
  ---
13
 
14
+ Progress tracker for greek-cc Common Crawl extractions. Runs entirely in the
15
+ browser: after signing in with Hugging Face, the page reads the manifests repo
16
+ and the dataset repo with your own token and infers each crawl's progress from
17
+ the uploaded chunk offsets. Fresh on every load; no server, no stored secrets.
index.html CHANGED
@@ -1,19 +1,90 @@
1
  <!doctype html>
2
- <html>
3
- <head>
4
- <meta charset="utf-8" />
5
- <meta name="viewport" content="width=device-width" />
6
- <title>My static Space</title>
7
- <link rel="stylesheet" href="style.css" />
8
- </head>
9
- <body>
10
- <div class="card">
11
- <h1>Welcome to your static Space!</h1>
12
- <p>You can modify this app directly by editing <i>index.html</i> in the Files and versions tab.</p>
13
- <p>
14
- Also don't forget to check the
15
- <a href="https://huggingface.co/docs/hub/spaces" target="_blank">Spaces documentation</a>.
16
- </p>
17
- </div>
18
- </body>
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
19
  </html>
 
1
  <!doctype html>
2
+ <html lang="en">
3
+ <head>
4
+ <meta charset="utf-8">
5
+ <meta name="viewport" content="width=device-width, initial-scale=1">
6
+ <title>greek-cc progress</title>
7
+ <link rel="stylesheet" href="style.css">
8
+ </head>
9
+ <body>
10
+ <main>
11
+ <header>
12
+ <div>
13
+ <h1>greek-cc extraction progress</h1>
14
+ <p>Greek text extraction from Common Crawl</p>
15
+ </div>
16
+ <div id="actions"></div>
17
+ </header>
18
+ <div id="view"><p class="loading">Loading…</p></div>
19
+ </main>
20
+
21
+ <script type="module">
22
+ import { oauthLoginUrl, oauthHandleRedirectIfPresent } from "https://cdn.jsdelivr.net/npm/@huggingface/hub@2.17.5/+esm";
23
+ import { collect } from "./tracker.js";
24
+ import { render } from "./render.js";
25
+
26
+ const view = document.getElementById("view");
27
+ const actions = document.getElementById("actions");
28
+ const KEY = "greek-cc-oauth";
29
+
30
+ function stored() {
31
+ try {
32
+ const s = JSON.parse(localStorage.getItem(KEY));
33
+ return s && new Date(s.accessTokenExpiresAt) > new Date() ? s : null;
34
+ } catch { return null; }
35
+ }
36
+
37
+ async function signIn() {
38
+ // huggingface.co refuses to be framed, so the OAuth round-trip has to
39
+ // happen in a top-level tab, not inside the Space page's iframe
40
+ if (window.top !== window) {
41
+ window.open(location.href, "_blank");
42
+ return;
43
+ }
44
+ location.href = await oauthLoginUrl({ scopes: window.huggingface?.variables?.OAUTH_SCOPES });
45
+ }
46
+
47
+ function showSignIn(msg = "Sign in with your Hugging Face account to read the (private) greek-cc repos.") {
48
+ actions.innerHTML = "";
49
+ view.innerHTML = `<div class="signin"><p>${msg}</p><button class="primary" id="signin">Sign in with Hugging Face</button></div>`;
50
+ document.getElementById("signin").onclick = signIn;
51
+ }
52
+
53
+ async function load(token) {
54
+ const btn = document.getElementById("refresh");
55
+ if (btn) { btn.disabled = true; btn.textContent = "Refreshing…"; }
56
+ try {
57
+ view.innerHTML = render(await collect(token));
58
+ } catch (e) {
59
+ view.innerHTML = `<p class="err">Failed to read from the Hub: ${String(e.message || e).replace(/</g, "&lt;")}</p>`;
60
+ } finally {
61
+ if (btn) { btn.disabled = false; btn.textContent = "Refresh"; }
62
+ }
63
+ }
64
+
65
+ let oauth = null;
66
+ try {
67
+ oauth = await oauthHandleRedirectIfPresent();
68
+ if (oauth) {
69
+ try { localStorage.setItem(KEY, JSON.stringify(oauth)); } catch {}
70
+ history.replaceState(null, "", location.pathname);
71
+ }
72
+ } catch (e) {
73
+ console.warn("OAuth redirect handling failed", e);
74
+ }
75
+ oauth ||= stored();
76
+
77
+ if (!oauth) {
78
+ showSignIn();
79
+ } else {
80
+ actions.innerHTML = `<button id="refresh">Refresh</button> <button id="signout">Sign out</button>`;
81
+ document.getElementById("refresh").onclick = () => load(oauth.accessToken);
82
+ document.getElementById("signout").onclick = () => {
83
+ try { localStorage.removeItem(KEY); } catch {}
84
+ showSignIn();
85
+ };
86
+ load(oauth.accessToken);
87
+ }
88
+ </script>
89
+ </body>
90
  </html>
render.js ADDED
@@ -0,0 +1,76 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import { STALL_AFTER_H } from "./tracker.js";
2
+
3
+ const LABEL = {
4
+ extracting: "Extracting",
5
+ deduping: "Deduplicating",
6
+ stalled: "Stalled",
7
+ done: "Done",
8
+ queued: "Queued",
9
+ };
10
+
11
+ const esc = (s) => String(s).replace(/[&<>"]/g, (c) => ({ "&": "&amp;", "<": "&lt;", ">": "&gt;", '"': "&quot;" })[c]);
12
+ const pct = (f) => `${(f * 100).toFixed(1)}%`;
13
+
14
+ function ago(d, now) {
15
+ if (!d) return "—";
16
+ const s = Math.floor((now - d) / 1000);
17
+ for (const [u, n] of [["d", 86400], ["h", 3600], ["m", 60]]) if (s >= n) return `${Math.floor(s / n)}${u} ago`;
18
+ return "just now";
19
+ }
20
+
21
+ function dur(s) {
22
+ if (s == null) return "—";
23
+ const m = Math.floor(s / 60), h = Math.floor(m / 60);
24
+ return h ? `~${h}h ${String(m % 60).padStart(2, "0")}m` : `~${m}m`;
25
+ }
26
+
27
+ const bar = (f, st) => `<div class="bar"><div class="fill s-${st}" style="width:${(f * 100).toFixed(1)}%"></div></div>`;
28
+ const badge = (st) => `<span class="badge s-${st}">${LABEL[st]}</span>`;
29
+
30
+ export function render(data) {
31
+ const now = data.fetchedAt;
32
+ const live = data.crawls.filter((c) => ["extracting", "deduping", "stalled"].includes(c.status));
33
+
34
+ const stats = `
35
+ <div class="stats">
36
+ <div class="stat wide"><div class="k">Overall progress</div><div class="v">${pct(data.overall)}</div>${bar(data.overall, "done")}</div>
37
+ <div class="stat"><div class="k">Crawls finished</div><div class="v">${data.done} <span class="of">/ ${data.crawls.length}</span></div></div>
38
+ <div class="stat"><div class="k">In progress</div><div class="v">${data.active}</div></div>
39
+ </div>`;
40
+
41
+ const cards = live.length
42
+ ? `<div class="cards">${live.map((c) => `
43
+ <div class="card">
44
+ <div class="card-head"><span class="mono">${esc(c.crawl)}</span>${badge(c.status)}</div>
45
+ <div class="pct">${pct(c.frac)}</div>
46
+ ${bar(c.frac, c.status)}
47
+ <div class="meta">
48
+ <span>chunk ${c.chunksDone} / ${c.nChunks}</span>
49
+ <span>last upload ${ago(c.last, now)}</span>
50
+ <span>ETA ${c.status === "extracting" ? dur(c.etaS) : "—"}</span>
51
+ </div>
52
+ </div>`).join("")}</div>`
53
+ : `<p class="muted">Nothing is running.</p>`;
54
+
55
+ const rows = data.crawls.map((c) => `
56
+ <tr>
57
+ <td class="mono">${esc(c.crawl)}</td>
58
+ <td>${badge(c.status)}</td>
59
+ <td><div class="prog">${bar(c.frac, c.status)}<span>${pct(c.frac)}</span></div></td>
60
+ <td class="num">${c.chunksDone} / ${c.nChunks}</td>
61
+ <td class="num">${c.rows.toLocaleString("en-US")}</td>
62
+ <td class="num">${c.finalFiles || "—"}</td>
63
+ <td class="num">${ago(c.last, now)}</td>
64
+ </tr>`).join("");
65
+
66
+ return `${stats}
67
+ <h2>Running now</h2>${cards}
68
+ <h2>All crawls</h2>
69
+ <div class="tablewrap"><table>
70
+ <thead><tr><th>Crawl</th><th>Status</th><th>Progress</th><th class="num">Chunks</th>
71
+ <th class="num">Manifest rows</th><th class="num">Output files</th><th class="num">Last activity</th></tr></thead>
72
+ <tbody>${rows}</tbody>
73
+ </table></div>
74
+ <p class="muted foot">As of ${now.toLocaleString()} · progress is estimated from the highest uploaded
75
+ chunk offset · "Stalled" means no chunk upload for ${STALL_AFTER_H}h+</p>`;
76
+ }
style.css CHANGED
@@ -1,28 +1,78 @@
1
- body {
2
- padding: 2rem;
3
- font-family: -apple-system, BlinkMacSystemFont, "Arial", sans-serif;
 
4
  }
5
-
6
- h1 {
7
- font-size: 16px;
8
- margin-top: 0;
 
 
9
  }
10
 
11
- p {
12
- color: rgb(107, 114, 128);
13
- font-size: 15px;
14
- margin-bottom: 10px;
15
- margin-top: 5px;
16
  }
 
 
 
 
 
 
 
 
 
17
 
18
- .card {
19
- max-width: 620px;
20
- margin: 0 auto;
21
- padding: 16px;
22
- border: 1px solid lightgray;
23
- border-radius: 16px;
24
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
25
 
26
- .card p:last-child {
27
- margin-bottom: 0;
 
28
  }
 
1
+ :root {
2
+ --bg: #f7f7f5; --surface: #ffffff; --text: #1d1f23; --muted: #6b7079; --line: #e3e4e8; --track: #ebecef;
3
+ --ok: #23884d; --run: #2f6fd0; --wait: #b37a0e; --bad: #c93c3c; --idle: #858a93;
4
+ color-scheme: light;
5
  }
6
+ @media (prefers-color-scheme: dark) {
7
+ :root:not([data-theme="light"]) {
8
+ --bg: #131417; --surface: #1b1d21; --text: #e8e9ec; --muted: #9a9fa8; --line: #2c2f35; --track: #2c2f35;
9
+ --ok: #4cbf7b; --run: #6aa2f2; --wait: #e0a53a; --bad: #ec6b6b; --idle: #8d929b;
10
+ color-scheme: dark;
11
+ }
12
  }
13
 
14
+ * { box-sizing: border-box; }
15
+ body {
16
+ margin: 0; background: var(--bg); color: var(--text);
17
+ font: 15px/1.45 system-ui, -apple-system, "Segoe UI", sans-serif; font-variant-numeric: tabular-nums;
 
18
  }
19
+ main { max-width: 1040px; margin: 0 auto; padding: 32px 16px 48px; }
20
+ header { display: flex; justify-content: space-between; align-items: flex-end; gap: 16px; flex-wrap: wrap; margin-bottom: 20px; }
21
+ h1 { font-size: 22px; margin: 0; }
22
+ header p { margin: 2px 0 0; color: var(--muted); }
23
+ h2 { font-size: 15px; margin: 28px 0 10px; }
24
+ .muted { color: var(--muted); font-size: 13px; }
25
+ .foot { margin-top: 12px; }
26
+ .err { color: var(--bad); }
27
+ .mono { font-family: ui-monospace, "SF Mono", Menlo, monospace; font-size: 13px; }
28
 
29
+ button {
30
+ font: inherit; font-size: 14px; padding: 7px 14px; border-radius: 8px; cursor: pointer;
31
+ border: 1px solid var(--line); background: var(--surface); color: var(--text);
 
 
 
32
  }
33
+ button:hover { border-color: var(--muted); }
34
+ button.primary { background: var(--text); color: var(--bg); border-color: var(--text); }
35
+ .signin { text-align: center; padding: 64px 16px; }
36
+ .signin p { color: var(--muted); margin-bottom: 18px; }
37
+
38
+ .stats { display: grid; grid-template-columns: 2fr 1fr 1fr; gap: 12px; }
39
+ .stat, .card { background: var(--surface); border: 1px solid var(--line); border-radius: 12px; padding: 14px 16px; }
40
+ .stat .k { font-size: 12px; color: var(--muted); text-transform: uppercase; letter-spacing: .04em; }
41
+ .stat .v { font-size: 28px; font-weight: 650; margin: 4px 0 8px; }
42
+ .stat .of { font-size: 16px; font-weight: 400; color: var(--muted); }
43
+
44
+ .bar { height: 8px; border-radius: 99px; background: var(--track); overflow: hidden; }
45
+ .fill { height: 100%; border-radius: 99px; }
46
+ .fill.s-done { background: var(--ok); } .fill.s-extracting { background: var(--run); }
47
+ .fill.s-deduping { background: var(--wait); } .fill.s-stalled { background: var(--bad); }
48
+ .fill.s-queued { background: var(--idle); }
49
+
50
+ .badge { font-size: 12px; font-weight: 600; padding: 2px 8px; border-radius: 99px; white-space: nowrap; }
51
+ .badge.s-done { color: var(--ok); background: color-mix(in srgb, var(--ok) 14%, transparent); }
52
+ .badge.s-extracting { color: var(--run); background: color-mix(in srgb, var(--run) 14%, transparent); }
53
+ .badge.s-deduping { color: var(--wait); background: color-mix(in srgb, var(--wait) 14%, transparent); }
54
+ .badge.s-stalled { color: var(--bad); background: color-mix(in srgb, var(--bad) 14%, transparent); }
55
+ .badge.s-queued { color: var(--idle); background: color-mix(in srgb, var(--idle) 14%, transparent); }
56
+
57
+ .cards { display: grid; grid-template-columns: repeat(auto-fill, minmax(280px, 1fr)); gap: 12px; }
58
+ .card-head { display: flex; justify-content: space-between; align-items: center; }
59
+ .card .pct { font-size: 26px; font-weight: 650; margin: 8px 0; }
60
+ .card .meta { display: flex; flex-wrap: wrap; gap: 4px 14px; margin-top: 10px; font-size: 13px; color: var(--muted); }
61
+
62
+ .tablewrap { overflow-x: auto; background: var(--surface); border: 1px solid var(--line); border-radius: 12px; }
63
+ table { width: 100%; border-collapse: collapse; font-size: 14px; }
64
+ th { text-align: left; font-size: 12px; font-weight: 600; color: var(--muted); padding: 10px 12px;
65
+ border-bottom: 1px solid var(--line); white-space: nowrap; }
66
+ td { padding: 9px 12px; border-bottom: 1px solid var(--line); white-space: nowrap; }
67
+ tr:last-child td { border-bottom: 0; }
68
+ .num { text-align: right; }
69
+ .prog { display: flex; align-items: center; gap: 10px; min-width: 170px; }
70
+ .prog .bar { flex: 1; }
71
+ .prog span { width: 48px; text-align: right; }
72
+
73
+ .loading { color: var(--muted); padding: 48px 0; text-align: center; }
74
 
75
+ @media (max-width: 640px) {
76
+ .stats { grid-template-columns: 1fr 1fr; }
77
+ .stats .wide { grid-column: 1 / -1; }
78
  }
tracker.js ADDED
@@ -0,0 +1,136 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // greek-cc extraction progress, inferred purely from what's on the HF Hub --
2
+ // the extraction CLI keeps no central state:
3
+ //
4
+ // - MANIFEST_REPO holds <crawl>.parquet per crawl queued for extraction; its
5
+ // row count (read from the parquet footer only) is the work total.
6
+ // - DATASET_REPO holds raw/<crawl>/chunk_<offset>.parquet, uploaded after
7
+ // every stage-1 chunk, and <crawl>/*.parquet once stage 2 (dedup) finishes.
8
+ //
9
+ // Chunks run in offset order, so the highest uploaded offset (+ one chunk)
10
+ // over the manifest row count is the progress. Chunks with zero survivors
11
+ // upload nothing, which is why we use the max offset rather than a file count.
12
+
13
+ import { asyncBufferFromUrl, parquetMetadataAsync } from "https://cdn.jsdelivr.net/npm/hyparquet@1.31.1/+esm";
14
+
15
+ export const MANIFEST_REPO = "alexliap/greek-cc-manifests";
16
+ export const DATASET_REPO = "alexliap/greek-cc";
17
+ const DEFAULT_CHUNK_SIZE = 200_000; // fallback only; inferred per crawl from offset gaps
18
+ export const STALL_AFTER_H = 6;
19
+
20
+ const HUB = "https://huggingface.co";
21
+ const CHUNK_RE = /^raw\/([^/]+)\/chunk_(\d+)\.parquet$/;
22
+
23
+ async function tree(token, repo, path = "", { recursive = false, expand = false } = {}) {
24
+ const params = new URLSearchParams({ recursive, expand });
25
+ let url = `${HUB}/api/datasets/${repo}/tree/main/${encodeURIComponent(path)}?${params}`;
26
+ const out = [];
27
+ while (url) {
28
+ const r = await fetch(url, { headers: { Authorization: `Bearer ${token}` } });
29
+ if (!r.ok) throw new Error(`${r.status} listing ${repo}/${path}`);
30
+ out.push(...(await r.json()));
31
+ url = r.headers.get("link")?.match(/<([^>]+)>;\s*rel="next"/)?.[1];
32
+ }
33
+ return out;
34
+ }
35
+
36
+ async function manifestRows(token, file) {
37
+ const buf = await asyncBufferFromUrl({
38
+ url: `${HUB}/datasets/${MANIFEST_REPO}/resolve/main/${file.path}`,
39
+ byteLength: file.size,
40
+ requestInit: { headers: { Authorization: `Bearer ${token}` } },
41
+ });
42
+ return Number((await parquetMetadataAsync(buf)).num_rows);
43
+ }
44
+
45
+ async function commitDates(token, path) {
46
+ const files = await tree(token, DATASET_REPO, path, { expand: true });
47
+ return files
48
+ .filter((f) => f.type === "file" && f.lastCommit)
49
+ .map((f) => new Date(f.lastCommit.date))
50
+ .sort((a, b) => a - b);
51
+ }
52
+
53
+ function chunkSize(offsets) {
54
+ const gaps = new Map();
55
+ for (let i = 1; i < offsets.length; i++) {
56
+ const g = offsets[i] - offsets[i - 1];
57
+ if (g > 0) gaps.set(g, (gaps.get(g) || 0) + 1);
58
+ }
59
+ let best = DEFAULT_CHUNK_SIZE, n = 0;
60
+ for (const [g, c] of gaps) if (c > n) [best, n] = [g, c];
61
+ return best;
62
+ }
63
+
64
+ export async function collect(token) {
65
+ const now = new Date();
66
+ const [manifestFiles, datasetFiles] = await Promise.all([
67
+ tree(token, MANIFEST_REPO),
68
+ tree(token, DATASET_REPO, "", { recursive: true }),
69
+ ]);
70
+
71
+ const offsets = {};
72
+ const finalFiles = {};
73
+ for (const f of datasetFiles) {
74
+ if (f.type !== "file") continue;
75
+ const m = f.path.match(CHUNK_RE);
76
+ if (m) (offsets[m[1]] ||= []).push(Number(m[2]));
77
+ else if (f.path.endsWith(".parquet") && f.path.split("/").length === 2) {
78
+ const crawl = f.path.split("/")[0];
79
+ finalFiles[crawl] = (finalFiles[crawl] || 0) + 1;
80
+ }
81
+ }
82
+
83
+ const manifests = manifestFiles
84
+ .filter((f) => f.type === "file" && f.path.endsWith(".parquet"))
85
+ .sort((a, b) => b.path.localeCompare(a.path));
86
+
87
+ const crawls = await Promise.all(manifests.map(async (file) => {
88
+ const crawl = file.path.replace(/\.parquet$/, "");
89
+ const offs = (offsets[crawl] || []).sort((a, b) => a - b);
90
+ const step = chunkSize(offs);
91
+ const hasFinal = Boolean(finalFiles[crawl]);
92
+ const [rows, dates] = await Promise.all([
93
+ manifestRows(token, file),
94
+ hasFinal ? commitDates(token, crawl) : offs.length ? commitDates(token, `raw/${crawl}`) : [],
95
+ ]);
96
+ const nChunks = Math.max(1, Math.ceil(rows / step));
97
+ const c = {
98
+ crawl, rows, step, nChunks,
99
+ chunksDone: offs.length ? Math.min(nChunks, Math.floor(offs.at(-1) / step) + 1) : 0,
100
+ finalFiles: finalFiles[crawl] || 0,
101
+ last: dates.at(-1) || null,
102
+ etaS: null,
103
+ };
104
+ const idleS = c.last ? (now - c.last) / 1000 : Infinity;
105
+
106
+ if (hasFinal) {
107
+ c.status = "done";
108
+ c.chunksDone = nChunks;
109
+ } else if (!offs.length) {
110
+ c.status = "queued";
111
+ } else if (c.chunksDone >= nChunks) {
112
+ c.status = "deduping";
113
+ } else if (idleS > STALL_AFTER_H * 3600) {
114
+ c.status = "stalled";
115
+ } else {
116
+ c.status = "extracting";
117
+ const recent = dates.slice(-11);
118
+ if (recent.length >= 2) {
119
+ const perChunk = (recent.at(-1) - recent[0]) / 1000 / (recent.length - 1);
120
+ c.etaS = Math.max(0, perChunk * (nChunks - c.chunksDone) - idleS);
121
+ }
122
+ }
123
+ c.frac = c.chunksDone / nChunks;
124
+ return c;
125
+ }));
126
+
127
+ const total = crawls.reduce((s, c) => s + c.nChunks, 0);
128
+ const done = crawls.reduce((s, c) => s + c.chunksDone, 0);
129
+ return {
130
+ crawls,
131
+ fetchedAt: now,
132
+ overall: total ? done / total : 0,
133
+ done: crawls.filter((c) => c.status === "done").length,
134
+ active: crawls.filter((c) => c.status === "extracting" || c.status === "deduping").length,
135
+ };
136
+ }