Prune log repo on file count, not just storage size (#44)

The Hub commit endpoint rejects pushes once a directory holds more than
10000 files. Storage-size-only retention didn't bound this: data/ and
images/ files are tiny relative to videos/, so thousands of stems
accumulated in those directories long before the byte cap ever
triggered a prune, eventually hitting the Hub's per-directory limit.

Add a LOG_MAX_FILES cap (default 8000 stems) that prunes alongside the
existing LOG_STORAGE_CAP_GB, whichever is hit first.


Claude-Session: https://claude.ai/code/session_01WHfM7C6kNJh15dWfYt8agd

Co-authored-by: Claude Sonnet 5 <noreply@anthropic.com>

app.py

@@ -61,6 +61,9 @@ _log_uploader = LogUploader(
     token=os.environ.get("LOG_HF_TOKEN"),
     repo_id=os.environ.get("LOG_DATASET_REPO"),
     max_bytes=int(float(os.environ.get("LOG_STORAGE_CAP_GB", "10")) * 1024**3),
+    # Hub's commit endpoint rejects pushes once a directory holds >10000 files; stay well under
+    # that per-directory cap (each logged stem adds one file to data/, images/, and videos/).
+    max_files=int(os.environ.get("LOG_MAX_FILES", "8000")),
     batch_interval=int(os.environ.get("LOG_BATCH_INTERVAL", "60")),
 )
 

logging_utils.py

@@ -3,8 +3,11 @@ Face Hub dataset repo (LOG-2 through LOG-9).
 
 Modeled on the FireRed-Image-Edit-1.0-Fast reference project's logging_utils.py/inference.py
 pipeline, adapted for video: logged media is referenced by path rather than embedded as bytes
-(LOG-5), the logged output video is the pre-upscale result (LOG-4), and retention is a
-total-storage-size cap rather than a file-count cap (LOG-7).
+(LOG-5), the logged output video is the pre-upscale result (LOG-4), and retention prunes on
+both a total-storage-size cap and a per-directory file-count cap (LOG-7) — the latter exists
+because the Hub commit endpoint rejects pushes once a directory (data/, images/, videos/)
+holds more than 10000 files, which a size-only cap doesn't bound since data/ and images/
+files are tiny compared to videos/.
 """
 
 import json
@@ -271,19 +274,26 @@ def _group_by_stem(existing: list[tuple[str, int]]) -> dict[str, dict[str, Any]]
     return groups
 
 
-def _build_delete_ops_by_size(existing: list[tuple[str, int]], new_batch_size: int,
-                               max_bytes: int) -> list[CommitOperationDelete]:
+def _build_delete_ops(existing: list[tuple[str, int]], new_batch: list[tuple[str, str]],
+                       max_bytes: int, max_files: int) -> list[CommitOperationDelete]:
     groups = _group_by_stem(existing)
-    total = sum(g["size"] for g in groups.values()) + new_batch_size
-    if max_bytes <= 0 or total <= max_bytes:
+    new_stems = {_stem_of(p) for p, _ in new_batch}
+    total_bytes = sum(g["size"] for g in groups.values()) + sum(os.path.getsize(local) for _, local in new_batch)
+    total_files = len(groups) + len(new_stems)
+
+    def _over_cap() -> bool:
+        return (max_bytes > 0 and total_bytes > max_bytes) or (max_files > 0 and total_files > max_files)
+
+    if not _over_cap():
         return []
     ops: list[CommitOperationDelete] = []
     for stem in sorted(groups):
-        if total <= max_bytes:
+        if not _over_cap():
             break
         group = groups[stem]
         ops.extend(CommitOperationDelete(path_in_repo=p) for p in group["paths"])
-        total -= group["size"]
+        total_bytes -= group["size"]
+        total_files -= 1
     return ops
 
 
@@ -313,10 +323,12 @@ def _delete_temp_files(paths: list[str]) -> None:
 
 
 class LogUploader:
-    def __init__(self, token: str | None, repo_id: str | None, max_bytes: int, batch_interval: int = 60) -> None:
+    def __init__(self, token: str | None, repo_id: str | None, max_bytes: int, max_files: int,
+                 batch_interval: int = 60) -> None:
         self._token = token
         self._repo_id = repo_id
         self._max_bytes = max_bytes
+        self._max_files = max_files
         self._batch_interval = batch_interval
         self._pending: list[tuple[str, str]] = []
         self._lock = threading.Lock()
@@ -412,9 +424,8 @@ class LogUploader:
         api = HfApi(token=self._token)
         api.create_repo(repo_id=self._repo_id, repo_type="dataset", private=True, exist_ok=True)
         existing = _list_existing_files_with_sizes(api, self._repo_id)
-        new_batch_size = sum(os.path.getsize(local) for _, local in batch)
         add_ops = [CommitOperationAdd(path_in_repo=p, path_or_fileobj=local) for p, local in batch]
-        del_ops = _build_delete_ops_by_size(existing, new_batch_size, self._max_bytes)
+        del_ops = _build_delete_ops(existing, batch, self._max_bytes, self._max_files)
         api.create_commit(
             repo_id=self._repo_id, repo_type="dataset",
             operations=[*add_ops, *del_ops],