Initial Space upload

.gitattributes

@@ -33,3 +33,13 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+confusion_matrix.png filter=lfs diff=lfs merge=lfs -text
+examples/Abuse005_x264.mp4 filter=lfs diff=lfs merge=lfs -text
+examples/Arson003_x264.mp4 filter=lfs diff=lfs merge=lfs -text
+examples/Explosion001_x264.mp4 filter=lfs diff=lfs merge=lfs -text
+examples/Fighting004_x264.mp4 filter=lfs diff=lfs merge=lfs -text
+examples/Normal_Videos_150_x264.mp4 filter=lfs diff=lfs merge=lfs -text
+examples/Robbery005_x264.mp4 filter=lfs diff=lfs merge=lfs -text
+examples/Shoplifting001_x264.mp4 filter=lfs diff=lfs merge=lfs -text
+examples/Stealing002_x264.mp4 filter=lfs diff=lfs merge=lfs -text
+per_class_f1.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,12 +1,79 @@
 ---
-title: Xclip Crime Demo
-emoji: 👀
+title: X-CLIP Crime Classifier
+emoji: 🎬
 colorFrom: blue
-colorTo: green
+colorTo: indigo
 sdk: gradio
-sdk_version: 6.12.0
+sdk_version: "6.12.0"
 app_file: app.py
 pinned: false
+license: mit
+short_description: X-CLIP fine-tuned on UCF-Crime with LLM descriptions
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# Crime Detection with X-CLIP + LLM Descriptions
+
+Fine-tunes [X-CLIP](https://huggingface.co/microsoft/xclip-base-patch16) on the
+UCF-Crime dataset using rich scene descriptions generated by Google Gemini Flash,
+instead of bare category labels. Multi-class classification over 14 categories
+(13 crime types + Normal).
+
+## Quick try
+
+Click any of the 8 example videos on the **Live Prediction** tab. Each example is
+labelled with one of three scenarios:
+
+- 🟢 **All pass** — simple case where CLIP, X-CLIP, and our model all get it right
+- 💡 **Ours only** — baselines fail, the fine-tuned model recovers it
+- 🔴 **All fail** — open limitation (mostly visually ambiguous classes like Abuse / Robbery)
+
+The **Method Comparison** tab runs all three side-by-side on the same video
+(slower — ~45s on the free CPU tier because models are swapped sequentially).
+
+## Method in one paragraph
+
+Each of the 532 training videos gets a unique paragraph-long description from
+Gemini Flash. These pairs are used to contrastively fine-tune X-CLIP's visual
+encoder (last 3 transformer blocks + CCT + MIT). At inference, we average 5
+hand-crafted descriptive prompts per class and classify test videos by cosine
+similarity in the shared embedding space.
+
+## Numbers (UCF-Crime Action Recognition test split, 168 videos)
+
+| Method | Accuracy | F1 (macro) | AUC-ROC |
+|---|---|---|---|
+| CLIP zero-shot | 23.2% | 21.0% | 0.77 |
+| X-CLIP zero-shot | 17.3% | 13.9% | 0.70 |
+| **Ours (X-CLIP + LLM)** | **26.2%** | **22.1%** | **0.80** |
+
+## Free CPU tier note
+
+This Space runs on CPU. Inference takes roughly 30-60 seconds per video on the
+Live Prediction tab, and ~2-3 minutes on the Method Comparison tab (3 models
+loaded sequentially). GPU-backed inference (~2-5s per video) would require a
+paid Space upgrade.
+
+## Dataset & license
+
+- **Dataset:** [UCF-Crime](https://webpages.uncc.edu/cchen62/dataset.html)
+  (Sultani et al., CVPR 2018). Used here under its research-only license.
+- **Example videos:** 8 clips from the public UCF-Crime test split.
+- **Code:** MIT.
+
+## Citation
+
+If this project is useful to you, please cite the original UCF-Crime and X-CLIP papers:
+
+```
+@inproceedings{sultani2018real,
+  title={Real-world anomaly detection in surveillance videos},
+  author={Sultani, Waqas and Chen, Chen and Shah, Mubarak},
+  booktitle={CVPR}, year={2018}
+}
+
+@inproceedings{ni2022expanding,
+  title={Expanding language-image pretrained models for general video recognition},
+  author={Ni, Bolin and others},
+  booktitle={ECCV}, year={2022}
+}
+```

app.py

@@ -0,0 +1,363 @@
+"""Gradio app for the Hugging Face Space.
+
+Adapted from the local demo. Entry point: `app.py` at the Space root.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import gradio as gr
+
+import config_local as config
+from descriptive_prompts import PROMPTS
+from inference import (
+    CLASS_DISPLAY,
+    extract_frames_from_video,
+    predict_clip_zs,
+    predict_ours,
+    predict_xclip_zs,
+)
+from example_videos import EXAMPLES, get_example_video_paths
+
+APP_DIR = Path(__file__).resolve().parent
+PLOTS_DIR = APP_DIR
+
+# Map filename -> example dict for ground-truth display
+_EXAMPLE_BY_FILENAME: dict[str, dict] = {
+    f"{ex['name']}.mp4": ex for ex in EXAMPLES
+}
+
+
+def _prepare_examples() -> list[list]:
+    paths = get_example_video_paths(APP_DIR / "examples")
+    return [[str(p)] for p in paths if p.exists()]
+
+
+# ── Tab 1: Live Prediction ────────────────────────────────────────────────
+
+
+def classify_video(video_file: str | None, progress=gr.Progress()) -> tuple:
+    if not video_file:
+        return None, None, "_Pick an example or upload a video to begin._"
+
+    progress(0.1, desc="Extracting 8 frames from video...")
+    try:
+        frames = extract_frames_from_video(video_file, num_frames=config.NUM_FRAMES)
+    except Exception as e:
+        return None, None, f"**Error extracting frames:** {e}"
+
+    progress(0.5, desc="Running fine-tuned X-CLIP inference (may take 30-60s on CPU)...")
+    try:
+        result = predict_ours(frames)
+    except Exception as e:
+        return None, None, f"**Error during inference:** {e}"
+
+    gallery = [(frame, f"Frame {i+1}/8") for i, frame in enumerate(frames)]
+    label_dict = result["all_scores"]
+
+    filename = Path(video_file).name
+    gt_msg = ""
+    if filename in _EXAMPLE_BY_FILENAME:
+        ex = _EXAMPLE_BY_FILENAME[filename]
+        gt_display = CLASS_DISPLAY[ex["category"]]
+        pred = result["top3_labels"][0]
+        scenario = ex.get("scenario", "")
+        preds = ex.get("preds", {})
+        scenario_tag = {
+            "ALL_PASS": "\U0001f7e2 All 3 methods agree",
+            "OURS_ONLY": "\U0001f4a1 Only Ours gets this right",
+            "ALL_FAIL": "\U0001f534 Known failure: all 3 methods are wrong",
+        }.get(scenario, "")
+
+        ok_icon = "\u2705" if pred == gt_display else "\u274c"
+        header = f"### {ok_icon} Ground Truth: **{gt_display}**   Predicted: **{pred}**"
+        if scenario_tag:
+            header += f"\n\n{scenario_tag}"
+        if preds:
+            header += (
+                f"\n\n| Method | Prediction |\n|---|---|\n"
+                f"| CLIP zero-shot | {preds.get('clip', '?')} |\n"
+                f"| X-CLIP zero-shot | {preds.get('xclip', '?')} |\n"
+                f"| **Ours** | **{pred}** |\n"
+            )
+        header += f"\n\n_{ex['note']}_"
+        gt_msg = header
+    else:
+        gt_msg = f"### Predicted: **{result['top3_labels'][0]}**"
+
+    progress(1.0, desc="Done!")
+    return gallery, label_dict, gt_msg
+
+
+# ── Tab 2: Method Comparison ──────────────────────────────────────────────
+
+
+def compare_methods(video_file: str | None, progress=gr.Progress()) -> tuple:
+    if not video_file:
+        empty = {"(no video)": 1.0}
+        return None, empty, empty, empty, "Please upload or select a video first."
+
+    progress(0.05, desc="Extracting frames...")
+    try:
+        frames = extract_frames_from_video(video_file, num_frames=config.NUM_FRAMES)
+    except Exception as e:
+        empty = {"(error)": 1.0}
+        return None, empty, empty, empty, f"Error: {e}"
+
+    gallery = [(frame, f"Frame {i+1}/8") for i, frame in enumerate(frames)]
+
+    progress(0.20, desc="Running CLIP zero-shot...")
+    clip_result = predict_clip_zs(frames)
+
+    progress(0.50, desc="Running X-CLIP zero-shot...")
+    xclip_result = predict_xclip_zs(frames)
+
+    progress(0.80, desc="Running our fine-tuned model...")
+    ours_result = predict_ours(frames)
+
+    filename = Path(video_file).name
+    gt_msg = ""
+    if filename in _EXAMPLE_BY_FILENAME:
+        ex = _EXAMPLE_BY_FILENAME[filename]
+        gt = CLASS_DISPLAY[ex["category"]]
+        clip_ok = "\u2705" if clip_result["top3_labels"][0] == gt else "\u274c"
+        xclip_ok = "\u2705" if xclip_result["top3_labels"][0] == gt else "\u274c"
+        ours_ok = "\u2705" if ours_result["top3_labels"][0] == gt else "\u274c"
+        gt_msg = (
+            f"### Ground Truth: **{gt}**\n"
+            f"- CLIP zero-shot: {clip_ok} (predicted: {clip_result['top3_labels'][0]})\n"
+            f"- X-CLIP zero-shot: {xclip_ok} (predicted: {xclip_result['top3_labels'][0]})\n"
+            f"- **Ours:** {ours_ok} (predicted: {ours_result['top3_labels'][0]})\n\n"
+            f"_{ex['note']}_"
+        )
+
+    progress(1.0, desc="Done!")
+    return gallery, clip_result["all_scores"], xclip_result["all_scores"], ours_result["all_scores"], gt_msg
+
+
+# ── Tab 3: How It Works ──────────────────────────────────────────────────
+
+
+HOW_IT_WORKS_INTRO = """
+## The Idea
+
+Standard zero-shot classifiers ask a video-language model: *"Is this a video of robbery?"*
+The model has only the **word** to work with.
+
+We instead train it with rich, scene-level descriptions:
+- **Training (per video):** Each of 532 training videos gets a unique description from Gemini Flash.
+- **Inference (per class):** 5 hand-crafted descriptive prompts per class are averaged to form
+  the class embedding.
+
+This gives the text encoder much more information about *what each crime actually looks like*.
+
+## Pipeline
+
+```
+[Video] -> [Extract 8 frames] -> [X-CLIP visual encoder] -> [Video embedding]
+                                                                        |
+                                                                        v
+[LLM description] -> [X-CLIP text encoder] -> [Text embedding] -> [Cosine similarity] -> [Class]
+```
+
+## What's frozen vs. trained
+
+We freeze the first 9 of 12 transformer blocks in both encoders, training only:
+- The last 3 transformer blocks (visual + text)
+- X-CLIP's cross-frame communication module (CCT)
+- The multi-frame integration transformer (MIT)
+- Projection heads
+
+That's about **50M trainable parameters out of 195M** (~25%).
+"""
+
+
+SAMPLE_TRAINING_DESC = """**An example Gemini description** from the training set (Robbery):
+
+> _A dimly lit convenience store is shown. A person wearing dark clothing and a hood enters the frame holding what appears to be a firearm, pointing it at the cashier behind the counter. The cashier raises their hands in apparent compliance. The aggressor moves quickly around the counter area, and after several seconds of tense interaction leaves through the main door. The confrontation is brief but clearly threatening, and the victim's body language (raised hands, stiff posture) indicates they perceive a real threat._
+
+Every training video has its own unique paragraph like this one, generated by Gemini Flash over 8 sampled frames.
+"""
+
+
+def _format_sample_prompts() -> str:
+    prompts = PROMPTS["Robbery"]
+    out = "**Inference prompts for `Robbery`** (averaged into one class embedding):\n\n"
+    for i, p in enumerate(prompts, 1):
+        out += f"{i}. *{p}*\n"
+    return out
+
+
+# ── Tab 4: Results ────────────────────────────────────────────────────────
+
+
+RESULTS_INTRO = """
+## Results on UCF-Crime Action Recognition Test Set
+
+168 test videos, 14 classes, 12 samples per class.
+
+| Method | Accuracy | F1 (macro) | AUC-ROC |
+|---|---|---|---|
+| CLIP zero-shot | 23.2% | 21.0% | 0.77 |
+| X-CLIP zero-shot | 17.3% | 13.9% | 0.70 |
+| **Ours (X-CLIP + LLM descriptions)** | **26.2%** | **22.1%** | **0.80** |
+
+The 10-point AUC-ROC improvement (0.70 -> 0.80) over the base X-CLIP shows the model
+has learned meaningful structure even when its top-1 pick is wrong.
+"""
+
+
+def build_app() -> gr.Blocks:
+    examples_list = _prepare_examples()
+
+    with gr.Blocks(title="X-CLIP Crime Classifier") as app:
+        gr.Markdown(
+            "# Crime Detection with X-CLIP + LLM Descriptions\n\n"
+            "Multi-class crime classification on the **UCF-Crime** dataset, using "
+            "X-CLIP fine-tuned with per-video descriptions from Gemini Flash. "
+            "**Click any example below** to see it in action. "
+            "_(Running on free CPU tier — first prediction takes 30-60s while "
+            "the model downloads and loads.)_"
+        )
+
+        with gr.Tabs():
+            # Tab 1
+            with gr.Tab("\U0001f3ac Live Prediction"):
+                gr.Markdown(
+                    "Pick an example below (auto-classifies) or upload your own "
+                    "video and click **Classify**."
+                )
+                with gr.Row():
+                    with gr.Column(scale=1):
+                        video_input = gr.Video(label="Input Video", height=320)
+                        gr.Examples(
+                            examples=examples_list,
+                            inputs=[video_input],
+                            outputs=[],
+                            fn=lambda x: x,
+                            label="Example Videos (click to load and classify)",
+                            examples_per_page=8,
+                            run_on_click=False,
+                        )
+                        classify_btn = gr.Button("Classify", variant="primary", size="lg")
+                    with gr.Column(scale=1):
+                        gt_status = gr.Markdown("")
+                        confidence = gr.Label(
+                            num_top_classes=5,
+                            label="Top-5 Predictions (confidence)",
+                        )
+
+                gr.Markdown("### What the model sees: 8 uniformly-sampled frames")
+                frame_gallery = gr.Gallery(
+                    label="Extracted Frames", columns=4, rows=2,
+                    height=280, object_fit="contain",
+                )
+
+                classify_btn.click(
+                    fn=classify_video, inputs=[video_input],
+                    outputs=[frame_gallery, confidence, gt_status],
+                )
+                video_input.change(
+                    fn=classify_video, inputs=[video_input],
+                    outputs=[frame_gallery, confidence, gt_status],
+                )
+
+            # Tab 2
+            with gr.Tab("\u2696\ufe0f Method Comparison"):
+                gr.Markdown(
+                    "Run the same video through all three methods. "
+                    "Takes ~2-3 minutes on the free CPU tier because the three "
+                    "models must be loaded sequentially."
+                )
+                with gr.Row():
+                    with gr.Column(scale=1):
+                        comp_video_input = gr.Video(label="Input Video", height=280)
+                        gr.Examples(
+                            examples=examples_list,
+                            inputs=[comp_video_input],
+                            label="Example Videos",
+                            examples_per_page=8,
+                        )
+                        compare_btn = gr.Button("Run Comparison", variant="primary", size="lg")
+                    with gr.Column(scale=1):
+                        comp_gt = gr.Markdown("")
+
+                with gr.Row():
+                    with gr.Column():
+                        gr.Markdown("### CLIP zero-shot")
+                        clip_label = gr.Label(num_top_classes=5)
+                    with gr.Column():
+                        gr.Markdown("### X-CLIP zero-shot")
+                        xclip_label = gr.Label(num_top_classes=5)
+                    with gr.Column():
+                        gr.Markdown("### Ours (X-CLIP + LLM)")
+                        ours_label = gr.Label(num_top_classes=5)
+
+                gr.Markdown("### Frames")
+                comp_gallery = gr.Gallery(
+                    label="Extracted Frames", columns=4, rows=2,
+                    height=240, object_fit="contain",
+                )
+
+                compare_btn.click(
+                    fn=compare_methods, inputs=[comp_video_input],
+                    outputs=[comp_gallery, clip_label, xclip_label, ours_label, comp_gt],
+                )
+
+            # Tab 3
+            with gr.Tab("\U0001f4d6 How It Works"):
+                gr.Markdown(HOW_IT_WORKS_INTRO)
+                with gr.Row():
+                    with gr.Column():
+                        gr.Markdown("### Sample Training Description")
+                        gr.Markdown(SAMPLE_TRAINING_DESC)
+                    with gr.Column():
+                        gr.Markdown("### Sample Inference Prompts")
+                        gr.Markdown(_format_sample_prompts())
+
+            # Tab 4
+            with gr.Tab("\U0001f4ca Results"):
+                gr.Markdown(RESULTS_INTRO)
+                with gr.Row():
+                    with gr.Column():
+                        gr.Markdown("### Training Loss")
+                        if (PLOTS_DIR / "training_curve.png").exists():
+                            gr.Image(value=str(PLOTS_DIR / "training_curve.png"),
+                                     label="Loss converges from 0.72 to 0.0005",
+                                     show_label=True, height=280)
+                    with gr.Column():
+                        gr.Markdown("### Per-Class F1")
+                        if (PLOTS_DIR / "per_class_f1.png").exists():
+                            gr.Image(value=str(PLOTS_DIR / "per_class_f1.png"),
+                                     label="Three-way comparison",
+                                     show_label=True, height=280)
+                with gr.Row():
+                    with gr.Column():
+                        gr.Markdown("### Confusion Matrix (Ours)")
+                        if (PLOTS_DIR / "confusion_matrix.png").exists():
+                            gr.Image(value=str(PLOTS_DIR / "confusion_matrix.png"),
+                                     label="Robbery is consistently confused with Shoplifting",
+                                     show_label=True, height=420)
+                    with gr.Column():
+                        gr.Markdown("### Summary Table")
+                        if (PLOTS_DIR / "summary_table.png").exists():
+                            gr.Image(value=str(PLOTS_DIR / "summary_table.png"),
+                                     label="Method comparison",
+                                     show_label=True, height=420)
+
+        gr.Markdown(
+            "---\n"
+            "_Powered by [X-CLIP](https://huggingface.co/microsoft/xclip-base-patch16) "
+            "fine-tuned on UCF-Crime with descriptions from "
+            "[Gemini Flash](https://ai.google.dev/gemini-api). "
+            "Built with [Gradio](https://gradio.app)._"
+        )
+
+    return app
+
+
+if __name__ == "__main__":
+    app = build_app()
+    app.launch()

config_local.py

@@ -0,0 +1,32 @@
+"""Minimal config for the Hugging Face Space.
+
+A stripped-down version of the project's config.py — only the constants
+needed at inference time. Dataset paths are removed since the Space runs
+with the bundled example videos only.
+"""
+
+from pathlib import Path
+
+# ── Classes (14 total: 13 anomaly + Normal) ───────────────────────────────
+CLASSES = [
+    "Abuse", "Arrest", "Arson", "Assault", "Burglary",
+    "Explosion", "Fighting", "Normal_Videos_event",
+    "RoadAccidents", "Robbery", "Shooting",
+    "Shoplifting", "Stealing", "Vandalism",
+]
+NUM_CLASSES = len(CLASSES)
+
+# ── Inference ─────────────────────────────────────────────────────────────
+NUM_FRAMES = 8
+IMAGE_SIZE = 224
+NUM_PROMPTS_PER_CLASS = 5
+
+# ── Paths (relative to this file) ─────────────────────────────────────────
+APP_DIR = Path(__file__).resolve().parent
+EXAMPLES_DIR = APP_DIR / "examples"
+PLOTS_DIR = APP_DIR
+
+# Fine-tuned checkpoint is downloaded from the HF Model repo at startup.
+# Set HF_MODEL_REPO to the repo ID holding best_model.pt.
+HF_MODEL_REPO = "yashppawar/xclip-ucf-crime"
+HF_MODEL_FILENAME = "best_model.pt"

descriptive_prompts.py

@@ -0,0 +1,106 @@
+"""Descriptive prompts for each crime category (M=5 per class).
+
+Used at inference time: each prompt is encoded by the text encoder,
+averaged per category, then compared to video embeddings via cosine similarity.
+"""
+
+PROMPTS = {
+    "Abuse": [
+        "A person physically strikes, pushes, or forcefully handles another individual who appears defenseless or restrained in a confined space.",
+        "An aggressor repeatedly hits or kicks a victim who is on the ground or unable to fight back, showing a clear power imbalance.",
+        "One individual dominates another through physical force, with the victim showing signs of distress, pain, or submission.",
+        "A person violently grabs, shoves, or beats another person in what appears to be a domestic or interpersonal conflict.",
+        "An individual is being physically mistreated by another, with visible signs of coercion, restraint, or repeated violent contact.",
+    ],
+    "Arrest": [
+        "Law enforcement officers physically restrain a person, placing them in handcuffs or pinning them against a surface.",
+        "Police officers approach and detain a suspect, using physical force to bring them under control.",
+        "Multiple uniformed officers surround and apprehend an individual, often forcing them to the ground.",
+        "A person is being taken into custody by authorities, with visible restraining actions like handcuffing or arm-locking.",
+        "Officers chase and tackle a suspect, then proceed to restrain and escort them to a vehicle.",
+    ],
+    "Arson": [
+        "A person deliberately sets fire to a building, vehicle, or property, with flames visibly spreading from the ignition point.",
+        "An individual uses an accelerant or lighter to ignite materials, causing a fire that grows rapidly in the scene.",
+        "Flames erupt from a structure or object after a person is seen placing or throwing an incendiary device.",
+        "A deliberate fire is started by an individual, with smoke and flames engulfing the targeted property.",
+        "Someone intentionally lights a fire that spreads to nearby structures, with the perpetrator fleeing the scene.",
+    ],
+    "Assault": [
+        "Two or more individuals engage in a one-sided physical attack where one person overpowers another with punches, kicks, or weapons.",
+        "An aggressor attacks a victim unprovoked, striking them repeatedly while the victim attempts to defend or escape.",
+        "A person is violently attacked by one or more assailants in a public space, with clear aggressive intent.",
+        "An individual uses a weapon or physical force to harm another person who is caught off guard or unable to resist.",
+        "A sudden violent confrontation where one party aggressively attacks another, causing visible injury or distress.",
+    ],
+    "Burglary": [
+        "A person breaks into a building or property by forcing open doors, windows, or locks, often under cover of darkness.",
+        "An individual enters a closed establishment through unauthorized means, such as climbing through a window or picking a lock.",
+        "A person is seen sneaking into a building, searching through rooms, and taking items while trying to avoid detection.",
+        "Someone forces entry into a premises when it appears unoccupied, systematically searching for valuables.",
+        "An intruder breaks into a home or business, moving cautiously through the space while gathering stolen goods.",
+    ],
+    "Explosion": [
+        "A sudden violent blast occurs, sending debris, smoke, and shockwaves through the environment, damaging surrounding structures.",
+        "A detonation creates a large fireball and plume of smoke, with visible destruction to vehicles or buildings nearby.",
+        "An explosive device goes off in a public or commercial area, causing immediate destruction and chaos.",
+        "A powerful blast rips through a location, with the explosion captured as a bright flash followed by expanding smoke and debris.",
+        "An explosion destroys part of a structure, with fire, flying debris, and a visible shockwave emanating from the blast point.",
+    ],
+    "Fighting": [
+        "Two or more individuals engage in mutual physical combat, exchanging punches, kicks, and grappling with each other.",
+        "A group of people are involved in a brawl, with multiple participants throwing punches and wrestling on the ground.",
+        "Two people fight aggressively in a public space, with both parties actively attacking and defending.",
+        "A violent altercation between multiple individuals, with mutual aggression and no clear victim or aggressor.",
+        "People engage in a physical confrontation, shoving, hitting, and tackling each other in a chaotic scene.",
+    ],
+    "Normal_Videos_event": [
+        "People walk, stand, or sit in a public space, going about routine daily activities with no unusual behavior.",
+        "Normal pedestrian and vehicle traffic flows through a street or parking lot with no signs of disturbance.",
+        "Individuals engage in everyday activities such as shopping, walking, or socializing in a calm environment.",
+        "A typical surveillance scene showing regular human movement and interactions without any anomalous events.",
+        "People move naturally through a space, with standard patterns of walking, entering, and exiting buildings.",
+    ],
+    "RoadAccidents": [
+        "Vehicles collide at an intersection or on a road, with visible impact damage and debris scattered across the roadway.",
+        "A car crashes into another vehicle or object, causing visible damage, spinning, or flipping of the vehicles involved.",
+        "A traffic accident occurs where vehicles suddenly collide, often with one running a signal or losing control.",
+        "Multiple vehicles are involved in a collision, with the impact causing cars to spin, crumple, or come to abrupt stops.",
+        "A vehicle loses control and crashes, with visible signs of impact such as shattered glass, deformed metal, and skid marks.",
+    ],
+    "Robbery": [
+        "A person threatens another with a weapon and forcibly takes their belongings in a store or street.",
+        "An armed individual demands valuables from a victim, using intimidation and the threat of violence.",
+        "A person confronts a store clerk or pedestrian with a weapon, stealing cash or property through force.",
+        "An individual uses clear threats and force to obtain personal belongings from a frightened victim.",
+        "A robbery occurs where the perpetrator uses a weapon to threaten the victim while stealing their possessions.",
+    ],
+    "Shooting": [
+        "A person fires a gun at another individual or into a crowd, with visible muzzle flash or victims falling.",
+        "An armed individual discharges a firearm in a public space, causing people to flee in panic.",
+        "A shooting incident where a person aims and fires a weapon, with visible recoil and people reacting in fear.",
+        "Gunshots are fired in a confrontation, with individuals seen drawing weapons and shooting at targets.",
+        "A violent shooting occurs where one or more individuals fire weapons, causing casualties or widespread panic.",
+    ],
+    "Shoplifting": [
+        "A person discreetly conceals merchandise on their person while inside a retail store, looking around nervously.",
+        "An individual picks up store items and hides them in their bag or clothing, attempting to leave without paying.",
+        "A person in a retail environment secretly takes products off shelves and conceals them before heading toward the exit.",
+        "Someone stealthily pockets small items from store displays while pretending to browse or shop normally.",
+        "A shoplifter carefully selects items and conceals them in their clothing or bag while avoiding store staff.",
+    ],
+    "Stealing": [
+        "A person takes property that does not belong to them from an outdoor or public setting, such as a parked vehicle or yard.",
+        "An individual approaches an unattended item or vehicle and takes it, quickly leaving the area.",
+        "Someone steals objects from outside a building, car, or public space, acting quickly to avoid being caught.",
+        "A thief takes items from an unattended area, such as bicycles, packages, or belongings left outside.",
+        "A person sneaks up to and takes property from a public or semi-public area, departing swiftly after the theft.",
+    ],
+    "Vandalism": [
+        "A person deliberately damages property by smashing windows, spray-painting walls, or breaking objects in public.",
+        "An individual uses tools or physical force to destroy or deface property, such as cars, buildings, or street furniture.",
+        "Someone intentionally causes damage to public or private property through acts like breaking, smashing, or graffiti.",
+        "A person vandalizes a vehicle or building, kicking, striking, or throwing objects to cause visible damage.",
+        "An individual engages in deliberate destruction of property, leaving visible signs of damage like broken glass or paint.",
+    ],
+}

example_videos.py

@@ -0,0 +1,119 @@
+"""Curated example videos for the Hugging Face Space.
+
+Same list as the local demo. Video files live under `./examples/` (bundled
+with the Space repo).
+"""
+
+from pathlib import Path
+
+EXAMPLES = [
+    {
+        "name": "Shoplifting001_x264",
+        "category": "Shoplifting",
+        "scenario": "ALL_PASS",
+        "display_label": "[All pass] Shoplifting — easy case",
+        "preds": {"clip": "Shoplifting", "xclip": "Shoplifting", "ours": "Shoplifting"},
+        "note": (
+            "All three methods identify the shoplifting correctly. Concealment "
+            "in a retail setting is a visually distinctive pattern that even "
+            "zero-shot models catch."
+        ),
+    },
+    {
+        "name": "Explosion001_x264",
+        "category": "Explosion",
+        "scenario": "ALL_PASS",
+        "display_label": "[All pass] Explosion — bright visual signature",
+        "preds": {"clip": "Explosion", "xclip": "Explosion", "ours": "Explosion"},
+        "note": (
+            "Fire, smoke, and shockwaves are highly distinctive. All three "
+            "methods classify this correctly with high confidence."
+        ),
+    },
+    {
+        "name": "Fighting004_x264",
+        "category": "Fighting",
+        "scenario": "OURS_ONLY",
+        "display_label": "[Ours only] Fighting — baselines guess Shooting / Arrest",
+        "preds": {"clip": "Shooting", "xclip": "Arrest", "ours": "Fighting"},
+        "note": (
+            "Baselines are wildly off (CLIP: Shooting, X-CLIP: Arrest), while "
+            "our fine-tuned model correctly identifies the physical altercation. "
+            "This is exactly the kind of visual ambiguity that fine-tuning helps "
+            "resolve."
+        ),
+    },
+    {
+        "name": "Stealing002_x264",
+        "category": "Stealing",
+        "scenario": "OURS_ONLY",
+        "display_label": "[Ours only] Stealing — baselines score 0% F1 on this class",
+        "preds": {"clip": "Normal", "xclip": "Road Accident", "ours": "Stealing"},
+        "note": (
+            "The Stealing class has 0.0% F1 for both CLIP and X-CLIP zero-shot. "
+            "Our fine-tuned model learned to recognise outdoor theft from 38 "
+            "training examples — a clear demonstration of the paper's central "
+            "claim."
+        ),
+    },
+    {
+        "name": "Arson003_x264",
+        "category": "Arson",
+        "scenario": "OURS_ONLY",
+        "display_label": "[Ours only] Arson — baselines confuse fire cues",
+        "preds": {"clip": "Explosion", "xclip": "Road Accident", "ours": "Arson"},
+        "note": (
+            "CLIP mistakes deliberate fire-setting for an Explosion; X-CLIP "
+            "guesses Road Accident. The LLM descriptions taught our model to "
+            "distinguish the temporal build-up of arson from an instantaneous "
+            "blast."
+        ),
+    },
+    {
+        "name": "Normal_Videos_150_x264",
+        "category": "Normal_Videos_event",
+        "scenario": "OURS_ONLY",
+        "display_label": "[Ours only] Normal — baselines over-fire on benign scenes",
+        "preds": {"clip": "Shoplifting", "xclip": "Arrest", "ours": "Normal"},
+        "note": (
+            "A calm surveillance scene. Both baselines hallucinate a crime "
+            "(Shoplifting / Arrest). Our model correctly says Normal — a "
+            "critical property for any real deployment, since false alarms on "
+            "benign footage would make the system unusable."
+        ),
+    },
+    {
+        "name": "Robbery005_x264",
+        "category": "Robbery",
+        "scenario": "ALL_FAIL",
+        "display_label": "[All fail] Robbery — famous confusion with Shoplifting",
+        "preds": {"clip": "Shooting", "xclip": "Shoplifting", "ours": "Shoplifting"},
+        "note": (
+            "The paper's most visible failure: Robbery is systematically "
+            "misclassified as Shoplifting (11/12 in our confusion matrix). "
+            "The visual setting — retail environment, someone taking goods — "
+            "overlaps too much with shoplifting for any of the three methods "
+            "to distinguish the two without weapon/threat cues."
+        ),
+    },
+    {
+        "name": "Abuse005_x264",
+        "category": "Abuse",
+        "scenario": "ALL_FAIL",
+        "display_label": "[All fail] Abuse — interpersonal violence is hard",
+        "preds": {"clip": "Arrest", "xclip": "Road Accident", "ours": "Shooting"},
+        "note": (
+            "Abuse scored 0.0% F1 across the test set. The distinguishing cue "
+            "(one-sided power dynamic between people) is extremely subtle in "
+            "low-resolution surveillance footage, so all three methods guess "
+            "something else entirely."
+        ),
+    },
+]
+
+
+def get_example_video_paths(examples_dir: Path | None = None) -> list[Path]:
+    """Return the list of mp4 paths for all curated examples."""
+    if examples_dir is None:
+        examples_dir = Path(__file__).resolve().parent / "examples"
+    return [examples_dir / f"{ex['name']}.mp4" for ex in EXAMPLES]

examples/Abuse005_x264.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:425744aa3472e424d52d7ce97bf6d0bdd445ad62ad1be110095d2027a31550cc
+size 6250495

examples/Arson003_x264.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6a17a5ae6d7a01eb84eb0afa976d8cb493ac52051ca8301d4428e53af91b7e7e
+size 24229193

examples/Explosion001_x264.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4d9dddadf4f102e5f017e098f345b30cacc499058f0f69ef1f2787618721bf4
+size 1187504

examples/Fighting004_x264.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e93bb27e96279ff1a740d96f4e4bc5de5a5fe324deffa435e2dd58fd285f9916
+size 121757725

examples/Normal_Videos_150_x264.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9241161d31599a70264edc2dff747030a3a25d4b7a296a7809e1cad28d026aa4
+size 7315607

examples/Robbery005_x264.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dfea68e67bf1a15d4dcba919e8dfa462189c6dacb047b7445bfeaebbaea22c03
+size 15221600

examples/Shoplifting001_x264.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7af64fd7e96f81da3f91c9cd4bf531dc2d5968718cdb646822ea9895d71ff060
+size 35944805

examples/Stealing002_x264.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:88acfce0d4442084f313e9a6537d93b49567a83f5c1756ccaf5ca21d9d6d455b
+size 15233758

inference.py

@@ -0,0 +1,283 @@
+"""Inference utilities for the Hugging Face Space.
+
+Adapted from the local demo — downloads the fine-tuned checkpoint from a
+HF Model repo at first use, and manages VRAM/RAM by lazy-loading one of
+three models at a time.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Optional
+
+import numpy as np
+import torch
+import torch.nn.functional as F
+from PIL import Image
+
+import config_local as config
+from descriptive_prompts import PROMPTS
+
+# ── Display names ─────────────────────────────────────────────────────────
+CLASS_DISPLAY = {
+    "Abuse": "Abuse",
+    "Arrest": "Arrest",
+    "Arson": "Arson",
+    "Assault": "Assault",
+    "Burglary": "Burglary",
+    "Explosion": "Explosion",
+    "Fighting": "Fighting",
+    "Normal_Videos_event": "Normal",
+    "RoadAccidents": "Road Accident",
+    "Robbery": "Robbery",
+    "Shooting": "Shooting",
+    "Shoplifting": "Shoplifting",
+    "Stealing": "Stealing",
+    "Vandalism": "Vandalism",
+}
+
+_CLASS_PROMPT_SHORT = {
+    "Abuse": "abuse", "Arrest": "arrest", "Arson": "arson", "Assault": "assault",
+    "Burglary": "burglary", "Explosion": "explosion", "Fighting": "fighting",
+    "Normal_Videos_event": "normal activity", "RoadAccidents": "road accident",
+    "Robbery": "robbery", "Shooting": "shooting", "Shoplifting": "shoplifting",
+    "Stealing": "stealing", "Vandalism": "vandalism",
+}
+
+# ── Module state ──────────────────────────────────────────────────────────
+_device: Optional[torch.device] = None
+_xclip_ft = None
+_xclip_zs = None
+_clip = None
+_current_model: Optional[str] = None
+_checkpoint_path: Optional[Path] = None
+
+
+def _get_device() -> torch.device:
+    global _device
+    if _device is None:
+        _device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    return _device
+
+
+def _get_checkpoint_path() -> Path:
+    """Download the fine-tuned checkpoint from the HF Model repo on first call."""
+    global _checkpoint_path
+    if _checkpoint_path is not None and _checkpoint_path.exists():
+        return _checkpoint_path
+
+    from huggingface_hub import hf_hub_download
+    print(f"[inference] Downloading checkpoint from {config.HF_MODEL_REPO}...")
+    p = hf_hub_download(
+        repo_id=config.HF_MODEL_REPO,
+        filename=config.HF_MODEL_FILENAME,
+        repo_type="model",
+    )
+    _checkpoint_path = Path(p)
+    print(f"[inference] Checkpoint cached at {_checkpoint_path}")
+    return _checkpoint_path
+
+
+def _pad_frames(frames: list[Image.Image], num_frames: int) -> list[Image.Image]:
+    if len(frames) == 0:
+        raise ValueError("No frames provided")
+    frames = list(frames[:num_frames])
+    while len(frames) < num_frames:
+        frames.append(frames[-1])
+    return frames
+
+
+def _unload(name: str) -> None:
+    global _xclip_ft, _xclip_zs, _clip
+    if name == "ft":
+        _xclip_ft = None
+    elif name == "xclip_zs":
+        _xclip_zs = None
+    elif name == "clip":
+        _clip = None
+    if torch.cuda.is_available():
+        torch.cuda.empty_cache()
+
+
+def _switch_to(target: str) -> None:
+    global _current_model
+    if _current_model is not None and _current_model != target:
+        _unload(_current_model)
+        _current_model = None
+
+
+def unload_all_models() -> None:
+    global _xclip_ft, _xclip_zs, _clip, _current_model
+    _xclip_ft = None
+    _xclip_zs = None
+    _clip = None
+    _current_model = None
+    if torch.cuda.is_available():
+        torch.cuda.empty_cache()
+
+
+# ── Frame extraction ──────────────────────────────────────────────────────
+
+
+def extract_frames_from_video(video_path: str, num_frames: int = 8) -> list[Image.Image]:
+    import decord
+    decord.bridge.set_bridge("native")
+    vr = decord.VideoReader(str(video_path), num_threads=1)
+    total = len(vr)
+    if total == 0:
+        return []
+    indices = np.linspace(0, total - 1, num_frames, dtype=int)
+    frames_np = vr.get_batch(indices).asnumpy()
+    return [Image.fromarray(frames_np[i]) for i in range(len(frames_np))]
+
+
+# ── Model loaders ─────────────────────────────────────────────────────────
+
+
+def _load_xclip_finetuned():
+    from transformers import XCLIPModel, XCLIPProcessor
+
+    device = _get_device()
+    processor = XCLIPProcessor.from_pretrained("microsoft/xclip-base-patch16")
+    model = XCLIPModel.from_pretrained("microsoft/xclip-base-patch16")
+
+    ckpt_path = _get_checkpoint_path()
+    cp = torch.load(ckpt_path, map_location=device, weights_only=True)
+    model.load_state_dict(cp["model_state_dict"])
+    model = model.to(device).eval()
+
+    text_embeds = _compute_descriptive_text_embeds_xclip(model, processor, device)
+    return model, processor, text_embeds
+
+
+def _load_xclip_zero_shot():
+    from transformers import XCLIPModel, XCLIPProcessor
+
+    device = _get_device()
+    processor = XCLIPProcessor.from_pretrained("microsoft/xclip-base-patch16")
+    model = XCLIPModel.from_pretrained("microsoft/xclip-base-patch16").to(device).eval()
+
+    texts = [f"a surveillance video of {_CLASS_PROMPT_SHORT[c]}" for c in config.CLASSES]
+    all_embeds = []
+    with torch.inference_mode():
+        for t in texts:
+            ti = processor.tokenizer(
+                t, return_tensors="pt", padding="max_length",
+                truncation=True, max_length=77,
+            ).to(device)
+            te = model.get_text_features(**ti).pooler_output
+            te = F.normalize(te, dim=-1)
+            all_embeds.append(te)
+    text_embeds = torch.cat(all_embeds, dim=0)
+    return model, processor, text_embeds
+
+
+def _load_clip_zero_shot():
+    from transformers import CLIPModel, CLIPProcessor
+
+    device = _get_device()
+    processor = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch16")
+    model = CLIPModel.from_pretrained("openai/clip-vit-base-patch16").to(device).eval()
+
+    texts = [f"a surveillance video of {_CLASS_PROMPT_SHORT[c]}" for c in config.CLASSES]
+    with torch.inference_mode():
+        text_inputs = processor(text=texts, return_tensors="pt", padding=True).to(device)
+        text_embeds = model.get_text_features(**text_inputs).pooler_output
+        text_embeds = F.normalize(text_embeds, dim=-1)
+    return model, processor, text_embeds
+
+
+def _compute_descriptive_text_embeds_xclip(model, processor, device):
+    all_embeds = []
+    with torch.inference_mode():
+        for cls in config.CLASSES:
+            prompts = PROMPTS[cls]
+            cls_embeds = []
+            for p in prompts:
+                ti = processor.tokenizer(
+                    p, return_tensors="pt", padding="max_length",
+                    truncation=True, max_length=77,
+                ).to(device)
+                te = model.get_text_features(**ti).pooler_output
+                te = F.normalize(te, dim=-1)
+                cls_embeds.append(te)
+            avg = F.normalize(
+                torch.mean(torch.cat(cls_embeds), dim=0, keepdim=True), dim=-1
+            )
+            all_embeds.append(avg)
+    return torch.cat(all_embeds, dim=0)
+
+
+# ── Shared result formatter ───────────────────────────────────────────────
+
+
+def _format_result(sims_row: torch.Tensor) -> dict:
+    probs = F.softmax(sims_row, dim=-1).detach().cpu().numpy()
+    order = np.argsort(-probs)
+    top3_idx = order[:3]
+    top3_labels = [CLASS_DISPLAY[config.CLASSES[i]] for i in top3_idx]
+    top3_scores = [float(probs[i]) for i in top3_idx]
+    all_scores = {
+        CLASS_DISPLAY[config.CLASSES[i]]: float(probs[i])
+        for i in range(len(config.CLASSES))
+    }
+    return {"top3_labels": top3_labels, "top3_scores": top3_scores, "all_scores": all_scores}
+
+
+# ── Prediction entry points ───────────────────────────────────────────────
+
+
+def predict_ours(frames: list[Image.Image]) -> dict:
+    global _xclip_ft, _current_model
+    _switch_to("ft")
+    if _xclip_ft is None:
+        _xclip_ft = _load_xclip_finetuned()
+        _current_model = "ft"
+    model, processor, text_embeds = _xclip_ft
+    device = _get_device()
+    frames = _pad_frames(frames, config.NUM_FRAMES)
+    inputs = processor(images=frames, return_tensors="pt")
+    pixel_values = inputs["pixel_values"].to(device)
+    with torch.inference_mode():
+        vid_embeds = model.get_video_features(pixel_values=pixel_values).pooler_output
+        vid_embeds = F.normalize(vid_embeds, dim=-1)
+        sims = vid_embeds @ text_embeds.T
+    return _format_result(sims[0])
+
+
+def predict_xclip_zs(frames: list[Image.Image]) -> dict:
+    global _xclip_zs, _current_model
+    _switch_to("xclip_zs")
+    if _xclip_zs is None:
+        _xclip_zs = _load_xclip_zero_shot()
+        _current_model = "xclip_zs"
+    model, processor, text_embeds = _xclip_zs
+    device = _get_device()
+    frames = _pad_frames(frames, config.NUM_FRAMES)
+    inputs = processor(images=frames, return_tensors="pt")
+    pixel_values = inputs["pixel_values"].to(device)
+    with torch.inference_mode():
+        vid_embeds = model.get_video_features(pixel_values=pixel_values).pooler_output
+        vid_embeds = F.normalize(vid_embeds, dim=-1)
+        sims = vid_embeds @ text_embeds.T
+    return _format_result(sims[0])
+
+
+def predict_clip_zs(frames: list[Image.Image]) -> dict:
+    global _clip, _current_model
+    _switch_to("clip")
+    if _clip is None:
+        _clip = _load_clip_zero_shot()
+        _current_model = "clip"
+    model, processor, text_embeds = _clip
+    device = _get_device()
+    if len(frames) == 0:
+        raise ValueError("No frames provided")
+    mid = frames[len(frames) // 2]
+    inputs = processor(images=mid, return_tensors="pt")
+    pixel_values = inputs["pixel_values"].to(device)
+    with torch.inference_mode():
+        img_embeds = model.get_image_features(pixel_values=pixel_values).pooler_output
+        img_embeds = F.normalize(img_embeds, dim=-1)
+        sims = img_embeds @ text_embeds.T
+    return _format_result(sims[0])

requirements.txt

@@ -0,0 +1,7 @@
+gradio>=6.12.0
+torch>=2.1
+transformers>=4.40
+decord>=0.6.0
+pillow>=10.0
+numpy>=1.24
+huggingface_hub>=0.24