final demo #1

app.py

@@ -1,23 +1,14 @@
 import gradio as gr
 import os
 os.environ["KMP_DUPLICATE_LIB_OK"]="TRUE"
-
-from huggingface_hub import hf_hub_download
-# ==========================================
-# 1. DOWNLOAD THE LARGE FILE ON STARTUP
-# ==========================================
-print("Checking for large model file...")
-
-# UPDATE THESE TWO LINES WITH YOUR EXACT DETAILS
-model_path = hf_hub_download(
-    repo_id="plice13/sign-language-weights", 
-    filename="best_checkpoint.pth"                  
-)
-print(f"File successfully loaded at: {model_path}")
-# ==========================================
+from backend import process_input
 
 def process_video(input_video_path):
-   return "DONE", None
+    # Generate a translation in the backend.
+    translation = process_input(input_video_path)
+    
+    # Return just the translation if no visualization is found
+    return translation, None
 
 # Custom CSS for the dark green background and centered layout
 custom_css = """

backend.py

@@ -0,0 +1,184 @@
+import os
+import time
+import torch
+import numpy as np
+import matplotlib.pyplot as plt
+from dotenv import load_dotenv
+
+# Import tvého nového modelu a pomocných funkcí
+from Uni_Sign.models import Uni_Sign
+from Uni_Sign.datasets import load_part_kp_YTASL, YTASL_GROUP_SIZES, _fill_missing_landmarks, select_frame_indices
+from predict_pose import create_mediapipe_models, predict_pose, load_video_cv
+
+os.environ['KMP_DUPLICATE_LIB_OK'] = 'TRUE'
+load_dotenv()
+
+# Globální proměnné pro modely, ať se nenačítají při každém videu znovu
+model = None
+pose_models = None
+args = None
+
+class InferenceConfig:
+    """Čistá konfigurace pro Uni_Sign inferenci"""
+    def __init__(self):
+        # Nezapomeň upravit cestu ke svým natrénovaným váhám
+        self.finetune = r"./Uni_Sign/unisign_model/best_checkpoint.pth" 
+        self.dataset = "YTASL"
+        self.task = "SLT"
+        self.max_length = 256
+        self.normalization = "none"
+        self.layout = "pruned"
+        self.n_registers = 0
+        self.hidden_dim = 256
+        self.rgb_support = False
+        self.no_adaptive_gcn = False
+        self.register_position = "before_all"
+        self.label_smoothing = 0.2
+        self.batch_size = 1
+
+def initialize_model():
+    """Inicializuje Uni_Sign a Mediapipe/YOLO modely pro extrakci pose."""
+    global model, pose_models, args
+    
+    if model is not None and pose_models is not None:
+        return
+    
+    print("Inicializuji Uni_Sign model...")
+    args = InferenceConfig()
+    model = Uni_Sign(args=args)
+    
+    # Načtení vah
+    if args.finetune and os.path.exists(args.finetune):
+        print(f"Načítám váhy z: {args.finetune}")
+        state_dict = torch.load(args.finetune, map_location='cpu')['model']
+        model.load_state_dict(state_dict, strict=False)
+    else:
+        print(f"VAROVÁNÍ: Checkpoint '{args.finetune}' nebyl nalezen!")
+    
+    device = "cuda" if torch.cuda.is_available() else "cpu"
+    model.to(device)
+    model.eval()
+
+    print("Inicializuji Mediapipe a YOLO modely...")
+    pose_checkpoint_folder = 'checkpoints/pose/'
+    pose_models = create_mediapipe_models(pose_checkpoint_folder)
+
+def process_pose_data_in_memory(pose_results, args):
+    """
+    Nahrazuje starý 'process_single_json'.
+    Zpracovává 'cropped_keypoints' přímo ze slovníku (v paměti).
+    """
+    raw_pose = pose_results.get('cropped_keypoints', [])
+    if not raw_pose:
+        raise ValueError("Video neobsahuje žádné detekované pose (cropped_keypoints).")
+
+    # NOVÝ KÓD: Převod numpy polí na čisté [X, Y] seznamy a ošetření chybějících částí těla
+    pose = []
+    for frame_data in raw_pose:
+        formatted_frame = {}
+        for part, expected_size in YTASL_GROUP_SIZES.items():
+            kps = frame_data.get(part, [])
+            
+            # Pokud část chybí (prázdný list/array), pošleme prázdno, ať si s tím poradí _fill_missing_landmarks
+            # Pokud část chybí (prázdný list/array), pošleme prázdno
+            # Pokud část chybí (prázdný list/array), pošleme prázdno
+            # Bezpečná kontrola Numpy pole:
+            if kps is None or len(kps) == 0:
+                 formatted_frame[part] = [] 
+            else:
+                 # Neprůstřelné řešení: převedeme data (ať už jsou cokoliv) na Numpy pole, 
+                 # ořízneme první dva sloupce a převedeme zpět na čistý list.
+                 formatted_frame[part] = np.array(kps)[:, :2].tolist()
+        pose.append(formatted_frame)
+
+    # ... Zbytek funkce pokračuje beze změny:
+
+    duration = len(pose)
+    tmp = select_frame_indices(duration, args.max_length, phase='test')
+    skeletons = [pose[i] for i in tmp]
+
+    confs = []
+    for i, skeleton in enumerate(skeletons):
+        conf = {}
+        for group_name, expected_size in YTASL_GROUP_SIZES.items():
+            _fill_missing_landmarks(
+                skeleton=skeleton,
+                conf=conf,
+                group_name=group_name,
+                expected_size=expected_size,
+                clip_name="gradio_video",
+                frame_idx=i,
+                error_group_label=f"group '{group_name}'",
+                include_size_details=False,
+                strict_key_access=True,
+            )
+        confs.append(conf)
+
+    kps_with_scores = load_part_kp_YTASL(skeletons, confs, args.normalization, args.layout)
+
+    src_input = {}
+    for key, val in kps_with_scores.items():
+        src_input[key] = val.unsqueeze(0)
+
+    seq_len = src_input['body'].shape[1]
+    mask_gen = torch.ones([seq_len]) + 7
+    src_input['attention_mask'] = (mask_gen != 0).long().unsqueeze(0)
+    src_input['src_length_batch'] = torch.LongTensor([seq_len])
+    src_input['name_batch'] = ["gradio_video"]
+
+    return src_input
+
+
+def process_input(input_video_path):
+    """
+    Hlavní vstupní bod pro Gradio. Spustí extrakci dat, vizualizaci 
+    a inference přes Uni_Sign.
+    """
+    try:
+        initialize_model()
+        
+        print("1. Extrahuji keypoints z videa...")
+        video_frames, _ = load_video_cv(input_video_path)
+        pose_results = predict_pose(video_frames, pose_models)
+        
+        print("2. Prostor pro vizualizaci...")
+
+        print("3. Přenáším data do Tenzorů pro Uni_Sign...")
+        src_input = process_pose_data_in_memory(pose_results, args)
+        
+        # Přesunutí Tenzorů na stejné zařízení (GPU/CPU) jako model
+        device = next(model.parameters()).device
+        for key in ['body', 'left', 'right', 'face_all', 'attention_mask']:
+            if key in src_input:
+                # Ošetření typu dat na float
+                if key != 'attention_mask':
+                    src_input[key] = src_input[key].to(device, dtype=torch.float32)
+                else:
+                    src_input[key] = src_input[key].to(device)
+                    
+        tgt_input = {'gt_sentence': [""], 'gt_gloss': [""]}
+        
+        print("4. Generuji překlad...")
+        with torch.no_grad():
+            stack_out = model(src_input, tgt_input)
+            output = model.generate(
+                stack_out,
+                max_new_tokens=100,
+                num_beams=4,
+            )
+            
+        tokenizer = model.mt5_tokenizer
+        tgt_pres = tokenizer.batch_decode(output, skip_special_tokens=True)
+        
+        result = tgt_pres[0].strip()
+        if not result:
+            return "Model nedokázal generovat text. Zkus jiné video."
+        
+        print("5. VŠE HOTOVO!")            
+        return result
+            
+    except Exception as e:
+        print(f"Chyba při zpracování: {e}")
+        import traceback
+        traceback.print_exc()
+        return f"Chyba při zpracování videa: {str(e)}"

configs/README.md

@@ -0,0 +1,70 @@
+## Training config
+
+```
+ModelArguments:
+  base_model_name: Name of the model to be downloaded from HF
+  sign_input_dim (depricated, it is calculated automatically now): Dimmension of sign features on the model input (to be changed with multimodal input)
+  hidden_dropout_prob: Dropout probability of the input layer
+  max_length: Max sequence length to be generated by the model
+
+# TrainingArguments are overwritten by not-None arguments
+TrainingArguments:
+  project_name: Name of wandb project
+  model_name: Name of the run name (will affect wandb and checkpoint path)
+  output_dir: Output path for checkpoints
+  resume_from_checkpoint: Path to checkpoint to be loaded. The path needs to contain model.safetensors file
+  load_only_weights: True for loading only weights to use different scheduler or optimizer. False for loading the whole checkpoint
+#  Logging and saving
+  report_to: "wandb" for wandb report. Otherwise wandb not used
+#  Debugging
+  max_train_samples: Maximum size of training dataset for debugging. "none" for default size
+  max_val_samples: Maximum size of validation dataset for debugging. "none" for default size
+#  Data processing
+  max_sequence_length: Max input sequence length (decoder context window) - data sequence is cropped by this length
+  max_token_length: Dataloader tokenizer max_length
+  skip_frames: Use only each n-th frame. "True" for each 2nd frame 
+
+SignDataArguments:
+  data_dir: Prefix of the path to data including annotation file and metadatafile. This path is joined by paths bellow.
+  annotation_path: Path to train/dev annotation file
+  visual_features: Path to train/dev metadata file
+    pose:
+      enable_input: Whether to use this modality. Our code supports only "pose" so far
+      train: Path to train metafile
+      dev: Path to validation metafile
+
+SignModelArguments: Dimmensions per each sign-features modality
+```
+
+
+## Predict config
+
+```
+ModelArguments:
+  base_model_name: Name of the model to be downloaded from HF
+  sign_input_dim: Dimmension of sign features on the model input (to be changed with multimodal input)
+  max_length: Max sequence length to be generated by the model
+
+EvaluationArguments:
+  output_dir: Output path for predictions
+  model_name: Only for logging purposes
+  skip_frames: Use only each n-th frame. "True" for each 2nd frame 
+#  Data processing
+  split: test 
+  max_sequence_length: Max input sequence length (decoder context window) - data sequence is cropped by this length
+  max_token_length: Dataloader tokenizer max_length
+#  Generation parameters
+  model_dir: Path to checkpoint to be evaluated. The path needs to contain model.safetensors file
+#  Debugging
+  max_val_samples: Maximum size of validation dataset for debugging. "none" for default size
+
+SignDataArguments:
+  data_dir: Prefix of the path to data including annotation file and metadatafile. This path is joined by paths bellow.
+  annotation_path: Path to test annotation file
+  visual_features: Path to test metadata file
+    pose:
+      enable_input: Whether to use this modality. Our code supports only "pose" so far
+      test: Path to test metafile
+
+SignModelArguments: Dimmensions per each sign-features modality
+```

configs/predict_config_demo.yaml

@@ -0,0 +1,58 @@
+ModelArguments:
+  base_model_name: google/t5-v1_1-base
+  hidden_dropout_prob: 0.1
+  num_beams: 5
+  max_length: 128
+  top_k: 50
+  top_p: 0.9
+  temperature: 1.0
+  length_penalty: 1.0
+  repetition_penalty: 1.0
+  early_stopping: False
+  no_repeat_ngram_size: 3
+  do_sample: False
+
+EvaluationArguments:
+  output_dir: ./results
+  model_name: T5_YTASL_pretrain
+  skip_frames: False
+#  Data processing
+  split: test
+  max_sequence_length: 250
+  max_token_length: 128
+#  Generation parameters
+  model_dir: ./checkpoints/t5-v1_1-base/model.safetensors
+  batch_size: 1
+#  Debugging
+  max_val_samples: none
+
+SignDataArguments:
+  data_dir: ./data
+  annotation_path:
+    train: YT.annotations.train.json
+    dev: YT.annotations.dev.json
+    test: YT.annotations.dev.json
+  visual_features:
+    sign2vec:
+      enable_input: False
+      test: sign2vec/metadata_sign2vec.dev.json
+    mae:
+      enable_input: False
+      test: mae/metadata_mae.dev.json
+    dino:
+      enable_input: False
+      test: dino/metadata_dino.dev.json
+    pose:
+      enable_input: True
+      test: YouTubeASL.keypoints.dev.json
+
+SignModelArguments:
+  projectors:
+    sign2vec:
+      dim: 768
+    mae:
+      dim: 768
+    dino:
+      dim: 1152
+    pose:
+      dim: 208

predict_pose.py

@@ -0,0 +1,529 @@
+import os
+from copy import deepcopy
+from typing import List
+
+import cv2
+import mediapipe as mp
+import numpy as np
+from mediapipe.tasks import python
+from mediapipe.tasks.python import vision
+from scipy.optimize import linear_sum_assignment
+from ultralytics import YOLO
+
+import os
+import json
+import numpy as np
+from datetime import datetime
+import cv2
+
+
+def crop_frame(image, bounding_box):
+    x, y, w, h = bounding_box
+    cropped_frame = image[y:y + h, x:x + w]
+    return cropped_frame
+
+
+def get_centered_box(keypoints, box_size, scale_factor=1.2):
+    center_x, center_y = np.mean(keypoints, axis=0, dtype=int)
+    half_size = box_size // 2
+    x = center_x - half_size
+    y = center_y - half_size
+    w = box_size
+    h = box_size
+
+    w_padding = int((scale_factor - 1) * w / 2)
+    h_padding = int((scale_factor - 1) * h / 2)
+    x -= w_padding
+    y -= h_padding
+    w += 2 * w_padding
+    h += 2 * h_padding
+
+    return x, y, w, h
+
+
+def get_bounding_box(keypoints, scale_factor=1.2):
+    keypoints = np.round(keypoints).astype(int)
+    x, y, w, h = cv2.boundingRect(keypoints)
+    w_padding = int((scale_factor - 1) * w / 2)
+    h_padding = int((scale_factor - 1) * h / 2)
+    x -= w_padding
+    y -= h_padding
+    w += 2 * w_padding
+    h += 2 * h_padding
+    return x, y, w, h
+
+
+def adjust_bounding_box(bounding_box, image_shape):
+    x, y, w, h = bounding_box
+    ih, iw, _ = image_shape
+
+    # Adjust x-coordinate if the bounding box extends beyond the image's right edge
+    if x + w > iw:
+        x = iw - w
+
+    # Adjust y-coordinate if the bounding box extends beyond the image's bottom edge
+    if y + h > ih:
+        y = ih - h
+
+    # Ensure bounding box's x and y coordinates are not negative
+    x = max(x, 0)
+    y = max(y, 0)
+
+    return x, y, w, h
+
+
+def create_mediapipe_models(checkpoint_folder: str, min_confidence: float = 0.4) -> (object, object, object, object):
+    BaseOptions = mp.tasks.BaseOptions
+
+    # mediapipe
+    num_poses = 1
+    hand_model_path = os.path.join(checkpoint_folder, 'hand_landmarker.task')
+    pose_model_path = os.path.join(checkpoint_folder, 'pose_landmarker_full.task')
+    face_model_path = os.path.join(checkpoint_folder, 'face_landmarker.task')
+    yolo_model_path = os.path.join(checkpoint_folder, "yolov8n-pose.pt")
+
+    # yolov8
+    yolo_model = YOLO(yolo_model_path)
+
+    # define hand model
+    hand_options = vision.HandLandmarkerOptions(
+        base_options=BaseOptions(model_asset_path=hand_model_path),
+        min_hand_detection_confidence=min_confidence,
+        min_hand_presence_confidence=min_confidence,
+        num_hands=num_poses * 2)
+    hand_detector = vision.HandLandmarker.create_from_options(hand_options)
+
+    # define body model
+    pose_options = vision.PoseLandmarkerOptions(
+        base_options=BaseOptions(model_asset_path=pose_model_path),
+        min_pose_detection_confidence=min_confidence,
+        min_pose_presence_confidence=min_confidence,
+        num_poses=num_poses
+    )
+    pose_detector = vision.PoseLandmarker.create_from_options(pose_options)
+
+    # define face model
+    face_options = vision.FaceLandmarkerOptions(
+        base_options=BaseOptions(model_asset_path=face_model_path),
+        min_face_detection_confidence=min_confidence,
+        min_face_presence_confidence=min_confidence,
+        num_faces=num_poses
+    )
+    face_detector = vision.FaceLandmarker.create_from_options(face_options)
+
+    return hand_detector, pose_detector, face_detector, yolo_model
+
+
+def yolo_predict(image: np.ndarray, model, min_conf: float = 0.5):
+    yolo_results = model(image, verbose=False)
+
+    bboxes = yolo_results[0].boxes.xyxy
+    keypoints = yolo_results[0].keypoints.xy
+    bboxes = bboxes.cpu().numpy()
+    keypoints = keypoints.cpu().numpy()
+
+    conf = yolo_results[0].boxes.conf
+    conf = conf.cpu().numpy()
+    select_mask_kp = np.sum(keypoints, axis=(1, 2)) > 0.0001
+    select_mask_bb = conf > min_conf
+    select_mask = select_mask_kp & select_mask_bb
+
+    conf = conf[select_mask]
+    bboxes = bboxes[select_mask]
+    keypoints = keypoints[select_mask]
+
+    return bboxes, keypoints, conf
+
+
+def load_video_cv(path: str):
+    video = []
+
+    cap = cv2.VideoCapture(path)
+    fps = cap.get(cv2.CAP_PROP_FPS)
+    ret = True
+    while ret:
+        ret, img = cap.read()
+        if ret:
+            img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
+            video.append(img)
+    cap.release()
+    return video, fps
+
+
+def new_bbox(image, keypoints, lsi=5, rsi=6, sign_space=5):
+    h, w = image.shape[:2]
+    l_shoulder = keypoints[lsi]
+    r_shoulder = keypoints[rsi]
+    distance = np.sqrt((l_shoulder[0] - r_shoulder[0]) ** 2 + (l_shoulder[1] - r_shoulder[1]) ** 2)
+
+    center_x = np.abs(l_shoulder[0] - r_shoulder[0]) / 2 + np.min([l_shoulder[0], r_shoulder[0]], 0)
+    center_y = np.abs(l_shoulder[1] - r_shoulder[1]) / 2 + np.min([l_shoulder[1], r_shoulder[1]], 0)
+
+    new_x0 = center_x - (distance * (sign_space / 2))
+    new_x1 = center_x + (distance * (sign_space / 2))
+    new_y0 = center_y - (distance * (sign_space / 2))
+    new_y1 = center_y + (distance * (sign_space / 2))
+
+    idx_x = keypoints[:, 0] > 0
+    idx_y = keypoints[:, 1] > 0
+    new_x0 = np.min([new_x0, *keypoints[idx_x, 0]])
+    new_x1 = np.max([new_x1, *keypoints[idx_x, 0]])
+    new_y0 = np.min([new_y0, *keypoints[idx_y, 1]])
+    new_y1 = np.max([new_y1, *keypoints[idx_y, 1]])
+
+    new_x0 = np.round(np.clip(new_x0, 0, w)).astype(int)
+    new_x1 = np.round(np.clip(new_x1, 0, w)).astype(int)
+    new_y0 = np.round(np.clip(new_y0, 0, h)).astype(int)
+    new_y1 = np.round(np.clip(new_y1, 0, h)).astype(int)
+
+    return new_x0, new_y0, new_x1, new_y1
+
+
+def mdeiapipe_to_xy(data, image_size=None):
+    """image_size: (height, width)"""
+    x = np.array([kp.x for kp in data])
+    y = np.array([kp.y for kp in data])
+
+    if image_size is not None:
+        x = x * image_size[1]
+        y = y * image_size[0]
+
+    return x, y
+
+
+def crop_pad_image(image: np.ndarray, bbox: np.ndarray, border: float = 0.25) -> np.ndarray:
+    """Crop the image, pad to square and add a border."""
+    # get bbox and image
+    x0, y0, x1, y1 = bbox
+    w, h = x1 - x0, y1 - y0
+
+    # add padding
+    dif = np.abs(w - h)
+    pad_value_0 = np.floor(dif / 2).astype(int)
+    pad_value_1 = dif - pad_value_0
+
+    if w > h:
+        y0 -= pad_value_0
+        y1 += pad_value_1
+    else:
+        x0 -= pad_value_0
+        x1 += pad_value_1
+
+    border = np.round((np.max([w, h]) * border) / 2).astype(int)
+    ih, iw = image.shape[:2]
+    y0 -= border
+    y1 += border
+    x0 -= border
+    x1 += border
+
+    new_bbox = [x0, y0, x1, y1]
+
+    y0 += ih
+    y1 += ih
+    x0 += iw
+    x1 += iw
+
+    image = np.pad(image, ((ih, ih), (iw, iw), (0, 0)), mode='constant', constant_values=0)  # mode="reflect"
+    cropped_image = image[y0:y1, x0:x1]
+
+    return cropped_image, new_bbox
+
+
+def keypoints_out_format(mp_keypoints, image_size):
+    """image_size = (ih, iw)"""
+    if len(mp_keypoints) >= 1:
+        data = mp_keypoints[0]
+        x, y = mdeiapipe_to_xy(data, image_size)
+        z = np.array([kp.z for kp in data])
+        visibility = np.array([kp.visibility for kp in data])
+        data = np.array([x, y, z, visibility]).T
+        return data
+    else:
+        return []
+
+
+def distance_matrix(P, Q):
+    dis_max = np.zeros([len(P), len(Q)])
+    for i, p in enumerate(P):
+        for j, q in enumerate(Q):
+            dist = np.linalg.norm(np.array(p) - np.array(q))
+            dis_max[i, j] = dist
+    return dis_max
+
+
+def process_hands(mp_hand_keypoints, mp_handedness, pose_keypoints, image_size, yolo_pose_keypoints=None):
+    out = {"left": [], "right": []}
+
+    if len(mp_hand_keypoints) == 0:
+        return out
+
+    # transform keypoints
+    hand_keypoints = []
+    for data in mp_hand_keypoints:
+        hand_keypoints.append(keypoints_out_format([data], image_size))
+
+    if (mp_hand_keypoints) == 1:
+        side = mp_handedness[0]["category_name"].lower
+        out[side] = hand_keypoints[0]
+        return out
+
+    # calculate centers
+    hand_centers = []
+    for keypoints in hand_keypoints:
+        x = keypoints[0, 0]
+        y = keypoints[0, 1]
+        hand_center = [x, y]
+        hand_centers.append(hand_center)
+
+    # assign hands to sides
+    left_wrist = None
+    right_wrist = None
+
+    pose_keypoints = None if len(pose_keypoints) == 0 else pose_keypoints
+    if pose_keypoints is not None:
+        left_wrist = pose_keypoints[15, :2]
+        right_wrist = pose_keypoints[16, :2]
+
+    elif pose_keypoints is None and yolo_pose_keypoints is not None:
+        left_wrist = yolo_pose_keypoints[9, :2]
+        right_wrist = yolo_pose_keypoints[10, :2]
+        if (np.sum(left_wrist) == 0) or (np.sum(right_wrist) == 0):
+            left_wrist = None
+            right_wrist = None
+
+    if left_wrist is not None and right_wrist is not None:
+        wrists = [left_wrist, right_wrist]
+
+        dis_max = distance_matrix(wrists, hand_centers)
+        row_idx, col_idx = linear_sum_assignment(dis_max)
+
+        sides = list(out.keys())
+        for ridx, cidx in zip(row_idx, col_idx):
+            side = sides[ridx]
+            keypoints = hand_keypoints[cidx]
+            out[side] = keypoints
+    else:
+        hand_centers_x = np.array(hand_centers)[:, 0]
+        right_idx = np.argmin(hand_centers_x)
+        out["right"] = hand_keypoints[right_idx]
+        left_idx = np.argmax(hand_centers_x)
+        if right_idx != left_idx:
+            out["left"] = hand_keypoints[left_idx]
+
+    return out
+
+
+def predict_pose(video: List[np.ndarray], models: tuple, sign_space=4, yolo_sign_space=4) -> dict:
+    """
+        This function processes a video to detect and extract pose, hand, and face landmarks using Mediapipe models.
+        It also calculates the signing space and crops the images accordingly.
+
+        Parameters:
+            video (list): A list of images.
+            models (tuple): A tuple containing the Mediapipe models for pose, hand, and face detection and yolo model.
+            sign_space (int): The desired size of the signing space.
+                              Width and height calculated as shoulder distance * sign_space  Default is 4.
+
+        Returns:
+            (dict): A dictionary containing the processed video data, including images, keypoints, cropped images, cropped keypoints,
+        signing space, and bounding boxes for different body parts.
+    """
+    hand_detector, pose_detector, face_detector, yolo_model = models
+    results = {
+        "images": video,
+        "keypoints": [],
+        "cropped_images": [],
+        "cropped_keypoints": [],
+        "sign_space": [],
+        "cropped_left_hand": [],
+        "cropped_right_hand": [],
+        "cropped_face": [],
+        "bbox_left_hand": [],
+        "bbox_right_hand": [],
+        "bbox_face": [],
+    }
+
+    # yolo predict + crop images
+    yolo_predictions = []
+    num_predictions = []
+    for idx, image in enumerate(results["images"]):
+        bboxes, keypoints, confs = yolo_predict(image, yolo_model)
+        yolo_predictions.append([bboxes, keypoints, confs])
+        num_predictions.append(len(bboxes))
+        
+    # no predictions -> add empty values and return
+    if np.sum(num_predictions) == 0:
+        _h, _w = results["images"][0].shape[:2]
+        for idx in range(len(results["images"])):
+            results["keypoints"].append({'pose_landmarks': [], 'right_hand_landmarks': [], 'left_hand_landmarks': [], 'face_landmarks': []})
+            results["cropped_images"].append(results["images"][idx])
+            results["cropped_keypoints"].append({'pose_landmarks': [], 'right_hand_landmarks': [], 'left_hand_landmarks': [], 'face_landmarks': []})
+            results["sign_space"].append([0, 0, _w, _h])
+            results["cropped_left_hand"].append(np.zeros([224, 224, 3], dtype=np.uint8))
+            results["cropped_right_hand"].append(np.zeros([224, 224, 3], dtype=np.uint8))
+            results["cropped_face"].append(np.zeros([224, 224, 3], dtype=np.uint8))
+            results["bbox_left_hand"].append([])
+            results["bbox_right_hand"].append([])
+            results["bbox_face"].append([])    
+        return results
+
+    # get signing bbox
+    x0, y0, x1, y1 = [], [], [], []
+    for idx, (image, prediction) in enumerate(zip(results["images"], yolo_predictions)):
+        _, keypoints, _ = prediction
+        if len(keypoints) != 1:
+            continue
+
+        _x0, _y0, _x1, _y1 = new_bbox(image, keypoints[0], lsi=5, rsi=6, sign_space=yolo_sign_space)
+
+        x0.append(_x0)
+        y0.append(_y0)
+        x1.append(_x1)
+        y1.append(_y1)
+
+    x0y = np.round(np.median(x0)).astype(int)
+    y0y = np.round(np.median(y0)).astype(int)
+    x1y = np.round(np.median(x1)).astype(int)
+    y1y = np.round(np.median(y1)).astype(int)
+
+    # mediapipe predict + signing space
+    mp_predictions = []
+    x0, y0, x1, y1 = [], [], [], []
+    for idx, image in enumerate(results["images"]):
+        yolo_image = image[y0y:y1y, x0y:x1y]
+
+        ih, iw = yolo_image.shape[:2]
+        mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=np.array(yolo_image))
+
+        # HACK:
+        # if the YOLO model does not detect anything,
+        # pretend it detects a black square and give it to mediapipe
+        if yolo_image.shape == (0, 0, 3):
+            mp_image = mp.Image(
+                image_format=mp.ImageFormat.SRGB,
+                data=np.zeros(shape=(256, 256, 3), dtype=np.uint8)
+            )
+
+        pose_prediction = pose_detector.detect(mp_image)
+        hand_prediction = hand_detector.detect(mp_image)
+        face_prediction = face_detector.detect(mp_image)
+
+        mp_predictions.append([hand_prediction, face_prediction, pose_prediction])
+
+        if len(pose_prediction.pose_landmarks) != 1:
+            continue
+
+        kp_all_x = []
+        kp_all_y = []
+        mp_keypoints = [
+            pose_prediction.pose_landmarks[0][:25],
+            *face_prediction.face_landmarks,
+            *hand_prediction.hand_landmarks,
+        ]
+
+        for p in mp_keypoints:
+            x, y = mdeiapipe_to_xy(p, (ih, iw))
+            kp_all_x.extend(x)
+            kp_all_y.extend(y)
+        kp_all = np.array((kp_all_x, kp_all_y)).T
+
+        kp_all[:, 0] = kp_all[:, 0] + x0y
+        kp_all[:, 1] = kp_all[:, 1] + y0y
+
+        if len(kp_all) == 0:
+            continue
+
+        _x0, _y0, _x1, _y1 = new_bbox(image, kp_all, lsi=11, rsi=12, sign_space=sign_space)
+
+        x0.append(_x0)
+        y0.append(_y0)
+        x1.append(_x1)
+        y1.append(_y1)
+
+    # create signing space as median of all signing spaces
+    if len(x0) == 0:
+        ih, iw = video[0].shape[:2]
+        x0mp = 0
+        y0mp = 0
+        x1mp = iw
+        y1mp = ih
+    else:
+        x0mp = np.round(np.median(x0)).astype(int)
+        y0mp = np.round(np.median(y0)).astype(int)
+        x1mp = np.round(np.median(x1)).astype(int)
+        y1mp = np.round(np.median(y1)).astype(int)
+
+    for idx, (image, prediction) in enumerate(zip(results["images"], mp_predictions)):
+        yolo_image = image[y0y:y1y, x0y:x1y]
+        yih, yiw = yolo_image.shape[:2]
+
+        cropped_image, pad_bbox = crop_pad_image(image, (x0mp, y0mp, x1mp, y1mp), border=0)
+
+        hand_prediction, face_prediction, pose_prediction = prediction
+
+        face_keypoints = keypoints_out_format(face_prediction.face_landmarks, (yih, yiw))
+        pose_keypoints = keypoints_out_format(pose_prediction.pose_landmarks, (yih, yiw))
+        hand_keypoints = process_hands(
+            hand_prediction.hand_landmarks,
+            hand_prediction.handedness,
+            pose_keypoints,
+            (yih, yiw),
+            None
+        )
+
+        keypoints = {
+            'pose_landmarks': pose_keypoints,
+            'right_hand_landmarks': hand_keypoints["right"],
+            'left_hand_landmarks': hand_keypoints["left"],
+            'face_landmarks': face_keypoints
+        }
+
+        # move kp
+        x_move = x0y
+        y_move = y0y
+        for name in keypoints:
+            if len(keypoints[name]) > 0:
+                keypoints[name][:, 0] += x_move
+                keypoints[name][:, 1] += y_move
+
+        # get dino crops
+        name_to_keypoints = [
+            ("face", face_keypoints),
+            ("left_hand", hand_keypoints["left"]),
+            ("right_hand", hand_keypoints["right"])
+        ]
+        for name, kp in name_to_keypoints:
+            if len(kp) > 0:
+                kp = np.round(kp[:, :2]).astype(int)
+                x, y, w, h = cv2.boundingRect(kp)
+                cropped_local_bbox = get_centered_box(kp, np.max([w, h]), scale_factor=1.2)
+                cropped_local_bbox = adjust_bounding_box(cropped_local_bbox, image.shape)
+                cropped_local_image = crop_frame(image, cropped_local_bbox)
+                x0, y0, w, h = cropped_local_bbox
+                cropped_local_bbox = [x0, y0, x0 + w, y0 + h]
+
+            else:
+                cropped_local_image = np.zeros([224, 224, 3], dtype=np.uint8)
+                cropped_local_bbox = []
+            results[f"bbox_{name}"].append(cropped_local_bbox)
+            results[f"cropped_{name}"].append(cropped_local_image)
+
+        # move kp
+        x_move = pad_bbox[0]
+        y_move = pad_bbox[1]
+        keypoints_cropped = deepcopy(keypoints)
+        for name in keypoints_cropped:
+            if len(keypoints_cropped[name]) > 0:
+                keypoints_cropped[name][:, 0] -= x_move
+                keypoints_cropped[name][:, 1] -= y_move
+                # Slice [:, :2] to keep only the first two columns before saving
+                keypoints_cropped[name] = np.round(keypoints_cropped[name][:, :2], 3).tolist()
+                keypoints[name] = np.round(keypoints[name][:, :2], 3).tolist()
+        # save processed data
+        results["keypoints"].append(keypoints)
+        results["cropped_images"].append(cropped_image)
+        results["cropped_keypoints"].append(keypoints_cropped)
+        results["sign_space"].append(pad_bbox)
+    results["images"] = video
+
+    return results

utils/translation.py

@@ -0,0 +1,32 @@
+import torch
+
+def postprocess_text(preds, labels):
+    preds = [pred.strip() for pred in preds]
+    labels = [[label.strip()] for label in labels]
+
+    return preds, labels
+
+# Add collate_fn to DataLoader
+def collate_fn(batch):
+    # Add padding to the inputs 
+    # "inputs" must be 250 frames long
+    # "attention_mask" must be 250 frames long
+    # "labels" must be 128 tokens long
+    return {
+        "sign_inputs": torch.stack([
+            torch.cat((sample["sign_inputs"], torch.zeros(250 - sample["sign_inputs"].shape[0], 208)), dim=0)
+            for sample in batch
+        ]),
+        "attention_mask": torch.stack([
+            torch.cat((sample["attention_mask"], torch.zeros(250 - sample["attention_mask"].shape[0])), dim=0)
+            if sample["attention_mask"].shape[0] < 250
+            else sample["attention_mask"]
+            for sample in batch
+        ]),
+        "labels": torch.stack([
+            torch.cat((sample["labels"].squeeze(0), torch.zeros(128 - sample["labels"].shape[0])), dim=0)
+            if sample["labels"].shape[0] < 128
+            else sample["labels"]
+            for sample in batch
+        ]).squeeze(0).to(torch.long),
+    }