final demo #2

.gitattributes

@@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+checkpoints/pose/face_landmarker.task filter=lfs diff=lfs merge=lfs -text
+checkpoints/pose/hand_landmarker.task filter=lfs diff=lfs merge=lfs -text
+checkpoints/pose/pose_landmarker_full.task filter=lfs diff=lfs merge=lfs -text

Uni_Sign/datasets.py

@@ -0,0 +1,1178 @@
+import torch
+import torch.utils.data.dataset as Dataset
+from torch.nn.utils.rnn import pad_sequence
+from PIL import Image
+import os
+import random
+import numpy as np
+import copy
+import pickle
+from decord import VideoReader, cpu
+import json
+import pathlib
+import re
+from torchvision import transforms
+# from config import rgb_dirs, pose_dirs
+from Uni_Sign.normalization import (local_keypoint_normalization, global_keypoint_normalization)
+
+
+def all_same(keypoints):
+    return np.sum(keypoints == keypoints[0, 0]) == keypoints.size
+
+
+def sign_space_normalization(raw_keypoints, missing_values=None, layout='default'):
+    local_landmarks = {}
+    global_landmarks = {}
+    kp_normalization = ('global-body', 'local-right', 'local-left', 'local-face_all')
+    part_order = [i.removeprefix('local-').removeprefix('global-') for i in kp_normalization]
+    part_order = {k: v for v, k in enumerate(part_order)}
+
+    for idx, landmarks in enumerate(kp_normalization):
+        prefix, landmarks = landmarks.split("-")
+        if prefix == "local":
+            local_landmarks[idx] = landmarks
+        elif prefix == "global":
+            global_landmarks[idx] = landmarks
+
+    # local normalization
+    for idx, landmarks in local_landmarks.items():
+        normalized_keypoints = local_keypoint_normalization(raw_keypoints, landmarks, padding=0.2)
+        local_landmarks[idx] = normalized_keypoints
+
+    # global normalization
+    additional_landmarks = list(global_landmarks.values())
+    if "body" in additional_landmarks:
+        additional_landmarks.remove("body")
+
+    if layout == 'default':
+        l_shoulder_idx, r_shoulder_idx = 11, 12
+    else:
+        l_shoulder_idx, r_shoulder_idx = 3, 4
+    keypoints, additional_keypoints = global_keypoint_normalization(
+        raw_keypoints,
+        "body",
+        additional_landmarks,
+        l_shoulder_idx=l_shoulder_idx,
+        r_shoulder_idx=r_shoulder_idx,
+    )
+
+    for k, landmark in global_landmarks.items():
+        if landmark == "body":
+            global_landmarks[k] = keypoints
+        else:
+            global_landmarks[k] = additional_keypoints[landmark]
+
+    all_landmarks = {**local_landmarks, **global_landmarks}
+    all_landmarks_per_part = {k: all_landmarks[v] for k, v in part_order.items()}
+
+    if missing_values is not None:
+        for part, data in all_landmarks_per_part.items():
+            for fidx in range(len(data)):
+                if not all_same(data[fidx]):
+                    continue
+                all_landmarks_per_part[part][fidx] = np.zeros_like(data[fidx]) + missing_values
+
+    return all_landmarks_per_part
+
+
+def load_part_kp_YTASL(skeletons, confs, normalization, layout):
+    thr = 0.3
+    # kps_with_scores = {}
+    kps_all_parts = {}
+    confs_all_parts = {}
+    scale = None
+
+    for part in ['body', 'left', 'right', 'face_all']:
+        kps = []
+        confidences = []
+        for i, (skeleton, conf) in enumerate(zip(skeletons, confs)):
+
+            if part == 'body':
+                if layout == 'default':
+                    hand_kp2d = np.stack(skeleton['pose_landmarks'][:25])
+                    confidence = np.stack(conf['pose_landmarks'][:25])
+                elif layout == 'pruned':
+                    pose_landmarks = [0, 7, 8, 11, 12, 13, 14, 15, 16]
+                    hand_kp2d = np.stack([skeleton['pose_landmarks'][i] for i in pose_landmarks])
+                    confidence = np.stack([conf['pose_landmarks'][i] for i in pose_landmarks])
+                elif layout == 'isharah':
+                    pose_landmarks = [0, 7, 8, 11, 12, 13, 14, 15, 16]
+                    hand_kp2d = np.stack([skeleton['pose_landmarks'][i] for i in pose_landmarks])
+                    confidence = np.stack([conf['pose_landmarks'][i] for i in pose_landmarks])
+
+            elif part == 'left':
+                if layout in ['default', 'pruned', 'isharah']:
+                    hand_kp2d = np.stack(skeleton['left_hand_landmarks'])
+                    confidence = np.stack(conf['left_hand_landmarks'])
+
+            elif part == 'right':
+                if layout in ['default', 'pruned', 'isharah']:
+                    hand_kp2d = np.stack(skeleton['right_hand_landmarks'])
+                    confidence = np.stack(conf['right_hand_landmarks'])
+
+            elif part == 'face_all':
+                if layout == 'default':
+                    face_landmarks = [
+                        0, 4, 13, 14, 17, 33, 39, 46, 52, 55, 61, 64, 81,
+                        93, 133, 151, 152, 159, 172, 178, 181, 263, 269, 276,
+                        282, 285, 291, 294, 311, 323, 362, 386, 397, 402, 405, 468, 473
+                    ]
+                    hand_kp2d = np.stack([skeleton['face_landmarks'][i] for i in face_landmarks])
+                    confidence = np.stack([conf['face_landmarks'][i] for i in face_landmarks])
+                elif layout == 'pruned':
+                    face_landmarks = [4, 13, 14, 61, 81, 93, 152, 159, 172, 178, 291, 311, 323, 386, 397, 402, 472, 477]
+                    hand_kp2d = np.stack([skeleton['face_landmarks'][i] for i in face_landmarks])
+                    confidence = np.stack([conf['face_landmarks'][i] for i in face_landmarks])
+                elif layout == 'isharah':
+                    face_landmarks = [0, 17, 37, 39, 40, 61, 84, 91, 146, 181, 185, 267, 269, 270, 291, 314, 321, 375, 405]
+                    hand_kp2d = np.stack([skeleton['face_landmarks'][i] for i in face_landmarks])
+                    confidence = np.stack([conf['face_landmarks'][i] for i in face_landmarks])
+
+            else:
+                raise NotImplementedError
+            kps.append(hand_kp2d)
+            confidences.append(confidence)
+
+        kps = np.stack(kps, axis=0)
+        confidences = np.stack(confidences, axis=0)
+
+        kps_all_parts[part] = kps
+        confs_all_parts[part] = confidences[..., None]
+
+    if normalization == 'signspace':
+        normalized_kps = sign_space_normalization(kps_all_parts.copy(), layout=layout)
+    else:
+        normalized_kps = kps_all_parts
+
+    kps_with_scores = {}
+    for part in normalized_kps.keys():
+        kps_with_scores[part] = np.concatenate([normalized_kps[part], confs_all_parts[part]], axis=-1)
+
+    kps_with_scores = {k: torch.as_tensor(v, dtype=torch.float32) for k, v in kps_with_scores.items()}
+    return kps_with_scores
+
+
+def load_part_kp_Isharah(skeletons, confs, normalization, layout):
+    # kps_with_scores = {}
+    kps_all_parts = {}
+    confs_all_parts = {}
+
+    for part in ['body', 'left', 'right', 'face_all']:
+        kps = []
+        confidences = []
+        for i, (skeleton, conf) in enumerate(zip(skeletons, confs)):
+
+            if part == 'body':
+                pose_landmarks = [0, 7, 8, 11, 12, 13, 14, 15, 16]
+                hand_kp2d = np.stack([skeleton['pose_landmarks'][i] for i in pose_landmarks])
+                confidence = np.stack([conf['pose_landmarks'][i] for i in pose_landmarks])
+
+            elif part == 'left':
+                hand_kp2d = np.stack(skeleton['left_hand_landmarks'])
+                confidence = np.stack(conf['left_hand_landmarks'])
+
+            elif part == 'right':
+                hand_kp2d = np.stack(skeleton['right_hand_landmarks'])
+                confidence = np.stack(conf['right_hand_landmarks'])
+
+            elif part == 'face_all':
+                hand_kp2d = np.stack(skeleton['face_landmarks'])
+                confidence = np.stack(conf['face_landmarks'])
+
+            else:
+                raise NotImplementedError
+            kps.append(hand_kp2d)
+            confidences.append(confidence)
+
+        kps = np.stack(kps, axis=0)
+        confidences = np.stack(confidences, axis=0)
+
+        kps_all_parts[part] = kps
+        confs_all_parts[part] = confidences[..., None]
+
+    if normalization == 'signspace':
+        normalized_kps = sign_space_normalization(kps_all_parts.copy(), layout=layout)
+    else:
+        normalized_kps = kps_all_parts
+
+    kps_with_scores = {}
+    for part in normalized_kps.keys():
+        kps_with_scores[part] = np.concatenate([normalized_kps[part], confs_all_parts[part]], axis=-1)
+
+    kps_with_scores = {k: torch.as_tensor(v, dtype=torch.float32) for k, v in kps_with_scores.items()}
+    return kps_with_scores
+
+
+YTASL_GROUP_SIZES = {
+    'pose_landmarks': 33,
+    'right_hand_landmarks': 21,
+    'left_hand_landmarks': 21,
+    'face_landmarks': 478,
+}
+
+YTASL_GROUP_ERROR_LABELS = {
+    'pose_landmarks': 'a pose group',
+    'right_hand_landmarks': 'a Rhand group',
+    'left_hand_landmarks': 'a Lhand group',
+    'face_landmarks': 'a face group',
+}
+
+ISHARAH_GROUP_SIZES = {
+    'pose_landmarks': 25,
+    'right_hand_landmarks': 21,
+    'left_hand_landmarks': 21,
+    'face_landmarks': 19,
+}
+
+
+def _fill_missing_landmarks(
+    skeleton,
+    conf,
+    group_name,
+    expected_size,
+    clip_name,
+    frame_idx,
+    error_group_label=None,
+    include_size_details=True,
+    strict_key_access=False,
+):
+    points = skeleton[group_name] if strict_key_access else skeleton.get(group_name, [])
+    if len(points) == 0:
+        conf[group_name] = [0] * expected_size
+        skeleton[group_name] = [[0.0, 0.0]] * expected_size
+    elif len(points) != expected_size:
+        group_label = error_group_label or f"group '{group_name}'"
+        if include_size_details:
+            raise NotImplementedError(
+                f"Unexpected number of keypoints in {group_label}: {clip_name}, frame {frame_idx}, "
+                f"expected {expected_size}, got {len(points)}"
+            )
+        raise NotImplementedError(f"Unexpected number of keypoints in {group_label}: {clip_name}, {frame_idx}")
+    else:
+        conf[group_name] = [1] * expected_size
+
+
+# load sub-pose
+def load_part_kp(skeletons, confs, force_ok=False):
+    thr = 0.3
+    kps_with_scores = {}
+    scale = None
+
+    for part in ['body', 'left', 'right', 'face_all']:
+        kps = []
+        confidences = []
+
+        for skeleton, conf in zip(skeletons, confs):
+            if skeleton.ndim == 4:  # if (1,133,2) - wrapped in list
+                skeleton = skeleton[0]
+                conf = conf[0]
+
+            if part == 'body':  # [0, 3, 4, 5, 6, 7, 8, 9, 10]
+                hand_kp2d = skeleton[[0] + [i for i in range(3, 11)], :]
+                confidence = conf[[0] + [i for i in range(3, 11)]]
+            elif part == 'left':  # [91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111]
+                hand_kp2d = skeleton[91:112, :]
+                hand_kp2d = hand_kp2d - hand_kp2d[0, :]
+                confidence = conf[91:112]
+            elif part == 'right':  # [112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132]
+                hand_kp2d = skeleton[112:133, :]
+                hand_kp2d = hand_kp2d - hand_kp2d[0, :]
+                confidence = conf[112:133]
+            elif part == 'face_all':  # [23, 25, 27, 29, 31, 33, 35, 37, 39, 83, 84, 85, 86, 87, 88, 89, 90, 53]
+                hand_kp2d = skeleton[[i for i in list(range(23, 23 + 17))[::2]] + [i for i in range(83, 83 + 8)] + [53], :]
+                hand_kp2d = hand_kp2d - hand_kp2d[-1, :]
+                confidence = conf[[i for i in list(range(23, 23 + 17))[::2]] + [i for i in range(83, 83 + 8)] + [53]]
+
+            else:
+                raise NotImplementedError
+
+            kps.append(hand_kp2d)
+            confidences.append(confidence)
+
+        kps = np.stack(kps, axis=0)
+        confidences = np.stack(confidences, axis=0)
+
+        if part == 'body':
+            if force_ok:
+                result, scale, _ = crop_scale(np.concatenate([kps, confidences[..., None]], axis=-1), thr)
+
+            else:
+                result, scale, _ = crop_scale(np.concatenate([kps, confidences[..., None]], axis=-1), thr)
+        else:
+            assert not scale is None
+            result = np.concatenate([kps, confidences[..., None]], axis=-1)
+            if scale == 0:
+                result = np.zeros(result.shape)
+            else:
+                result[..., :2] = (result[..., :2]) / scale
+                result = np.clip(result, -1, 1)
+                # mask useless kp
+                result[result[..., 2] <= thr] = 0
+
+        kps_with_scores[part] = torch.tensor(result)
+
+    return kps_with_scores
+
+
+# input: T, N, 3
+# input is un-normed joints
+def crop_scale(motion, thr):
+    '''
+        Motion: [(M), T, 17, 3].
+        Normalize to [-1, 1]
+    '''
+    result = copy.deepcopy(motion)
+    valid_coords = motion[motion[..., 2] > thr][:, :2]
+    if len(valid_coords) < 4:
+        return np.zeros(motion.shape), 0, None
+    xmin = min(valid_coords[:, 0])
+    xmax = max(valid_coords[:, 0])
+    ymin = min(valid_coords[:, 1])
+    ymax = max(valid_coords[:, 1])
+    # ratio = np.random.uniform(low=scale_range[0], high=scale_range[1], size=1)[0]
+    ratio = 1
+    scale = max(xmax - xmin, ymax - ymin) * ratio
+    if scale == 0:
+        return np.zeros(motion.shape), 0, None
+    xs = (xmin + xmax - scale) / 2
+    ys = (ymin + ymax - scale) / 2
+    result[..., :2] = (motion[..., :2] - [xs, ys]) / scale
+    result[..., :2] = (result[..., :2] - 0.5) * 2
+    result = np.clip(result, -1, 1)
+    # mask useless kp
+    result[result[..., 2] <= thr] = 0
+    return result, scale, [xs, ys]
+
+
+# bbox of hands
+def bbox_4hands(left_keypoints, right_keypoints, hw):
+    # keypoints --> T,21,2
+    # keypoints --> T,21,2
+
+    def compute_bbox(keypoints):
+        min_x = np.min(keypoints[..., 0], axis=1)
+        min_y = np.min(keypoints[..., 1], axis=1)
+        max_x = np.max(keypoints[..., 0], axis=1)
+        max_y = np.max(keypoints[..., 1], axis=1)
+
+        return (max_x + min_x) / 2, (max_y + min_y) / 2, (max_x - min_x), (max_y - min_y)
+
+    H, W = hw
+
+    if left_keypoints is None:
+        left_keypoints = np.zeros([1, 21, 2])
+
+    if right_keypoints is None:
+        right_keypoints = np.zeros([1, 21, 2])
+    # [T, 21, 2]
+    left_mean_x, left_mean_y, left_diff_x, left_diff_y = compute_bbox(left_keypoints)
+    left_mean_x = W * left_mean_x
+    left_mean_y = H * left_mean_y
+
+    left_diff_x = W * left_diff_x
+    left_diff_y = H * left_diff_y
+
+    left_diff_x = max(left_diff_x)
+    left_diff_y = max(left_diff_y)
+    left_box_hw = max(left_diff_x, left_diff_y)
+
+    right_mean_x, right_mean_y, right_diff_x, right_diff_y = compute_bbox(right_keypoints)
+    right_mean_x = W * right_mean_x
+    right_mean_y = H * right_mean_y
+
+    right_diff_x = W * right_diff_x
+    right_diff_y = H * right_diff_y
+
+    right_diff_x = max(right_diff_x)
+    right_diff_y = max(right_diff_y)
+    right_box_hw = max(right_diff_x, right_diff_y)
+
+    box_hw = int(max(left_box_hw, right_box_hw) * 1.2 / 2) * 2
+    box_hw = max(box_hw, 0)
+
+    left_new_box = np.stack([left_mean_x - box_hw / 2, left_mean_y - box_hw / 2, left_mean_x + box_hw / 2,
+                             left_mean_y + box_hw / 2]).astype(np.int16)
+    right_new_box = np.stack([right_mean_x - box_hw / 2, right_mean_y - box_hw / 2, right_mean_x + box_hw / 2,
+                              right_mean_y + box_hw / 2]).astype(np.int16)
+
+    return left_new_box.transpose(1, 0), right_new_box.transpose(1, 0), box_hw
+
+
+def load_support_rgb_dict(tmp, skeletons, confs, full_path, data_transform):
+    support_rgb_dict = {}
+
+    confs = np.array(confs)
+    skeletons = np.array(skeletons)
+
+    # sample index of low scores
+    left_confs_filter = confs[:, 0, 91:112].mean(-1)
+    left_confs_filter_indices = np.where(left_confs_filter > 0.3)[0]
+
+    if len(left_confs_filter_indices) == 0:
+        left_sampled_indices = None
+        left_skeletons = None
+    else:
+
+        left_confs = confs[left_confs_filter_indices]
+        left_confs = left_confs[:, 0, [95, 99, 103, 107, 111]].min(-1)
+
+        left_weights = np.max(left_confs) - left_confs + 1e-5
+        left_probabilities = left_weights / np.sum(left_weights)
+
+        left_sample_size = int(np.ceil(0.1 * len(left_confs_filter_indices)))
+
+        left_sampled_indices = np.random.choice(left_confs_filter_indices.tolist(),
+                                                size=left_sample_size,
+                                                replace=False,
+                                                p=left_probabilities)
+        # left_sampled_indices: values: 0-255(0,max_len)
+        # tmp: values: 0-(end-start)
+        left_sampled_indices = np.sort(left_sampled_indices)
+
+        left_skeletons = skeletons[left_sampled_indices, 0, 91:112]
+
+    right_confs_filter = confs[:, 0, 112:].mean(-1)
+    right_confs_filter_indices = np.where(right_confs_filter > 0.3)[0]
+    if len(right_confs_filter_indices) == 0:
+        right_sampled_indices = None
+        right_skeletons = None
+
+    else:
+        right_confs = confs[right_confs_filter_indices]
+        right_confs = right_confs[:, 0, [95 + 21, 99 + 21, 103 + 21, 107 + 21, 111 + 21]].min(-1)
+
+        right_weights = np.max(right_confs) - right_confs + 1e-5
+        right_probabilities = right_weights / np.sum(right_weights)
+
+        right_sample_size = int(np.ceil(0.1 * len(right_confs_filter_indices)))
+
+        right_sampled_indices = np.random.choice(right_confs_filter_indices.tolist(),
+                                                 size=right_sample_size,
+                                                 replace=False,
+                                                 p=right_probabilities)
+        right_sampled_indices = np.sort(right_sampled_indices)
+
+        right_skeletons = skeletons[right_sampled_indices, 0, 112:133]
+
+    image_size = 112
+    all_indices = []
+    if not left_sampled_indices is None:
+        all_indices.append(left_sampled_indices)
+    if not right_sampled_indices is None:
+        all_indices.append(right_sampled_indices)
+    if len(all_indices) == 0:
+        support_rgb_dict['left_sampled_indices'] = torch.tensor([-1])
+        support_rgb_dict['left_hands'] = torch.zeros(1, 3, image_size, image_size)
+        support_rgb_dict['left_skeletons_norm'] = torch.zeros(1, 21, 2)
+
+        support_rgb_dict['right_sampled_indices'] = torch.tensor([-1])
+        support_rgb_dict['right_hands'] = torch.zeros(1, 3, image_size, image_size)
+        support_rgb_dict['right_skeletons_norm'] = torch.zeros(1, 21, 2)
+
+        return support_rgb_dict
+
+    sampled_indices = np.concatenate(all_indices)
+    sampled_indices = np.unique(sampled_indices)
+    sampled_indices_real = tmp[sampled_indices]
+
+    # load image sample
+    imgs = load_video_support_rgb(full_path, sampled_indices_real)
+
+    # get hand bbox
+    left_new_box, right_new_box, box_hw = bbox_4hands(left_skeletons,
+                                                      right_skeletons,
+                                                      imgs[0].shape[:2])
+
+    # crop left and right hand
+    image_size = 112
+    if box_hw == 0:
+        support_rgb_dict['left_sampled_indices'] = torch.tensor([-1])
+        support_rgb_dict['left_hands'] = torch.zeros(1, 3, image_size, image_size)
+        support_rgb_dict['left_skeletons_norm'] = torch.zeros(1, 21, 2)
+
+        support_rgb_dict['right_sampled_indices'] = torch.tensor([-1])
+        support_rgb_dict['right_hands'] = torch.zeros(1, 3, image_size, image_size)
+        support_rgb_dict['right_skeletons_norm'] = torch.zeros(1, 21, 2)
+
+        return support_rgb_dict
+
+    factor = image_size / box_hw
+
+    if left_sampled_indices is None:
+        left_hands = torch.zeros(1, 3, image_size, image_size)
+        left_skeletons_norm = torch.zeros(1, 21, 2)
+
+    else:
+        left_hands = torch.zeros(len(left_sampled_indices), 3, image_size, image_size)
+
+        left_skeletons_norm = left_skeletons * imgs[0].shape[:2][::-1] - left_new_box[:, None, [0, 1]]
+        left_skeletons_norm = left_skeletons_norm / box_hw
+        left_skeletons_norm = left_skeletons_norm.clip(0, 1)
+
+    if right_sampled_indices is None:
+        right_hands = torch.zeros(1, 3, image_size, image_size)
+        right_skeletons_norm = torch.zeros(1, 21, 2)
+
+    else:
+        right_hands = torch.zeros(len(right_sampled_indices), 3, image_size, image_size)
+
+        right_skeletons_norm = right_skeletons * imgs[0].shape[:2][::-1] - right_new_box[:, None, [0, 1]]
+        right_skeletons_norm = right_skeletons_norm / box_hw
+        right_skeletons_norm = right_skeletons_norm.clip(0, 1)
+    left_idx = 0
+    right_idx = 0
+
+    for idx, img in enumerate(imgs):
+        mapping_idx = sampled_indices[idx]
+        if not left_sampled_indices is None and left_idx < len(left_sampled_indices) and mapping_idx == \
+                left_sampled_indices[left_idx]:
+            box = left_new_box[left_idx]
+
+            img_draw = np.uint8(copy.deepcopy(img))[box[1]:box[3], box[0]:box[2], :]
+            img_draw = np.pad(img_draw,
+                              ((0, max(0, box_hw - img_draw.shape[0])), (0, max(0, box_hw - img_draw.shape[1])),
+                               (0, 0)), mode='constant', constant_values=0)
+
+            f_img = Image.fromarray(img_draw).convert('RGB').resize((image_size, image_size))
+            f_img = data_transform(f_img).unsqueeze(0)
+            left_hands[left_idx] = f_img
+            left_idx += 1
+
+        if not right_sampled_indices is None and right_idx < len(right_sampled_indices) and mapping_idx == \
+                right_sampled_indices[right_idx]:
+            box = right_new_box[right_idx]
+
+            img_draw = np.uint8(copy.deepcopy(img))[box[1]:box[3], box[0]:box[2], :]
+            img_draw = np.pad(img_draw,
+                              ((0, max(0, box_hw - img_draw.shape[0])), (0, max(0, box_hw - img_draw.shape[1])),
+                               (0, 0)), mode='constant', constant_values=0)
+
+            f_img = Image.fromarray(img_draw).convert('RGB').resize((image_size, image_size))
+            f_img = data_transform(f_img).unsqueeze(0)
+            right_hands[right_idx] = f_img
+            right_idx += 1
+
+    if left_sampled_indices is None:
+        left_sampled_indices = np.array([-1])
+
+    if right_sampled_indices is None:
+        right_sampled_indices = np.array([-1])
+
+    # get index, images and keypoints priors
+    support_rgb_dict['left_sampled_indices'] = torch.tensor(left_sampled_indices)
+    support_rgb_dict['left_hands'] = left_hands
+    support_rgb_dict['left_skeletons_norm'] = torch.tensor(left_skeletons_norm)
+
+    support_rgb_dict['right_sampled_indices'] = torch.tensor(right_sampled_indices)
+    support_rgb_dict['right_hands'] = right_hands
+    support_rgb_dict['right_skeletons_norm'] = torch.tensor(right_skeletons_norm)
+
+    return support_rgb_dict
+
+
+# use split rgb video for save time
+def load_video_support_rgb(path, tmp):
+    vr = VideoReader(path, num_threads=1, ctx=cpu(0))
+
+    vr.seek(0)
+    buffer = vr.get_batch(tmp).asnumpy()
+    batch_image = buffer
+    del vr
+
+    return batch_image
+
+
+def load_json(path):
+    with open(path, "r", encoding="utf-8") as f:
+        return json.load(f)
+
+def is_valid_metric_label(text):
+    if text is None:
+        return False
+    text = " ".join(str(text).split()).strip()
+    if not text:
+        return False
+    # Require at least one alnum/letter token after punctuation/symbol stripping.
+    return re.search(r"\w", text, flags=re.UNICODE) is not None
+
+def select_frame_indices(duration, max_length, phase):
+    if duration <= max_length:
+        return list(range(duration))
+    if phase == 'train':
+        return sorted(random.sample(range(duration), k=max_length))
+    # Deterministic, near-uniform coverage for dev/test.
+    return ((np.arange(max_length) * duration) // max_length).tolist()
+
+
+# build base dataset
+class Base_Dataset(Dataset.Dataset):
+    def collate_fn(self, batch):
+        tgt_batch, src_length_batch, name_batch, pose_tmp, gloss_batch = [], [], [], [], []
+
+        for name_sample, pose_sample, text, gloss, _ in batch:
+            name_batch.append(name_sample)
+            pose_tmp.append(pose_sample)
+            tgt_batch.append(text)
+            gloss_batch.append(gloss)
+
+        src_input = {}
+
+        keys = pose_tmp[0].keys()
+        for key in keys:
+            max_len = max([len(vid[key]) for vid in pose_tmp])
+            video_length = torch.LongTensor([len(vid[key]) for vid in pose_tmp])
+
+            padded_video = [torch.cat(
+                (
+                    vid[key],
+                    vid[key][-1][None].expand(max_len - len(vid[key]), -1, -1),
+                )
+                , dim=0)
+                for vid in pose_tmp]
+
+            img_batch = torch.stack(padded_video, 0)
+
+            src_input[key] = img_batch
+            if 'attention_mask' not in src_input.keys():
+                src_length_batch = video_length
+
+                mask_gen = []
+                for i in src_length_batch:
+                    tmp = torch.ones([i]) + 7
+                    mask_gen.append(tmp)
+                mask_gen = pad_sequence(mask_gen, padding_value=0, batch_first=True)
+                img_padding_mask = (mask_gen != 0).long()
+                src_input['attention_mask'] = img_padding_mask
+
+                src_input['name_batch'] = name_batch
+                src_input['src_length_batch'] = src_length_batch
+
+        if self.rgb_support:
+            support_rgb_dicts = {key: [] for key in batch[0][-1].keys()}
+            for _, _, _, _, support_rgb_dict in batch:
+                for key in support_rgb_dict.keys():
+                    support_rgb_dicts[key].append(support_rgb_dict[key])
+
+            for part in ['left', 'right']:
+                index_key = f'{part}_sampled_indices'
+                skeletons_key = f'{part}_skeletons_norm'
+                rgb_key = f'{part}_hands'
+                len_key = f'{part}_rgb_len'
+
+                index_batch = torch.cat(support_rgb_dicts[index_key], 0)
+                skeletons_batch = torch.cat(support_rgb_dicts[skeletons_key], 0)
+                img_batch = torch.cat(support_rgb_dicts[rgb_key], 0)
+
+                src_input[index_key] = index_batch
+                src_input[skeletons_key] = skeletons_batch
+                src_input[rgb_key] = img_batch
+                src_input[len_key] = [len(index) for index in support_rgb_dicts[index_key]]
+
+        tgt_input = {}
+        tgt_input['gt_sentence'] = tgt_batch
+        tgt_input['gt_gloss'] = gloss_batch
+
+        return src_input, tgt_input
+
+import gzip
+def load_dataset_file(filename):
+    with gzip.open(filename, "rb") as f:
+        loaded_object = pickle.load(f)
+        return loaded_object
+    
+class S2T_Dataset(Base_Dataset):
+    def __init__(self, path, args, phase):
+        super(S2T_Dataset, self).__init__()
+        self.args = args
+        self.rgb_support = self.args.rgb_support
+        self.max_length = args.max_length
+        self.raw_data = load_dataset_file(path)
+        self.phase = phase
+
+        if self.args.dataset == "CSL_Daily":
+            self.pose_dir = pose_dirs[args.dataset]
+            self.rgb_dir = rgb_dirs[args.dataset]
+
+        elif "WLASL" in self.args.dataset:
+            self.pose_dir = os.path.join(pose_dirs[args.dataset], phase)
+            self.rgb_dir = os.path.join(rgb_dirs[args.dataset], phase)
+
+        elif self.args.dataset == "Isharah":
+            self.pose_dir = pose_dirs[args.dataset]  # pose only
+            self.rgb_dir = rgb_dirs[args.dataset]  # ""
+
+        else:
+            raise NotImplementedError(f"dataset {self.args.dataset} not supported")
+
+        self.list = list(self.raw_data.keys())
+
+        self.data_transform = transforms.Compose([
+            transforms.ToTensor(),
+            transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
+        ])
+
+    def __len__(self):
+        return len(self.list)
+
+    def __getitem__(self, index):
+        key = self.list[index]
+        sample = self.raw_data[key]
+
+        text = sample['text']
+        if "gloss" in sample.keys():
+            gloss = " ".join(sample['gloss'])
+        else:
+            gloss = ''
+
+        name_sample = sample['name']
+        pose_sample, support_rgb_dict = self.load_pose(sample['video_path'])
+
+        return name_sample, pose_sample, text, gloss, support_rgb_dict
+
+    def load_pose(self, path):
+        pose = pickle.load(open(os.path.join(self.pose_dir, path.replace(".mp4", '.pkl')), 'rb'))
+
+        if 'start' in pose.keys():
+            assert pose['start'] < pose['end']
+            duration = pose['end'] - pose['start']
+            start = pose['start']
+        else:
+            duration = len(pose['scores'])
+            start = 0
+
+        tmp = select_frame_indices(duration, self.max_length, self.phase)
+
+        tmp = np.array(tmp) + start
+
+        skeletons = pose['keypoints']
+        confs = pose['scores']
+        skeletons_tmp = []
+        confs_tmp = []
+        for index in tmp:
+            skeletons_tmp.append(skeletons[index])
+            confs_tmp.append(confs[index])
+
+        skeletons = skeletons_tmp
+        confs = confs_tmp
+
+        kps_with_scores = load_part_kp(skeletons, confs, force_ok=True)
+
+        support_rgb_dict = {}
+        if self.rgb_support:
+            full_path = os.path.join(self.rgb_dir, path)
+            support_rgb_dict = load_support_rgb_dict(tmp, skeletons, confs, full_path, self.data_transform)
+
+        return kps_with_scores, support_rgb_dict
+
+    def __str__(self):
+        return f'#total {len(self)}'
+
+
+class S2T_Dataset_YTASL(Base_Dataset):
+    def __init__(self, path, args, phase):
+        super(S2T_Dataset_YTASL, self).__init__()
+        self.args = args
+        self.max_length = args.max_length
+        self.phase = phase
+        self.annotation = load_json(path)
+        self.rgb_support = self.args.rgb_support
+        self.normalization = self.args.normalization
+        self.layout = args.layout
+
+        self.pose_dir = pose_dirs[args.dataset]
+        self.rgb_dir = rgb_dirs[args.dataset]
+
+        self.list_data = []  # [(video_id, clip_id), ...]
+        self.clip_order_to_int = {}
+        self.clip_order_from_int = {}
+
+        for video_id in self.annotation.keys():
+            co = self.annotation[video_id]['clip_order']
+            self.clip_order_from_int[video_id] = dict(zip(range(len(co)), co))
+            self.clip_order_to_int[video_id] = dict(zip(co, range(len(co))))
+
+        for video_id, clip_dict in self.annotation.items():
+            for clip_name in clip_dict['clip_order']:
+                self.list_data.append((video_id, self.clip_order_to_int[video_id][clip_name]))
+
+        available_clip_names = {
+            pathlib.Path(clip).stem  # remove suffix
+            for clip in os.listdir(self.pose_dir)
+            if clip.endswith(".json")
+        }
+        video_clips = set()
+        for video_id, clip_dict in self.annotation.items():
+            for clip_name in clip_dict['clip_order']:
+                if clip_name in available_clip_names:
+                    video_clips.add((video_id, self.clip_order_to_int[video_id][clip_name]))
+
+        self.remove_missing_annotation(video_clips)  # Remove data in annotations that are missing in h5 file
+
+    def remove_missing_annotation(self, h5_video_clip):
+        annotations_to_delete = set(self.list_data) - h5_video_clip
+        for a in annotations_to_delete:
+            self.list_data.remove(a)
+
+    def __getitem__(self, index):
+        video_id, clip_id = self.list_data[index]
+        clip_name = self.clip_order_from_int[video_id][clip_id]
+
+        # Get translation
+        clip_dict = self.annotation[video_id][clip_name]
+        text = clip_dict['translation']
+
+        # Get the pose features
+        pose_sample = self.load_pose(clip_name)
+
+        # TODO: rgb support
+        video_path = ""
+        support_rgb_dict = {}
+
+        # sample = {"name": clip_name,
+        #           "video_path": video_path,
+        #           "pose_features": pose_features,
+        #           "text": translation}
+        # Crop long sequences to desired max length. Random sample
+
+        # skeletons = pose['keypoints']
+        # confs = pose['scores']
+        # skeletons_tmp = []
+        # confs_tmp = []
+        # for index in tmp:
+        #     skeletons_tmp.append(skeletons[index])
+        #     confs_tmp.append(confs[index])
+        #
+        # skeletons = skeletons_tmp
+        # confs = confs_tmp
+
+        # confs = [np.ones(int(pose_features.shape[1]/2)) for _ in range(pose_features.shape[0])]
+        # confs = [np.ones(pose_features.shape[0])] * pose_features.shape[1]
+        # skeletons = [] # List of ndarrays (133,2) - full keypoints
+        # kps_with_scores = load_part_kp(skeletons, confs, force_ok=True)
+
+        name_sample = clip_name
+        gloss = ''
+
+        return name_sample, pose_sample, text, gloss, support_rgb_dict
+
+    def load_pose(self, clip_name):
+        path = os.path.join(self.pose_dir, f"{clip_name}.json")
+        pose_data = load_json(path)
+        pose = pose_data['cropped_keypoints']
+
+        duration = len(pose)
+        start = 0
+
+        tmp = select_frame_indices(duration, self.max_length, self.phase)
+        tmp = np.array(tmp) + start
+        skeletons = [pose[i] for i in tmp]
+
+        confs = []
+        for i, skeleton in enumerate(skeletons):
+            conf = {}
+            for group_name, expected_size in YTASL_GROUP_SIZES.items():
+                _fill_missing_landmarks(
+                    skeleton=skeleton,
+                    conf=conf,
+                    group_name=group_name,
+                    expected_size=expected_size,
+                    clip_name=clip_name,
+                    frame_idx=i,
+                    error_group_label=YTASL_GROUP_ERROR_LABELS[group_name],
+                    include_size_details=False,
+                    strict_key_access=True,
+                )
+
+            confs.append(conf)
+
+        kps_with_scores = load_part_kp_YTASL(skeletons, confs, self.normalization, self.layout)
+        return kps_with_scores
+
+    def __len__(self):
+        return len(self.list_data)
+
+    def __str__(self):
+        return f'#total {len(self)}'
+
+
+class S2T_Dataset_Isharah(S2T_Dataset_YTASL):
+    def __init__(self, path, args, phase):
+        super(S2T_Dataset_Isharah, self).__init__(path=path, args=args, phase=phase)
+
+    def load_pose(self, clip_name):
+        path = os.path.join(self.pose_dir, f"{clip_name}.json")
+        pose_data = load_json(path)
+        pose = pose_data['cropped_keypoints']
+
+        duration = len(pose)
+        tmp = select_frame_indices(duration, self.max_length, self.phase)
+        tmp = np.array(tmp)
+        skeletons = [pose[i] for i in tmp]
+
+        confs = []
+        for i, skeleton in enumerate(skeletons):
+            conf = {}
+            for group_name, expected_size in ISHARAH_GROUP_SIZES.items():
+                _fill_missing_landmarks(
+                    skeleton=skeleton,
+                    conf=conf,
+                    group_name=group_name,
+                    expected_size=expected_size,
+                    clip_name=clip_name,
+                    frame_idx=i,
+                )
+            confs.append(conf)
+
+        kps_with_scores = load_part_kp_Isharah(skeletons, confs, self.normalization, self.layout)
+        return kps_with_scores
+
+
+# class S2T_Dataset_YTASL_h5(Base_Dataset):
+#     def __init__(self, path, args, phase):
+#         super(S2T_Dataset_YTASL_h5, self).__init__()
+#         self.args = args
+#         self.max_length = args.max_length
+#         self.phase = phase
+#         self.annotation = load_json(path)
+#         self.rgb_support = self.args.rgb_support
+#
+#         # Load poses
+#         self.list_data = []  # [(video_id, clip_id), ...]
+#         self.h5_data = {}
+#         self.h5shard = defaultdict(lambda: defaultdict(dict))
+#         self.clip_order_to_int = {}
+#         self.clip_order_from_int = {}
+#
+#         for video_id in self.annotation.keys():
+#             co = self.annotation[video_id]['clip_order']
+#             self.clip_order_from_int[video_id] = dict(zip(range(len(co)), co))
+#             self.clip_order_to_int[video_id] = dict(zip(co, range(len(co))))
+#
+#         for video_id, clip_dict in self.annotation.items():
+#             for clip_name in clip_dict:
+#                 if clip_name != "clip_order":
+#                     self.list_data.append((video_id, self.clip_order_to_int[video_id][clip_name]))
+#
+#         self.vf_path = os.path.join(pose_dirs["YTASL"], "YouTubeASL.keypoints.{}.json".format(phase))
+#
+#         h5_video_clip = self.read_multih5_json(self.vf_path)
+#         self.remove_missing_annotation(h5_video_clip)  # Remove data in annotations that are missing in h5 file
+#
+#     def read_multih5_json(self, json_filename):
+#         """Helper function for reading json specifications of multiple H5 files for visual features"""
+#         h5_video_clip = set()
+#         with open(json_filename, 'r') as F:
+#             self.h5shard = json.load(F)
+#         self.h5_data = {}
+#         print(f"Pose {self.phase} data are loaded from: ")
+#         for k in set(self.h5shard.values()):
+#             h5file = json_filename.replace('metadata_', '').replace('.json', ".%s.h5" % k)
+#             print("--" + h5file)  # ,k,json_filename,data_dir)
+#             self.h5_data[k] = h5py.File(h5file, 'r')
+#
+#             for vi in self.h5_data[k].keys():
+#                 for ci in self.h5_data[k][vi].keys():
+#                     if vi in self.clip_order_to_int:
+#                         if ci in self.clip_order_to_int[vi]:
+#                             clip_id = self.clip_order_to_int[vi][ci]
+#                             h5_video_clip.add((vi, clip_id))
+#         return h5_video_clip
+#
+#     def remove_missing_annotation(self, h5_video_clip):
+#         annotations_to_delete = set(self.list_data) - h5_video_clip
+#         for a in annotations_to_delete:
+#             self.list_data.remove(a)
+#
+#     def __getitem__(self, index):
+#         video_id, clip_id = self.list_data[index]
+#         clip_name = self.clip_order_from_int[video_id][clip_id]
+#
+#         # Get the pose features
+#         shard = self.h5shard[video_id]
+#         pose_features = torch.tensor(np.array(self.h5_data[shard][video_id][clip_name]))
+#
+#         # TODO: rgb support
+#         video_path = ""
+#
+#         # Get translation
+#         clip_dict = self.annotation[video_id][clip_name]
+#         translation = clip_dict['translation']
+#
+#         # sample = {"name": clip_name,
+#         #           "video_path": video_path,
+#         #           "pose_features": pose_features,
+#         #           "text": translation}
+#         #
+#         # # Crop long sequences to desired max length. Random sample
+#         # duration = len(pose_features) # TODO: works?
+#         # if duration > self.max_length:
+#         #     tmp = sorted(random.sample(range(duration), k=self.max_length))
+#         # else:
+#         #     tmp = list(range(duration))
+#         #
+#         # tmp = np.array(tmp)
+#
+#         # skeletons = pose['keypoints']
+#         # confs = pose['scores']
+#         # skeletons_tmp = []
+#         # confs_tmp = []
+#         # for index in tmp:
+#         #     skeletons_tmp.append(skeletons[index])
+#         #     confs_tmp.append(confs[index])
+#         #
+#         # skeletons = skeletons_tmp
+#         # confs = confs_tmp
+#
+#         # confs = [np.ones(int(pose_features.shape[1]/2)) for _ in range(pose_features.shape[0])]
+#         # confs = [np.ones(pose_features.shape[0])] * pose_features.shape[1]
+#         # skeletons = [] # List of ndarrays (133,2) - full keypoints
+#         # kps_with_scores = load_part_kp(skeletons, confs, force_ok=True)
+#
+#         # decoded = self.tokenizer(
+#         #     translation,
+#         #     max_length=self.max_token_length,
+#         #     padding="max_length",
+#         #     truncation=True,
+#         #     return_tensors="pt",
+#         # )
+#         # labels = decoded.input_ids
+#
+#         # Skip frames for the keypoints
+#         # if self.skip_frames:
+#         #     if type(self.skip_frames) == bool:
+#         #         for input_type in INPUT_TYPES:
+#         #             if visual_features[input_type] is not None:
+#         #                 visual_features[input_type] = visual_features[input_type][::2]
+#         #     elif type(self.skip_frames) == int:
+#         #         for input_type in INPUT_TYPES:
+#         #             if visual_features[input_type] is not None:
+#         #                 visual_features[input_type] = visual_features[input_type][::self.skip_frames]
+#         #
+#         # # Trim the keypoints to the max sequence length
+#         # if self.max_sequence_length:
+#         #     for input_type in INPUT_TYPES:
+#         #         if visual_features[input_type] is not None:
+#         #             visual_features[input_type] = visual_features[input_type][: self.max_sequence_length]
+#         #             seq_len = len(visual_features[input_type])
+#         #
+#         # assert seq_len, "No modality provided or clip has no length!"
+#         # attention_mask = torch.ones(seq_len)
+#         #
+#         # return {
+#         #     "sign_inputs": {'pose': visual_features['pose'],
+#         #                     'mae': visual_features['mae'],
+#         #                     'dino': visual_features['dino'],
+#         #                     'sign2vec': visual_features['sign2vec']},
+#         #     "sentence": translation,
+#         #     "labels": labels,
+#         #     "attention_mask": attention_mask,
+#         # }
+#         return kps
+#
+#     def __len__(self):
+#         return len(self.list_data)
+#
+#     def __str__(self):
+#         return f'#total {len(self)}'
+
+
+class S2T_Dataset_news(Base_Dataset):
+    def __init__(self, path, args, phase):
+        super(S2T_Dataset_news, self).__init__()
+        self.args = args
+        self.rgb_support = self.args.rgb_support
+        self.phase = phase
+        self.max_length = args.max_length
+
+        path = pathlib.Path(path)
+
+        with path.open(encoding='utf-8') as f:
+            self.annotation = json.load(f)
+
+        if self.args.dataset == "CSL_News":
+            self.pose_dir = pose_dirs[args.dataset]
+            self.rgb_dir = rgb_dirs[args.dataset]
+
+        else:
+            raise NotImplementedError
+        sum_sample = len(self.annotation)
+        self.data_transform = transforms.Compose([
+            transforms.ToTensor(),
+            transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
+        ])
+
+        if phase == 'train':
+            self.start_idx = int(sum_sample * 0.0)
+            self.end_idx = int(sum_sample * 0.99)
+        else:
+            self.start_idx = int(sum_sample * 0.99)
+            self.end_idx = int(sum_sample)
+
+    def __len__(self):
+        return self.end_idx - self.start_idx
+
+    def __getitem__(self, index):
+        num_retries = 10
+
+        # skip some invalid video sample
+        for _ in range(num_retries):
+            sample = self.annotation[self.start_idx:self.end_idx][index]
+
+            text = sample['text']
+            name_sample = sample['video']
+
+            try:
+                pose_sample, support_rgb_dict = self.load_pose(sample['pose'], sample['video'])
+
+            except:
+                import traceback
+
+                traceback.print_exc()
+                print(f"Failed to load examples with video: {name_sample}. "
+                      f"Will randomly sample an example as a replacement.")
+                index = random.randint(0, len(self) - 1)
+                continue
+
+            break
+
+        else:
+            raise RuntimeError(f"Failed to fetch video after {num_retries} retries.")
+
+        return name_sample, pose_sample, text, _, support_rgb_dict
+
+    def load_pose(self, pose_name, rgb_name):
+        pose = pickle.load(open(os.path.join(self.pose_dir, pose_name), 'rb'))
+        full_path = os.path.join(self.rgb_dir, rgb_name)
+
+        duration = len(pose['scores'])
+
+        tmp = select_frame_indices(duration, self.max_length, self.phase)
+
+        tmp = np.array(tmp)
+
+        # dict_keys(['keypoints', 'scores'])
+        # keypoints (1, 133, 2)
+        # scores (1, 133)
+
+        skeletons = pose['keypoints']
+        confs = pose['scores']
+        skeletons_tmp = []
+        confs_tmp = []
+
+        for index in tmp:
+            skeletons_tmp.append(skeletons[index])
+            confs_tmp.append(confs[index])
+
+        skeletons = skeletons_tmp
+        confs = confs_tmp
+
+        kps_with_scores = load_part_kp(skeletons, confs)
+
+        support_rgb_dict = {}
+        if self.rgb_support:
+            support_rgb_dict = load_support_rgb_dict(tmp, skeletons, confs, full_path, self.data_transform)
+
+        return kps_with_scores, support_rgb_dict
+
+    def __str__(self):
+        return f'#total {len(self)}'

Uni_Sign/deformable_attention_2d.py

@@ -0,0 +1,311 @@
+# Clone from https://github.com/lucidrains/deformable-attention
+import torch
+import torch.nn.functional as F
+from torch import nn, einsum
+
+from einops import rearrange, repeat
+from einops.layers.torch import Rearrange
+# helper functions
+from typing import Any, Optional, Tuple, Type
+import numpy as np
+
+def exists(val):
+    return val is not None
+
+def default(val, d):
+    return val if exists(val) else d
+
+def divisible_by(numer, denom):
+    return (numer % denom) == 0
+
+# tensor helpers
+
+def create_grid_like(t, dim = 0):
+    h, w, device = *t.shape[-2:], t.device
+
+    grid = torch.stack(torch.meshgrid(
+        torch.arange(w, device = device),
+        torch.arange(h, device = device),
+    indexing = 'xy'), dim = dim)
+
+    grid.requires_grad = False
+    grid = grid.type_as(t)
+    return grid
+
+def normalize_grid(grid, dim = 1, out_dim = -1):
+    # normalizes a grid to range from -1 to 1
+    h, w = grid.shape[-2:]
+    grid_h, grid_w = grid.unbind(dim = dim)
+
+    grid_h = 2.0 * grid_h / max(h - 1, 1) - 1.0
+    grid_w = 2.0 * grid_w / max(w - 1, 1) - 1.0
+
+    return torch.stack((grid_h, grid_w), dim = out_dim)
+
+def reshape_grid_1d(grid, dim = 1, out_dim = -1):
+    # normalizes a grid to range from -1 to 1
+    n = grid.shape[-1]
+    grid_h, grid_w = grid.unbind(dim = dim)
+
+    return torch.stack((grid_h, grid_w), dim = out_dim)
+
+class Scale(nn.Module):
+    def __init__(self, scale):
+        super().__init__()
+        self.scale = scale
+
+    def forward(self, x):
+        return x * self.scale
+
+# continuous positional bias from SwinV2
+
+class CPB(nn.Module):
+    """ https://arxiv.org/abs/2111.09883v1 """
+
+    def __init__(self, dim, *, heads, offset_groups, depth):
+        super().__init__()
+        self.heads = heads
+        self.offset_groups = offset_groups
+
+        self.mlp = nn.ModuleList([])
+
+        self.mlp.append(nn.Sequential(
+            nn.Linear(2, dim),
+            nn.ReLU()
+        ))
+
+        for _ in range(depth - 1):
+            self.mlp.append(nn.Sequential(
+                nn.Linear(dim, dim),
+                nn.ReLU()
+            ))
+
+        self.mlp.append(nn.Linear(dim, heads // offset_groups))
+
+    def forward(self, grid_q, grid_kv):
+        device, dtype = grid_q.device, grid_kv.dtype
+
+        if grid_q.ndim == 3:
+            raise AttributeError
+            grid_q = rearrange(grid_q, 'h w c -> 1 (h w) c')
+        elif grid_q.ndim == 4:
+            grid_q = rearrange(grid_q, 'b h w c -> b (h w) c')
+        else:
+            raise AttributeError
+        grid_kv = rearrange(grid_kv, 'b h w c -> b (h w) c')
+
+        pos = rearrange(grid_q, 'b i c -> b i 1 c') - rearrange(grid_kv, 'b j c -> b 1 j c')
+        bias = torch.sign(pos) * torch.log(pos.abs() + 1)  # log of distance is sign(rel_pos) * log(abs(rel_pos) + 1)
+
+        for layer in self.mlp:
+            bias = layer(bias)
+
+        bias = rearrange(bias, '(b g) i j o -> b (g o) i j', g = self.offset_groups)
+
+        return bias
+
+
+class PositionEmbeddingRandom(nn.Module):
+    """
+    Positional encoding using random spatial frequencies.
+    """
+
+    def __init__(self, num_pos_feats: int = 64, scale: Optional[float] = None) -> None:
+        super().__init__()
+        if scale is None or scale <= 0.0:
+            scale = 1.0
+        self.register_buffer(
+            "positional_encoding_gaussian_matrix",
+            scale * torch.randn((2, num_pos_feats)),
+        )
+
+    def _pe_encoding(self, coords: torch.Tensor) -> torch.Tensor:
+        """Positionally encode points that are normalized to [0,1]."""
+        # assuming coords are in [0, 1]^2 square and have d_1 x ... x d_n x 2 shape
+        coords = 2 * coords - 1
+        coords = coords @ self.positional_encoding_gaussian_matrix
+        coords = 2 * np.pi * coords
+        # outputs d_1 x ... x d_n x C shape
+        return torch.cat([torch.sin(coords), torch.cos(coords)], dim=-1)
+
+    def forward(self, size: Tuple[int, int]) -> torch.Tensor:
+        """Generate positional encoding for a grid of the specified size."""
+        h, w = size
+        device: Any = self.positional_encoding_gaussian_matrix.device
+        grid = torch.ones((h, w), device=device, dtype=torch.float32)
+        y_embed = grid.cumsum(dim=0) - 0.5
+        x_embed = grid.cumsum(dim=1) - 0.5
+        y_embed = y_embed / h
+        x_embed = x_embed / w
+
+        pe = self._pe_encoding(torch.stack([x_embed, y_embed], dim=-1))
+        return pe.permute(2, 0, 1)  # C x H x W
+
+    def forward_with_coords(
+        self, coords_input: torch.Tensor, image_size: Tuple[int, int]
+    ) -> torch.Tensor:
+        """Positionally encode points that are not normalized to [0,1]."""
+        coords = coords_input.clone()
+        coords[:, :, 0] = coords[:, :, 0] / image_size[1]
+        coords[:, :, 1] = coords[:, :, 1] / image_size[0]
+        return self._pe_encoding(coords.to(torch.float))  # B x N x C
+
+def get_sinusoid_encoding_table(n_position, d_hid): 
+    ''' Sinusoid position encoding table ''' 
+    # TODO: make it with torch instead of numpy 
+    def get_position_angle_vec(position): 
+        return [position / np.power(10000, 2 * (hid_j // 2) / d_hid) for hid_j in range(d_hid)] 
+
+    sinusoid_table = np.array([get_position_angle_vec(pos_i) for pos_i in range(n_position)]) 
+    sinusoid_table[:, 0::2] = np.sin(sinusoid_table[:, 0::2]) # dim 2i 
+    sinusoid_table[:, 1::2] = np.cos(sinusoid_table[:, 1::2]) # dim 2i+1 
+
+    return torch.tensor(sinusoid_table,dtype=torch.float, requires_grad=False).unsqueeze(0) 
+
+# main class
+
+class DeformableAttention2D(nn.Module):
+    def __init__(
+        self,
+        *,
+        dim,
+        dim_head = 64,
+        heads = 8,
+        dropout = 0.,
+        downsample_factor = 1,
+        offset_scale = None,
+        offset_groups = None,
+        offset_kernel_size = 1,
+        group_queries = True,
+        group_key_values = True
+    ):
+        super().__init__()
+        offset_scale = default(offset_scale, downsample_factor)
+        assert offset_kernel_size >= downsample_factor, 'offset kernel size must be greater than or equal to the downsample factor'
+        assert divisible_by(offset_kernel_size - downsample_factor, 2)
+
+        offset_groups = default(offset_groups, heads)
+        assert divisible_by(heads, offset_groups)
+
+        inner_dim = dim_head * heads
+        self.scale = dim_head ** -0.5
+        self.heads = heads
+        self.offset_groups = offset_groups
+
+        offset_dims = inner_dim // offset_groups
+
+        self.downsample_factor = downsample_factor
+        
+        self.to_offsets = nn.Sequential(
+            nn.Conv1d(offset_dims, offset_dims, kernel_size=1, groups = offset_dims, stride = 1, padding = 0),
+            nn.GELU(),
+            nn.Conv1d(offset_dims, 2, 1, bias = False),
+            # Rearrange('b 1 n -> b n'),
+            nn.Tanh(),
+            Scale(4/6)
+        )
+
+        self.rel_pos_bias = CPB(dim // 4, offset_groups = offset_groups, heads = heads, depth = 2)
+
+        self.dropout = nn.Dropout(dropout)
+        self.to_q = nn.Conv1d(dim, inner_dim, 1, groups = offset_groups if group_queries else 1, bias = False)
+        self.to_k = nn.Conv2d(dim, inner_dim, 1, groups = offset_groups if group_key_values else 1, bias = False)
+        self.to_v = nn.Conv2d(dim, inner_dim, 1, groups = offset_groups if group_key_values else 1, bias = False)
+        self.to_out = nn.Conv1d(inner_dim, dim, 1)
+
+        self.cross_attn = nn.MultiheadAttention(embed_dim=dim, num_heads=heads, batch_first=True)
+        # for pose
+        self.pe_layer = PositionEmbeddingRandom(dim//2)
+        # for rgb
+        self.pos_embed = get_sinusoid_encoding_table(4*4, dim)
+
+    def forward(self, pose_feat, rgb_feat, pose_init, return_vgrid = False):
+        """
+        b - batch
+        h - heads
+        x - height
+        y - width
+        d - dimension
+        g - offset groups
+        """
+
+        heads, b, h, w, downsample_factor, device = self.heads, rgb_feat.shape[0], *rgb_feat.shape[-2:], self.downsample_factor, rgb_feat.device
+
+        # queries
+        # pose_feat: bt, c, n
+        pose_feat_cross = rearrange(pose_feat, 'b d n -> b n d')
+        rgb_feat_cross = rearrange(rgb_feat, 'b d h w -> b (h w) d')
+        pose_init_cross = rearrange(pose_init, 'b d n -> b n d')
+
+        point_embedding = self.pe_layer._pe_encoding(pose_init_cross.detach())
+
+        kv = rgb_feat_cross + self.pos_embed.expand(b, -1, -1).type_as(rgb_feat_cross).to(rgb_feat_cross.device).clone().detach()
+        
+        pose_feat_cross, _ = self.cross_attn(pose_feat_cross + point_embedding, 
+                                          kv, kv)
+
+        pose_feat_cross = rearrange(pose_feat_cross, 'b n d -> b d n')
+
+        q = self.to_q(pose_feat + pose_feat_cross)
+
+        # calculate offsets - offset MLP shared across all groups
+
+        group_1d = lambda t: rearrange(t, 'b (g d) n -> (b g) d n', g = self.offset_groups)
+        group_2d = lambda t: rearrange(t, 'b (g d) ... -> (b g) d ...', g = self.offset_groups)
+
+        grouped_queries = group_1d(q)
+        
+        offsets = self.to_offsets(grouped_queries)
+
+        # calculate grid + offsets
+
+        # pose_init --> [0, 1]
+        grid = pose_init[:,None].repeat(1, self.offset_groups, 1, 1) * 2 - 1
+        # grid --> [-1, 1]
+        grid = grid.reshape(-1, *pose_init.shape[-2:])
+
+        vgrid = grid + offsets
+        vgrid_scaled = reshape_grid_1d(vgrid)[:,None]
+        # vgrid_scaled = normalize_grid(vgrid)
+
+        kv_feats = F.grid_sample(
+            group_2d(rgb_feat),
+            vgrid_scaled,
+        mode = 'bilinear', padding_mode = 'zeros', align_corners = False)
+
+        kv_feats = rearrange(kv_feats, '(b g) d ... -> b (g d) ...', b = b)
+
+        # derive key / values
+        k, v = self.to_k(kv_feats), self.to_v(kv_feats)
+
+        # scale queries
+        q = q * self.scale
+
+        # split out heads
+        q, k, v = map(lambda t: rearrange(t, 'b (h d) ... -> b h (...) d', h = heads), (q, k, v))
+
+        # query / key similarity
+        sim = einsum('b h i d, b h j d -> b h i j', q, k)
+
+        # relative positional bias
+        grid = reshape_grid_1d(grid)[:,None]
+
+        rel_pos_bias = self.rel_pos_bias(grid, vgrid_scaled)
+        sim = sim + rel_pos_bias
+
+        # numerical stability
+        sim = sim - sim.amax(dim = -1, keepdim = True).detach()
+
+        # attention
+        attn = sim.softmax(dim = -1)
+        attn = self.dropout(attn)
+
+        # aggregate and combine heads
+        out = einsum('b h i j, b h j d -> b h i d', attn, v)
+        out = rearrange(out, 'b h n d -> b (h d) n')
+        out = self.to_out(out)
+
+        if return_vgrid:
+            return out, vgrid
+
+        return out

Uni_Sign/models.py

@@ -0,0 +1,415 @@
+from torch import Tensor
+import torch
+from torch import nn
+import torch.utils.checkpoint
+import contextlib
+import torchvision
+from einops import rearrange
+
+import math
+from Uni_Sign.stgcn_layers import Graph, get_stgcn_chain
+from Uni_Sign.deformable_attention_2d import DeformableAttention2D
+from transformers import MT5ForConditionalGeneration, T5Tokenizer, MT5Config
+import warnings
+
+mt5_path = r"./Uni_Sign/unisign_model"
+
+def _no_grad_trunc_normal_(tensor, mean, std, a, b):
+    # Cut & paste from PyTorch official master until it's in a few official releases - RW
+    # Method based on https://people.sc.fsu.edu/~jburkardt/presentations/truncated_normal.pdf
+    def norm_cdf(x):
+        # Computes standard normal cumulative distribution function
+        return (1. + math.erf(x / math.sqrt(2.))) / 2.
+
+    if (mean < a - 2 * std) or (mean > b + 2 * std):
+        warnings.warn("mean is more than 2 std from [a, b] in nn.init.trunc_normal_. "
+                      "The distribution of values may be incorrect.",
+                      stacklevel=2)
+
+    with torch.no_grad():
+        # Values are generated by using a truncated uniform distribution and
+        # then using the inverse CDF for the normal distribution.
+        # Get upper and lower cdf values
+        l = norm_cdf((a - mean) / std)
+        u = norm_cdf((b - mean) / std)
+
+        # Uniformly fill tensor with values from [l, u], then translate to
+        # [2l-1, 2u-1].
+        tensor.uniform_(2 * l - 1, 2 * u - 1)
+
+        # Use inverse cdf transform for normal distribution to get truncated
+        # standard normal
+        tensor.erfinv_()
+
+        # Transform to proper mean, std
+        tensor.mul_(std * math.sqrt(2.))
+        tensor.add_(mean)
+
+        # Clamp to ensure it's in the proper range
+        tensor.clamp_(min=a, max=b)
+        return tensor
+
+
+def trunc_normal_(tensor, mean=0., std=1., a=-2., b=2.):
+    # type: (Tensor, float, float, float, float) -> Tensor
+    r"""Fills the input Tensor with values drawn from a truncated
+    normal distribution. The values are effectively drawn from the
+    normal distribution :math:`\mathcal{N}(\text{mean}, \text{std}^2)`
+    with values outside :math:`[a, b]` redrawn until they are within
+    the bounds. The method used for generating the random values works
+    best when :math:`a \leq \text{mean} \leq b`.
+    Args:
+        tensor: an n-dimensional `torch.Tensor`
+        mean: the mean of the normal distribution
+        std: the standard deviation of the normal distribution
+        a: the minimum cutoff value
+        b: the maximum cutoff value
+    Examples:
+        >>> w = torch.empty(3, 5)
+        >>> nn.init.trunc_normal_(w)
+    """
+    return _no_grad_trunc_normal_(tensor, mean, std, a, b)
+
+class Uni_Sign(nn.Module):
+    def __init__(self, args):
+        super(Uni_Sign, self).__init__()
+        self.args = args
+        
+        self.modes = ['body', 'left', 'right', 'face_all']
+        
+        self.graph, A = {}, []
+        # project (x,y,score) to hidden dim
+        hidden_dim = args.hidden_dim
+        self.proj_linear = nn.ModuleDict()
+        for mode in self.modes:
+            graph_layout = f'{args.layout}_ytasl_{mode}' if self.args.dataset in ["YTASL", "Isharah"] else f'{args.layout}_{mode}'
+            self.graph[mode] = Graph(layout=graph_layout, strategy='distance', max_hop=1)
+            A.append(torch.tensor(self.graph[mode].A, dtype=torch.float32, requires_grad=False))
+            self.proj_linear[mode] = nn.Linear(3, 64)
+
+        self.gcn_modules = nn.ModuleDict()
+        self.fusion_gcn_modules = nn.ModuleDict()
+        spatial_kernel_size = A[0].size(0)
+        for index, mode in enumerate(self.modes):
+            self.gcn_modules[mode], final_dim = get_stgcn_chain(64, 'spatial', (1, spatial_kernel_size), A[index].clone(), adaptive=not self.args.no_adaptive_gcn)
+            self.fusion_gcn_modules[mode], _ = get_stgcn_chain(final_dim, 'temporal', (5, spatial_kernel_size), A[index].clone(), adaptive=not self.args.no_adaptive_gcn)
+
+        self.gcn_modules['left'] = self.gcn_modules['right']
+        self.fusion_gcn_modules['left'] = self.fusion_gcn_modules['right']
+        self.proj_linear['left'] = self.proj_linear['right']
+
+        self.part_para = nn.Parameter(torch.zeros(hidden_dim*len(self.modes)))
+        self.pose_proj = nn.Linear(256*4, 768)
+
+        self.apply(self._init_weights)
+        
+        if self.args.dataset == "Isharah":
+            self.lang = 'Arabic'
+        elif "CSL" in self.args.dataset:
+            self.lang = 'Chinese'
+        else:
+            self.lang = 'English'
+        
+        if self.args.rgb_support:
+            self.rgb_support_backbone = torch.nn.Sequential(*list(torchvision.models.efficientnet_b0(pretrained=True).children())[:-2])
+            self.rgb_proj = nn.Conv2d(1280, hidden_dim, kernel_size=1)
+
+            self.fusion_pose_rgb_linear = nn.Linear(hidden_dim, hidden_dim)
+            
+            # PGF
+            self.fusion_pose_rgb_DA = DeformableAttention2D(
+                                        dim = hidden_dim,            # feature dimensions
+                                        dim_head = 32,               # dimension per head
+                                        heads = 8,                   # attention heads
+                                        dropout = 0.,                # dropout
+                                        downsample_factor = 1,       # downsample factor (r in paper)
+                                        offset_scale = None,         # scale of offset, maximum offset
+                                        offset_groups = None,        # number of offset groups, should be multiple of heads
+                                        offset_kernel_size = 1,      # offset kernel size
+                                    )
+            
+            self.fusion_gate = nn.Sequential(nn.Conv1d(hidden_dim*2, hidden_dim, 1),
+                                        nn.GELU(),
+                                        nn.Conv1d(hidden_dim, 1, 1),
+                                        nn.Tanh(),
+                                        nn.ReLU(),
+                                    )
+            
+            for layer in self.fusion_gate:
+                try:
+                    if isinDataLoaderance(layer, nn.Conv1d):
+                        nn.init.constant_(layer.weight, 0)
+                        nn.init.constant_(layer.bias, 0)
+                except:
+                    print("NOT IMPLEMENTED...")
+
+        # Načte pouze strukturu architektury z config.json
+        mt5_config = MT5Config.from_pretrained(mt5_path)
+        # Vytvoří model s prázdnými vahami, které hned v dalším kroku přepíšeme
+        self.mt5_model = MT5ForConditionalGeneration(mt5_config)
+
+        self.mt5_tokenizer = T5Tokenizer.from_pretrained(mt5_path, legacy=False)
+
+        self.n_registers = args.n_registers
+        self.register_position = args.register_position
+        self.d_model = self.mt5_model.config.d_model  # should be 768
+
+        if self.n_registers > 0:
+            self.register_tokens = nn.Parameter(torch.zeros(self.n_registers, self.d_model))
+            # init like other embeddings
+            trunc_normal_(self.register_tokens, std=0.02)
+        else:
+            self.register_tokens = None
+    
+        
+    def _init_weights(self, m):
+        if isinstance(m, nn.Linear):
+            trunc_normal_(m.weight, std=.02)
+            if isinstance(m, nn.Linear) and m.bias is not None:
+                nn.init.constant_(m.bias, 0)
+        elif isinstance(m, nn.LayerNorm):
+            nn.init.constant_(m.bias, 0)
+            nn.init.constant_(m.weight, 1.0)
+
+    def maybe_autocast(self, dtype=torch.float32):
+        # if on cpu, don't use autocast
+        # if on gpu, use autocast with dtype if provided, otherwise use torch.float16
+        # enable_autocast = self.device != torch.device("cpu")
+        enable_autocast = True
+
+        if enable_autocast:
+            return torch.cuda.amp.autocast(dtype=dtype)
+        else:
+            return contextlib.nullcontext()
+
+    def gather_feat_pose_rgb(self, gcn_feat, rgb_feat, indices, rgb_len, pose_init):
+        b, c, T, n = gcn_feat.shape
+        assert rgb_feat.shape[0] == indices.shape[0]
+        rgb_feat = self.rgb_proj(rgb_feat)
+        
+        assert len(rgb_len) == b
+        start = 0
+        for batch in range(b):
+            index = indices[start:start + rgb_len[batch]].to(torch.long)
+            # ignore some invalid rgb clip
+            if rgb_len[batch] == 1 and -1 in index:
+                start = start + rgb_len[batch]
+                continue
+            
+            # index selection
+            gcn_feat_selected = gcn_feat[batch, :, index]
+            rgb_feat_selected = rgb_feat[start:start + rgb_len[batch]]
+            pose_init_selected = pose_init[start:start + rgb_len[batch]]
+            
+            gcn_feat_selected = rearrange(gcn_feat_selected, 'c t n -> t c n')
+            pose_init_selected = rearrange(pose_init_selected, 't n c -> t c n')
+            
+            # PGF forward
+            with self.maybe_autocast():
+                fused_transposed = self.fusion_pose_rgb_DA(pose_feat=gcn_feat_selected,
+                                                            rgb_feat=rgb_feat_selected, 
+                                                            pose_init=pose_init_selected, )
+            
+            fused_transposed = fused_transposed.to(gcn_feat.dtype)
+            gate_feature = torch.concat([fused_transposed, gcn_feat_selected,], dim=-2)
+            gate_score = self.fusion_gate(gate_feature)
+            fused_transposed_post = (gate_score) * fused_transposed + (1 - gate_score) * gcn_feat_selected
+            
+            gcn_feat = gcn_feat.clone() 
+            fused_transposed_post = rearrange(fused_transposed_post, 't c n -> c t n')
+            
+            # replace gcn feature
+            gcn_feat[batch, :, index] = fused_transposed_post
+            start = start + rgb_len[batch]
+            
+        assert start == rgb_feat.shape[0]
+        return gcn_feat
+
+    def forward(self, src_input, tgt_input):
+        # RGB branch forward
+        if self.args.rgb_support:
+            rgb_support_dict = {}
+            for index_key, rgb_key in zip(['left_sampled_indices', 'right_sampled_indices'], ['left_hands', 'right_hands']):
+                rgb_feat = self.rgb_support_backbone(src_input[rgb_key])
+                
+                rgb_support_dict[index_key] = src_input[index_key]
+                rgb_support_dict[rgb_key] = rgb_feat
+        
+        # Pose branch forward
+        features = []
+
+        body_feat = None
+        for part in self.modes:
+            # project position to hidden dim
+            proj_feat = self.proj_linear[part](src_input[part]).permute(0,3,1,2) #B,C,T,V
+            # spatial gcn forward
+            gcn_feat = self.gcn_modules[part](proj_feat)
+            if part == 'body':
+                body_feat = gcn_feat
+
+            else:
+                assert not body_feat is None
+                if part == 'left':
+                    # Pose RGB fusion
+                    if self.args.rgb_support:
+                        gcn_feat = self.gather_feat_pose_rgb(gcn_feat, 
+                                                            rgb_support_dict[f'{part}_hands'], 
+                                                            rgb_support_dict[f'{part}_sampled_indices'], 
+                                                            src_input[f'{part}_rgb_len'],
+                                                            src_input[f'{part}_skeletons_norm'],
+                                                            )
+                        
+                    gcn_feat = gcn_feat + body_feat[..., -2][...,None].detach()
+                    
+                elif part == 'right':
+                    # Pose RGB fusion
+                    if self.args.rgb_support:
+                        gcn_feat = self.gather_feat_pose_rgb(gcn_feat, 
+                                                                rgb_support_dict[f'{part}_hands'], 
+                                                                rgb_support_dict[f'{part}_sampled_indices'],
+                                                                src_input[f'{part}_rgb_len'],
+                                                                src_input[f'{part}_skeletons_norm'],
+                                                                )
+                        
+                    gcn_feat = gcn_feat + body_feat[..., -1][...,None].detach()
+
+                elif part == 'face_all':
+                    gcn_feat = gcn_feat + body_feat[..., 0][...,None].detach()
+
+                else:
+                    raise NotImplementedError
+            
+            # temporal gcn forward
+            gcn_feat = self.fusion_gcn_modules[part](gcn_feat) #B,C,T,V
+            pool_feat = gcn_feat.mean(-1).transpose(1,2) #B,T,C
+            features.append(pool_feat)
+        
+        # concat sub-pose feature across token dimension
+        inputs_embeds = torch.cat(features, dim=-1) + self.part_para
+        inputs_embeds = self.pose_proj(inputs_embeds)
+
+        prefix_token = self.mt5_tokenizer(
+                                [f"Translate sign language video to {self.lang}: "] * len(tgt_input["gt_sentence"]),
+                                padding="longest",
+                                truncation=True,
+                                return_tensors="pt",
+                            ).to(inputs_embeds.device)
+        
+        prefix_embeds = self.mt5_model.encoder.embed_tokens(prefix_token['input_ids'])
+
+        if self.n_registers > 0:
+            B = inputs_embeds.size(0)
+
+            # expand registers for batch
+            register_embeds = self.register_tokens.unsqueeze(0).expand(B, -1, -1)
+            # shape: (B, 4, 768)
+
+            register_mask = torch.ones((B, self.n_registers), device=inputs_embeds.device,dtype=prefix_token['attention_mask'].dtype)
+
+            if self.register_position == 'before_all':
+                # prepend order: [registers | prefix | pose_tokens]
+                inputs_embeds = torch.cat([register_embeds, prefix_embeds, inputs_embeds], dim=1)
+                attention_mask = torch.cat([register_mask, prefix_token['attention_mask'], src_input['attention_mask']], dim=1)
+
+            elif self.register_position == 'after_prefix' or self.register_position == 'before_pose':
+                # prepend order: [prefix | registers | pose_tokens]
+                inputs_embeds = torch.cat([prefix_embeds, register_embeds, inputs_embeds], dim=1)
+                attention_mask = torch.cat([prefix_token['attention_mask'], register_mask, src_input['attention_mask']], dim=1)
+
+            elif self.register_position == 'after_valid_pose':
+                # prepend order: [prefix | valid_pose_tokens | registers | padded_pose_tokens]
+                inputs_list = []
+                mask_list = []
+
+                for b in range(B):
+                    valid_len = int(src_input['attention_mask'][b].sum().item())
+
+                    pose_valid = inputs_embeds[b, :valid_len]
+                    pose_pad = inputs_embeds[b, valid_len:]
+
+                    emb = torch.cat(
+                        [prefix_embeds[b],
+                         pose_valid,
+                         register_embeds[b],
+                         pose_pad],
+                        dim=0
+                    )
+
+                    m = torch.cat(
+                        [prefix_token['attention_mask'][b],
+                         src_input['attention_mask'][b, :valid_len],
+                         register_mask[b],
+                         src_input['attention_mask'][b, valid_len:]],
+                        dim=0
+                    )
+
+                    inputs_list.append(emb)
+                    mask_list.append(m)
+
+                inputs_embeds = torch.stack(inputs_list, dim=0)
+                attention_mask = torch.stack(mask_list, dim=0)
+
+            elif self.register_position == 'after_all':
+                # prepend order: [prefix | pose_tokens | registers]
+                inputs_embeds = torch.cat([prefix_embeds, inputs_embeds, register_embeds], dim=1)
+                attention_mask = torch.cat([prefix_token['attention_mask'], src_input['attention_mask'], register_mask], dim=1)
+
+        else:
+            # prepend order: [prefix | pose_tokens]
+            inputs_embeds = torch.cat([prefix_embeds, inputs_embeds], dim=1)
+            attention_mask = torch.cat([prefix_token['attention_mask'], src_input['attention_mask']], dim=1)
+
+        tgt_input_tokenizer = self.mt5_tokenizer(tgt_input['gt_sentence'],
+                                                return_tensors="pt", 
+                                                padding=True,
+                                                truncation=True,
+                                                max_length=50)
+            
+        labels = tgt_input_tokenizer['input_ids']
+        labels[labels == self.mt5_tokenizer.pad_token_id] = -100
+        
+        out = self.mt5_model(inputs_embeds = inputs_embeds,
+                    attention_mask = attention_mask,
+                    labels = labels.to(inputs_embeds.device),
+                    return_dict = True,
+                    )
+        
+        label = labels.reshape(-1)
+        out_logits = out['logits']
+        logits = out_logits.reshape(-1,out_logits.shape[-1])
+        loss_fct = torch.nn.CrossEntropyLoss(label_smoothing=self.args.label_smoothing, ignore_index=-100)
+        loss = loss_fct(logits, label.to(out_logits.device, non_blocking=True))
+
+        stack_out = {
+            # use for inference
+            'inputs_embeds':inputs_embeds,
+            'attention_mask':attention_mask,
+            'loss':loss,
+        }
+
+        return stack_out
+    
+    @torch.no_grad()
+    def generate(self,pre_compute_item,max_new_tokens,num_beams):
+        inputs_embeds = pre_compute_item['inputs_embeds']
+        attention_mask = pre_compute_item['attention_mask']
+       
+        out = self.mt5_model.generate(inputs_embeds = inputs_embeds,
+                                attention_mask = attention_mask,
+                                max_new_tokens=max_new_tokens,
+                                num_beams = num_beams,
+                            )
+
+        return out
+
+def get_requires_grad_dict(model):
+    param_requires_grad = {name: True for name, param in model.named_parameters()}
+    param_requires_grad_right = {}
+    for key in param_requires_grad.keys():
+        if 'left' in key:
+            param_requires_grad_right[key.replace("left", 'right')] = param_requires_grad[key]
+    param_requires_grad = {**param_requires_grad,
+                           **param_requires_grad_right}
+    params_to_update = {k: v for k, v in model.state_dict().items() if param_requires_grad.get(k, True)}
+
+    return params_to_update

Uni_Sign/normalization.py

@@ -0,0 +1,117 @@
+import numpy as np
+from typing import Tuple
+
+
+def get_keypoints(joints, landmarks_name):
+    frames_keypoints = joints[landmarks_name]
+    frames_keypoints = frames_keypoints[:, :, :2]
+
+    return frames_keypoints
+
+
+def output_keypoints(joints, valid_frames, frames_keypoints):
+    frames_names = np.array(list(joints.keys()))[valid_frames]
+    frames_keypoints = frames_keypoints[valid_frames]
+
+    return dict(zip(frames_names, frames_keypoints))
+
+
+def safe_divide(a, b):
+    return np.divide(a, b, out=np.zeros_like(a), where=b != 0)
+
+
+def local_keypoint_normalization(joints: dict, landmarks: str, select_idx: list = [], padding: float = 0.1) -> dict:
+    frames_keypoints = get_keypoints(joints, landmarks)
+
+    if select_idx:
+        frames_keypoints = frames_keypoints[:, select_idx, :]
+
+    # move to origin
+    xmin = np.min(frames_keypoints[:, :, 0], axis=1)
+    ymin = np.min(frames_keypoints[:, :, 1], axis=1)
+
+    frames_keypoints[:, :, 0] -= xmin[:, np.newaxis]
+    frames_keypoints[:, :, 1] -= ymin[:, np.newaxis]
+
+    # pad to square
+    xmax = np.max(frames_keypoints[:, :, 0], axis=1)
+    ymax = np.max(frames_keypoints[:, :, 1], axis=1)
+
+    dif_full = np.abs(xmax - ymax)
+    dif = np.floor(dif_full / 2)
+
+    for i in range(len(dif)):
+        if xmax[i] > ymax[i]:
+            ymax[i] += dif_full[i]
+            frames_keypoints[i, :, 1] += dif[i]
+        else:
+            xmax[i] += dif_full[i]
+            frames_keypoints[i, :, 0] += dif[i]
+
+    # add padding to all sides
+    side_size = np.max([xmax, ymax], axis=0)
+    padding = side_size * padding
+
+    frames_keypoints += padding[:, np.newaxis, np.newaxis]
+    xmax += padding * 2
+    ymax += padding * 2
+
+    # normalize to [-1, 1]
+    frames_keypoints = safe_divide(frames_keypoints, xmax[:, np.newaxis, np.newaxis])
+    # frames_keypoints /= xmax[:, np.newaxis, np.newaxis]
+    frames_keypoints = frames_keypoints * 2 - 1
+
+    return frames_keypoints
+
+
+def global_keypoint_normalization(
+        joints: dict,
+        landmarks: str,
+        add_landmarks_names: list,
+        face_select_idx: list = [],
+        sign_area_size: tuple = (1.5, 1.5),
+        l_shoulder_idx: int = 11,
+        r_shoulder_idx: int = 12) -> Tuple[dict, dict]:
+    frames_keypoints = get_keypoints(joints, landmarks)
+
+    # get distance between right and left shoulder
+    l_shoulder_points = frames_keypoints[:, l_shoulder_idx, :]
+    r_shoulder_points = frames_keypoints[:, r_shoulder_idx, :]
+    distance = np.sqrt((l_shoulder_points[:, 0] - r_shoulder_points[:, 0]) ** 2 + (
+            l_shoulder_points[:, 1] - r_shoulder_points[:, 1]) ** 2)
+
+    # get center point between shoulders
+    center_x = np.abs(l_shoulder_points[:, 0] - r_shoulder_points[:, 0]) / 2 + np.min(
+        [l_shoulder_points[:, 0], r_shoulder_points[:, 0]], 0)
+    center_y = np.abs(l_shoulder_points[:, 1] - r_shoulder_points[:, 1]) / 2 + np.min(
+        [l_shoulder_points[:, 1], r_shoulder_points[:, 1]], 0)
+    sign_area_size = np.array(sign_area_size) * distance[:, np.newaxis]
+
+    # normalize
+    frames_keypoints[:, :, 0] -= center_x[:, np.newaxis]
+    frames_keypoints[:, :, 1] -= center_y[:, np.newaxis]
+
+    # frames_keypoints[:, :, 0] /= sign_area_size[:, 1, np.newaxis]
+    # frames_keypoints[:, :, 1] /= sign_area_size[:, 0, np.newaxis]
+    frames_keypoints[:, :, 0] = safe_divide(frames_keypoints[:, :, 0], sign_area_size[:, 0, np.newaxis])
+    frames_keypoints[:, :, 1] = safe_divide(frames_keypoints[:, :, 1], sign_area_size[:, 1, np.newaxis])
+
+    # normalize additional landmarks
+    add_landmarks = {}
+    for add_landmarks_name in add_landmarks_names:
+        add_frames_keypoints = get_keypoints(joints, add_landmarks_name)
+
+        if face_select_idx and add_landmarks_name == "face_landmarks":
+            add_frames_keypoints = add_frames_keypoints[:, face_select_idx, :]
+
+        add_frames_keypoints[:, :, 0] -= center_x[:, np.newaxis]
+        add_frames_keypoints[:, :, 1] -= center_y[:, np.newaxis]
+
+        # add_frames_keypoints[:, :, 0] /= sign_area_size[:, 1, np.newaxis]
+        # add_frames_keypoints[:, :, 1] /= sign_area_size[:, 0, np.newaxis]
+        add_frames_keypoints[:, :, 0] = safe_divide(add_frames_keypoints[:, :, 0], sign_area_size[:, 0, np.newaxis])
+        add_frames_keypoints[:, :, 1] = safe_divide(add_frames_keypoints[:, :, 1], sign_area_size[:, 1, np.newaxis])
+
+        add_landmarks[add_landmarks_name] = add_frames_keypoints
+
+    return frames_keypoints, add_landmarks

Uni_Sign/stgcn_layers/__init__.py

@@ -0,0 +1,2 @@
+from .gcn_utils import Graph
+from .stgcn_block import get_stgcn_chain

Uni_Sign/stgcn_layers/gcn_utils.py

@@ -0,0 +1,350 @@
+import torch
+import numpy as np
+import torch.nn as nn
+import pdb
+import math
+import copy
+
+
+class Graph:
+    """The Graph to model the skeletons extracted by the openpose
+
+    Args:
+        strategy (string): must be one of the follow candidates
+        - uniform: Uniform Labeling
+        - distance: Distance Partitioning
+        - spatial: Spatial Configuration
+        For more information, please refer to the section 'Partition Strategies'
+            in our paper (https://arxiv.org/abs/1801.07455).
+
+        layout (string): must be one of the follow candidates
+        - openpose: Is consists of 18 joints. For more information, please
+            refer to https://github.com/CMU-Perceptual-Computing-Lab/openpose#output
+        - ntu-rgb+d: Is consists of 25 joints. For more information, please
+            refer to https://github.com/shahroudy/NTURGB-D
+
+        max_hop (int): the maximal distance between two connected nodes
+        dilation (int): controls the spacing between the kernel points
+
+    """
+
+    def __init__(self, layout='custom', strategy='uniform', max_hop=1, dilation=1):
+        self.max_hop = max_hop
+        self.dilation = dilation
+
+        self.get_edge(layout)
+        self.hop_dis = get_hop_distance(self.num_node, self.edge, max_hop=max_hop)
+        self.get_adjacency(strategy)
+
+    def __str__(self):
+        return self.A
+
+    def get_edge(self, layout):
+        # 'body', 'left', 'right', 'mouth', 'face'
+        # if layout == 'custom_hand21':
+
+        if layout == 'default_left' or layout == 'default_right':
+            self.num_node = 21
+            self_link = [(i, i) for i in range(self.num_node)]
+            neighbor_1base = [
+                [0, 1],
+                [1, 2],
+                [2, 3],
+                [3, 4],
+                [0, 5],
+                [5, 6],
+                [6, 7],
+                [7, 8],
+                [0, 9],
+                [9, 10],
+                [10, 11],
+                [11, 12],
+                [0, 13],
+                [13, 14],
+                [14, 15],
+                [15, 16],
+                [0, 17],
+                [17, 18],
+                [18, 19],
+                [19, 20],
+            ]
+            neighbor_link = neighbor_1base
+            self.edge = self_link + neighbor_link
+            self.center = 0
+        
+        elif layout == 'default_body':
+            self.num_node = 9
+            self_link = [(i, i) for i in range(self.num_node)]
+            neighbor_1base = [
+                [0, 1],
+                [0, 2],
+                [0, 3],
+                [0, 4],
+                [3, 5],
+                [5, 7],
+                [4, 6],
+                [6, 8],
+            ]
+            neighbor_link = neighbor_1base
+            self.edge = self_link + neighbor_link
+            self.center = 0
+
+        elif layout == 'default_face_all':
+            self.num_node = 9 + 8 + 1
+            self_link = [(i, i) for i in range(self.num_node)]
+            neighbor_1base = [[i, i + 1] for i in range(9 - 1)] + \
+                             [[i, i + 1] for i in range(9, 9 + 8 - 1)] + \
+                             [[9 + 8 - 1, 9]] + \
+                             [[17, i] for i in range(17)]
+            neighbor_link = neighbor_1base
+            self.edge = self_link + neighbor_link
+            self.center = self.num_node - 1
+
+        elif layout in ['default_ytasl_left', 'default_ytasl_right', 'pruned_ytasl_left', 'pruned_ytasl_right', 'isharah_ytasl_left', 'isharah_ytasl_right']:
+            self.num_node = 21
+            self_link = [(i, i) for i in range(self.num_node)]
+            neighbor_1base = [
+                [3, 4],
+                [0, 5],
+                [17, 18],
+                [0, 17],
+                [13, 14],
+                [13, 17],
+                [18, 19],
+                [5, 6],
+                [5, 9],
+                [14, 15],
+                [0, 1],
+                [9, 10],
+                [1, 2],
+                [9, 13],
+                [10, 11],
+                [19, 20],
+                [6, 7],
+                [15, 16],
+                [2, 3],
+                [11, 12],
+                [7, 8]
+            ]
+            neighbor_link = neighbor_1base
+            self.edge = self_link + neighbor_link
+            self.center = 0
+
+        elif layout == 'default_ytasl_body':
+            self.num_node = 25
+            self_link = [(i, i) for i in range(self.num_node)]
+            neighbor_1base = [
+                [15, 21],
+                [16, 20],
+                [18, 20],
+                [3, 7],
+                [14, 16],
+                [11, 23],
+                [6, 8],
+                [15, 17],
+                [16, 22],
+                [4, 5],
+                [5, 6],
+                [12, 24],
+                [23, 24],
+                [0, 1],
+                [9, 10],
+                [1, 2],
+                [0, 4],
+                [11, 13],
+                [15, 19],
+                [16, 18],
+                [12, 14],
+                [17, 19],
+                [2, 3],
+                [11, 12],
+                [13, 15]
+            ]
+            neighbor_link = neighbor_1base
+            self.edge = self_link + neighbor_link
+            self.center = 0
+
+        elif layout == 'default_ytasl_face_all':
+            self.num_node = 37
+            self_link = [(i, i) for i in range(self.num_node)]
+            neighbor_1base = [
+                [16, 18],
+                [18, 13],
+                [13, 7],
+                [7, 8],
+                [8, 15],
+                [15, 24],
+                [24, 23],
+                [23, 29],
+                [29, 32],
+                [32, 16],
+                [5, 17],
+                [17, 14],
+                [30, 31],
+                [31, 21],
+                [11, 1],
+                [1, 27],
+                [10, 6],
+                [6, 0],
+                [0, 22],
+                [22, 26],
+                [26, 34],
+                [34, 4],
+                [4, 20],
+                [20, 10],
+                [10, 12],
+                [12, 2],
+                [2, 28],
+                [28, 26],
+                [10, 19],
+                [19, 3],
+                [3, 33],
+                [33, 26]
+            ]
+            neighbor_link = neighbor_1base
+            self.edge = self_link + neighbor_link
+            self.center = self.num_node - 1
+
+        elif layout in ['pruned_ytasl_body', 'isharah_ytasl_body']:
+            self.num_node = 9
+            self_link = [(i, i) for i in range(self.num_node)]
+            neighbor_1base = [
+                [0, 1],
+                [0, 2],
+                [0, 3],
+                [0, 4],
+                [3, 5],
+                [4, 6],
+                [5, 7],
+                [6, 8],
+            ]
+            neighbor_link = neighbor_1base
+            self.edge = self_link + neighbor_link
+            self.center = 0
+
+        elif layout == 'pruned_ytasl_face_all':
+            self.num_node = 18
+            self_link = [(i, i) for i in range(self.num_node)]
+            neighbor_1base = [
+                [5, 8],
+                [8, 6],
+                [6, 14],
+                [14, 12],
+                [3, 4],
+                [4, 1],
+                [1, 11],
+                [11, 10],
+                [3, 9],
+                [9, 2],
+                [2, 15],
+                [15, 10],
+                [7, 16],
+                [13, 17],
+            ]
+            neighbor_link = neighbor_1base
+            self.edge = self_link + neighbor_link
+            self.center = self.num_node - 1
+
+        elif layout == 'isharah_ytasl_face_all':
+            self.num_node = 19
+            self_link = [(i, i) for i in range(self.num_node)]
+            neighbor_1base = [
+                [0, 2],
+                [2, 3],
+                [3, 4],
+                [4, 10],
+                [10, 5],
+                [5, 8],
+                [8, 7],
+                [7, 9],
+                [9, 6],
+                [6, 1],
+                [1, 15],
+                [15, 18],
+                [18, 16],
+                [16, 17],
+                [17, 14],
+                [14, 13],
+                [13, 12],
+                [12, 11],
+                [11, 0],
+            ]
+            neighbor_link = neighbor_1base
+            self.edge = self_link + neighbor_link
+            self.center = self.num_node - 1
+
+        else:
+            raise NotImplementedError(f"Layout not implemented for: {layout}")
+
+    def get_adjacency(self, strategy):
+        valid_hop = range(0, self.max_hop + 1, self.dilation)
+        adjacency = np.zeros((self.num_node, self.num_node))
+        for hop in valid_hop:
+            adjacency[self.hop_dis == hop] = 1
+        normalize_adjacency = normalize_digraph(adjacency)
+
+        if strategy == 'uniform':
+            A = np.zeros((1, self.num_node, self.num_node))
+            A[0] = normalize_adjacency
+            self.A = A
+        elif strategy == 'distance':
+            A = np.zeros((len(valid_hop), self.num_node, self.num_node))
+            for i, hop in enumerate(valid_hop):
+                A[i][self.hop_dis == hop] = normalize_adjacency[self.hop_dis == hop]
+            self.A = A
+        elif strategy == 'spatial':
+            A = []
+            for hop in valid_hop:
+                a_root = np.zeros((self.num_node, self.num_node))
+                a_close = np.zeros((self.num_node, self.num_node))
+                a_further = np.zeros((self.num_node, self.num_node))
+                for i in range(self.num_node):
+                    for j in range(self.num_node):
+                        if self.hop_dis[j, i] == hop:
+                            if (
+                                self.hop_dis[j, self.center]
+                                == self.hop_dis[i, self.center]
+                            ):
+                                a_root[j, i] = normalize_adjacency[j, i]
+                            elif (
+                                self.hop_dis[j, self.center]
+                                > self.hop_dis[i, self.center]
+                            ):
+                                a_close[j, i] = normalize_adjacency[j, i]
+                            else:
+                                a_further[j, i] = normalize_adjacency[j, i]
+                if hop == 0:
+                    A.append(a_root)
+                else:
+                    A.append(a_root + a_close)
+                    A.append(a_further)
+            A = np.stack(A)
+            self.A = A
+        else:
+            raise ValueError("Do Not Exist This Strategy")
+
+
+def get_hop_distance(num_node, edge, max_hop=1):
+    A = np.zeros((num_node, num_node))
+    for i, j in edge:
+        A[j, i] = 1
+        A[i, j] = 1
+
+    # compute hop steps
+    hop_dis = np.zeros((num_node, num_node)) + np.inf
+    transfer_mat = [np.linalg.matrix_power(A, d) for d in range(max_hop + 1)]
+    arrive_mat = np.stack(transfer_mat) > 0
+    for d in range(max_hop, -1, -1):
+        hop_dis[arrive_mat[d]] = d
+    return hop_dis
+
+
+def normalize_digraph(A):
+    Dl = np.sum(A, 0)
+    num_node = A.shape[0]
+    Dn = np.zeros((num_node, num_node))
+    for i in range(num_node):
+        if Dl[i] > 0:
+            Dn[i, i] = Dl[i] ** (-1)
+    AD = np.dot(A, Dn)
+    return AD

Uni_Sign/stgcn_layers/stgcn_block.py

@@ -0,0 +1,128 @@
+import torch
+import numpy as np
+import torch.nn as nn
+import pdb
+import math
+import copy
+
+class GCN_unit(nn.Module):
+    def __init__(
+        self,
+        in_channels,
+        out_channels,
+        kernel_size,
+        A,
+        adaptive=True,
+        t_kernel_size=1,
+        t_stride=1,
+        t_padding=0,
+        t_dilation=1,
+        bias=True,
+    ):
+        super().__init__()
+        self.kernel_size = kernel_size
+        assert A.size(0) == self.kernel_size
+        self.conv = nn.Conv2d(
+            in_channels,
+            out_channels * kernel_size,
+            kernel_size=(t_kernel_size, 1),
+            padding=(t_padding, 0),
+            stride=(t_stride, 1),
+            dilation=(t_dilation, 1),
+            bias=bias,
+        )
+        self.adaptive = adaptive
+        # print(self.adaptive)
+        if self.adaptive:
+            self.A = nn.Parameter(A.clone())
+        else:
+            self.register_buffer('A', A)
+        self.bn = nn.BatchNorm2d(out_channels)
+        self.relu = nn.ReLU(inplace=True)
+
+    def forward(self, x, len_x):
+        x = self.conv(x)
+
+        n, kc, t, v = x.size()
+        x = x.view(n, self.kernel_size, kc // self.kernel_size, t, v)
+        x = torch.einsum('nkctv,kvw->nctw', (x, self.A)).contiguous()
+        y = self.bn(x)
+        y = self.relu(y)
+        return y
+
+class STGCN_block(nn.Module):
+    def __init__(
+        self,
+        in_channels,
+        out_channels,
+        kernel_size,
+        A,
+        adaptive=True,
+        stride=1,
+        dropout=0,
+        residual=True,
+    ):
+        super().__init__()
+
+        assert len(kernel_size) == 2
+        assert kernel_size[0] % 2 == 1
+        padding = ((kernel_size[0] - 1) // 2, 0)
+        self.gcn = GCN_unit(
+            in_channels,
+            out_channels,
+            kernel_size[1],
+            A,
+            adaptive=adaptive,
+        )
+        if kernel_size[0] > 1:
+            self.tcn = nn.Sequential(
+                nn.Conv2d(
+                    out_channels,
+                    out_channels,
+                    (kernel_size[0], 1),
+                    (stride, 1),
+                    padding,
+                ),
+                nn.BatchNorm2d(out_channels),
+                nn.Dropout(dropout, inplace=True),
+            )
+        else:
+            self.tcn = nn.Identity()
+
+        if not residual:
+            self.residual = lambda x: 0
+
+        elif (in_channels == out_channels) and (stride == 1):
+            self.residual = lambda x: x
+
+        else:
+            self.residual = nn.Sequential(
+                nn.Conv2d(in_channels, out_channels, kernel_size=1, stride=(stride, 1)),
+                nn.BatchNorm2d(out_channels),
+            )
+
+        self.relu = nn.ReLU(inplace=True)
+
+    def forward(self, x, len_x=None):
+        res = self.residual(x)
+        x = self.gcn(x, len_x)
+        x = self.tcn(x) + res
+        return self.relu(x)
+
+class STGCNChain(nn.Sequential):
+    def __init__(self, in_dim, block_args, kernel_size, A, adaptive):
+        super(STGCNChain, self).__init__()
+        last_dim = in_dim
+        for i, [channel, depth] in enumerate(block_args):
+            for j in range(depth):
+                self.add_module(f'layer{i}_{j}', STGCN_block(last_dim, channel, kernel_size, A.clone(), adaptive))
+                last_dim = channel
+
+def get_stgcn_chain(in_dim, level, kernel_size, A, adaptive):
+    if level == 'spatial':
+        block_args = [[64,1], [128,1], [256,1]]
+    elif level == 'temporal':
+        block_args = [[256,3]]
+    else:
+        raise NotImplementedError
+    return STGCNChain(in_dim, block_args, kernel_size, A, adaptive), block_args[-1][0]

Uni_Sign/unisign_model/config.json

@@ -0,0 +1,28 @@
+{
+  "_name_or_path": "/home/patrick/hugging_face/t5/mt5-base",
+  "architectures": [
+    "MT5ForConditionalGeneration"
+  ],
+  "d_ff": 2048,
+  "d_kv": 64,
+  "d_model": 768,
+  "decoder_start_token_id": 0,
+  "dropout_rate": 0.1,
+  "eos_token_id": 1,
+  "feed_forward_proj": "gated-gelu",
+  "initializer_factor": 1.0,
+  "is_encoder_decoder": true,
+  "layer_norm_epsilon": 1e-06,
+  "model_type": "mt5",
+  "num_decoder_layers": 12,
+  "num_heads": 12,
+  "num_layers": 12,
+  "output_past": true,
+  "pad_token_id": 0,
+  "relative_attention_num_buckets": 32,
+  "tie_word_embeddings": false,
+  "tokenizer_class": "T5Tokenizer",
+  "transformers_version": "4.10.0.dev0",
+  "use_cache": true,
+  "vocab_size": 250112
+}

Uni_Sign/unisign_model/generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "decoder_start_token_id": 0,
+  "eos_token_id": 1,
+  "pad_token_id": 0,
+  "transformers_version": "4.27.0.dev0"
+}

Uni_Sign/unisign_model/special_tokens_map.json

@@ -0,0 +1 @@
+{"eos_token": "</s>", "unk_token": "<unk>", "pad_token": "<pad>"}

Uni_Sign/unisign_model/spiece.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef78f86560d809067d12bac6c09f19a462cb3af3f54d2b8acbba26e1433125d6
+size 4309802

Uni_Sign/unisign_model/tokenizer_config.json

@@ -0,0 +1 @@
+{"eos_token": "</s>", "unk_token": "<unk>", "pad_token": "<pad>", "extra_ids": 0, "additional_special_tokens": null, "special_tokens_map_file": "/home/patrick/.cache/torch/transformers/685ac0ca8568ec593a48b61b0a3c272beee9bc194a3c7241d15dcadb5f875e53.f76030f3ec1b96a8199b2593390c610e76ca8028ef3d24680000619ffb646276", "tokenizer_file": null, "name_or_path": "google/mt5-small"}

checkpoints/pose/face_landmarker.task

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:64184e229b263107bc2b804c6625db1341ff2bb731874b0bcc2fe6544e0bc9ff
+size 3758596

checkpoints/pose/hand_landmarker.task

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fbc2a30080c3c557093b5ddfc334698132eb341044ccee322ccf8bcf3607cde1
+size 7819105

checkpoints/pose/pose_landmarker_full.task

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4eaa5eb7a98365221087693fcc286334cf0858e2eb6e15b506aa4a7ecdcec4ad
+size 9398198

checkpoints/pose/yolov8n-pose.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c6fa93dd1ee4a2c18c900a45c1d864a1c6f7aba75d84f91648a30b7fb641d212
+size 6832633