Upload 62 files

app.py

@@ -0,0 +1,72 @@
+import gradio as gr
+import base64
+from io import BytesIO
+from PIL import Image
+import numpy as np
+import os
+from pipeline.pipeline import pp_humanv2
+
+
+# UGC: Define the inference fn() for your models
+def model_inference(input_date, avtivity_list):
+    
+    if isinstance(input_date, str):
+        if  os.path.splitext(input_date)[-1] not in ['.avi','.mp4']:
+            return None
+            
+    if 'do_entrance_counting'in avtivity_list or 'draw_center_traj' in avtivity_list:
+        if 'MOT' not in avtivity_list:
+            avtivity_list.append('MOT')
+    
+    result = pp_humanv2(input_date, avtivity_list)
+
+    return result
+
+
+def clear_all():
+    return None, None, None
+
+
+with gr.Blocks() as demo:
+    gr.Markdown("PP-Human Pipeline")
+
+    with gr.Tabs():
+
+        with gr.TabItem("image"):
+
+            img_in = gr.Image(value="https://paddledet.bj.bcebos.com/modelcenter/images/PP-Human/human_attr.jpg",label="Input")
+            img_out = gr.Image(label="Output")
+
+            img_avtivity_list = gr.CheckboxGroup(["ATTR"])
+            img_button1 = gr.Button("Submit")
+            img_button2 = gr.Button("Clear")
+
+        with gr.TabItem("video"):
+
+            video_in = gr.Video(value="https://paddledet.bj.bcebos.com/modelcenter/images/PP-Human/human_attr.mp4",label="Input only support .mp4 or .avi")
+            video_out = gr.Video(label="Output")
+
+            video_avtivity_list = gr.CheckboxGroup(["MOT","ATTR","VIDEO_ACTION","SKELETON_ACTION","ID_BASED_DETACTION","ID_BASED_CLSACTION","REID",\
+                                                    "do_entrance_counting","draw_center_traj"],label="Task Choice (note: only one task should be checked)")
+            video_button1 = gr.Button("Submit")
+            video_button2 = gr.Button("Clear")
+
+    img_button1.click(
+        fn=model_inference,
+        inputs=[img_in, img_avtivity_list],
+        outputs=img_out)
+    img_button2.click(
+        fn=clear_all,
+        inputs=None,
+        outputs=[img_in, img_out, img_avtivity_list])
+
+    video_button1.click(
+        fn=model_inference,
+        inputs=[video_in, video_avtivity_list],
+        outputs=video_out)
+    video_button2.click(
+        fn=clear_all,
+        inputs=None,
+        outputs=[video_in, video_out, video_avtivity_list])
+
+demo.launch()

app.yml

@@ -0,0 +1,11 @@
+【PP-HumanV2-App-YAML】
+
+APP_Info:
+    title: PP-HumanV2-App
+    colorFrom: blue
+    colorTo: yellow
+    sdk: gradio
+    sdk_version: 3.9
+    app_file: app.py
+    license: apache-2.0
+    device: cpu

pipeline/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

pipeline/cfg_utils.py

@@ -0,0 +1,224 @@
+import ast
+import yaml
+import copy
+import argparse
+from argparse import ArgumentParser, RawDescriptionHelpFormatter
+
+
+class ArgsParser(ArgumentParser):
+    def __init__(self):
+        super(ArgsParser, self).__init__(
+            formatter_class=RawDescriptionHelpFormatter)
+        self.add_argument(
+            "-o", "--opt", nargs='*', help="set configuration options")
+
+    def parse_args(self, argv=None):
+        args = super(ArgsParser, self).parse_args(argv)
+        assert args.config is not None, \
+            "Please specify --config=configure_file_path."
+        args.opt = self._parse_opt(args.opt)
+        return args
+
+    def _parse_opt(self, opts):
+        config = {}
+        if not opts:
+            return config
+        for s in opts:
+            s = s.strip()
+            k, v = s.split('=', 1)
+            if '.' not in k:
+                config[k] = yaml.load(v, Loader=yaml.Loader)
+            else:
+                keys = k.split('.')
+                if keys[0] not in config:
+                    config[keys[0]] = {}
+                cur = config[keys[0]]
+                for idx, key in enumerate(keys[1:]):
+                    if idx == len(keys) - 2:
+                        cur[key] = yaml.load(v, Loader=yaml.Loader)
+                    else:
+                        cur[key] = {}
+                        cur = cur[key]
+        return config
+
+
+def argsparser():
+    parser = ArgsParser()
+
+    parser.add_argument(
+        "--config",
+        type=str,
+        default='pipeline/config/infer_cfg_pphuman.yml',
+        help=("Path of configure"))
+    parser.add_argument(
+        "--image_file", type=str, default=None, help="Path of image file.")
+    parser.add_argument(
+        "--image_dir",
+        type=str,
+        default=None,
+        help="Dir of image file, `image_file` has a higher priority.")
+    parser.add_argument(
+        "--video_file",
+        type=str,
+        default=None,
+        help="Path of video file, `video_file` or `camera_id` has a highest priority."
+    )
+    parser.add_argument(
+        "--video_dir",
+        type=str,
+        default=None,
+        help="Dir of video file, `video_file` has a higher priority.")
+    parser.add_argument(
+        "--rtsp",
+        type=str,
+        nargs='+',
+        default=None,
+        help="list of rtsp inputs, for one or multiple rtsp input.")
+    parser.add_argument(
+        "--camera_id",
+        type=int,
+        default=-1,
+        help="device id of camera to predict.")
+    parser.add_argument(
+        "--output_dir",
+        type=str,
+        default="output",
+        help="Directory of output visualization files.")
+    parser.add_argument(
+        "--pushurl",
+        type=str,
+        default="",
+        help="url of output visualization stream.")
+    parser.add_argument(
+        "--run_mode",
+        type=str,
+        default='paddle',
+        help="mode of running(paddle/trt_fp32/trt_fp16/trt_int8)")
+    parser.add_argument(
+        "--device",
+        type=str,
+        default='cpu',
+        help="Choose the device you want to run, it can be: CPU/GPU/XPU, default is CPU."
+    )
+    parser.add_argument(
+        "--enable_mkldnn",
+        type=ast.literal_eval,
+        default=False,
+        help="Whether use mkldnn with CPU.")
+    parser.add_argument(
+        "--cpu_threads", type=int, default=1, help="Num of threads with CPU.")
+    parser.add_argument(
+        "--trt_min_shape", type=int, default=1, help="min_shape for TensorRT.")
+    parser.add_argument(
+        "--trt_max_shape",
+        type=int,
+        default=1280,
+        help="max_shape for TensorRT.")
+    parser.add_argument(
+        "--trt_opt_shape",
+        type=int,
+        default=640,
+        help="opt_shape for TensorRT.")
+    parser.add_argument(
+        "--trt_calib_mode",
+        type=bool,
+        default=False,
+        help="If the model is produced by TRT offline quantitative "
+        "calibration, trt_calib_mode need to set True.")
+    parser.add_argument(
+        "--do_entrance_counting",
+        type=bool,
+        default=False,
+        help="Whether counting the numbers of identifiers entering "
+        "or getting out from the entrance. Note that only support single-class MOT."
+    )
+    parser.add_argument(
+        "--do_break_in_counting",
+        type=bool,
+        default=False,
+        help="Whether counting the numbers of identifiers break in "
+        "the area. Note that only support single-class MOT and "
+        "the video should be taken by a static camera.")
+    parser.add_argument(
+        "--illegal_parking_time",
+        type=int,
+        default=-1,
+        help="illegal parking time which units are seconds, default is -1 which means not recognition illegal parking"
+    )
+    parser.add_argument(
+        "--region_type",
+        type=str,
+        default='horizontal',
+        help="Area type for entrance counting or break in counting, 'horizontal' and "
+        "'vertical' used when do entrance counting. 'custom' used when do break in counting. "
+        "Note that only support single-class MOT, and the video should be taken by a static camera."
+    )
+    parser.add_argument(
+        '--region_polygon',
+        nargs='+',
+        type=int,
+        default=[],
+        help="Clockwise point coords (x0,y0,x1,y1...) of polygon of area when "
+        "do_break_in_counting. Note that only support single-class MOT and "
+        "the video should be taken by a static camera.")
+    parser.add_argument(
+        "--secs_interval",
+        type=int,
+        default=2,
+        help="The seconds interval to count after tracking")
+    parser.add_argument(
+        "--draw_center_traj",
+        type=bool,
+        default=False,
+        help="Whether drawing the trajectory of center")
+    parser.add_argument('--avtivity_list', nargs='+', type=str)
+
+    return parser
+
+
+def merge_cfg(args):
+    # load config
+    with open(args.config) as f:
+        pred_config = yaml.safe_load(f)
+
+    def merge(cfg, arg):
+        # update cfg from arg directly
+        merge_cfg = copy.deepcopy(cfg)
+        for k, v in cfg.items():
+            if k in arg:
+                merge_cfg[k] = arg[k]
+            else:
+                if isinstance(v, dict):
+                    merge_cfg[k] = merge(v, arg)
+
+        return merge_cfg
+
+    def merge_opt(cfg, arg):
+        merge_cfg = copy.deepcopy(cfg)
+        # merge opt
+        if 'opt' in arg.keys() and arg['opt']:
+            for name, value in arg['opt'].items(
+            ):  # example: {'MOT': {'batch_size': 3}}
+                if name not in merge_cfg.keys():
+                    print("No", name, "in config file!")
+                    continue
+                for sub_k, sub_v in value.items():
+                    if sub_k not in merge_cfg[name].keys():
+                        print("No", sub_k, "in config file of", name, "!")
+                        continue
+                    merge_cfg[name][sub_k] = sub_v
+
+        return merge_cfg
+
+    args_dict = vars(args)
+    pred_config = merge(pred_config, args_dict)
+    pred_config = merge_opt(pred_config, args_dict)
+
+    return pred_config
+
+
+def print_arguments(cfg):
+    print('-----------  Running Arguments -----------')
+    buffer = yaml.dump(cfg)
+    print(buffer)
+    print('------------------------------------------')

pipeline/config/infer_cfg_pphuman.yml

@@ -0,0 +1,63 @@
+crop_thresh: 0.5
+attr_thresh: 0.5
+kpt_thresh: 0.2
+visual: True
+warmup_frame: 50
+
+DET:
+  model_dir: https://bj.bcebos.com/v1/paddledet/models/pipeline/mot_ppyoloe_l_36e_pipeline.zip
+  batch_size: 1
+
+MOT:
+  model_dir: https://bj.bcebos.com/v1/paddledet/models/pipeline/mot_ppyoloe_l_36e_pipeline.zip
+  tracker_config: pipeline/config/tracker_config.yml
+  batch_size: 1
+  skip_frame_num: -1 # preferably no more than 3
+  enable: False
+
+KPT:
+  model_dir: https://bj.bcebos.com/v1/paddledet/models/pipeline/dark_hrnet_w32_256x192.zip
+  batch_size: 8
+
+ATTR:
+  model_dir:  https://bj.bcebos.com/v1/paddledet/models/pipeline/PPLCNet_x1_0_person_attribute_945_infer.zip
+  batch_size: 8
+  enable: False
+
+VIDEO_ACTION:
+  model_dir: https://videotag.bj.bcebos.com/PaddleVideo-release2.3/ppTSM_fight.zip
+  batch_size: 1
+  frame_len: 8
+  sample_freq: 7
+  short_size: 340
+  target_size: 320
+  enable: False
+
+SKELETON_ACTION:
+  model_dir: https://bj.bcebos.com/v1/paddledet/models/pipeline/STGCN.zip
+  batch_size: 1
+  max_frames: 50
+  display_frames: 80
+  coord_size: [384, 512]
+  enable: False
+
+ID_BASED_DETACTION:
+  model_dir:  https://bj.bcebos.com/v1/paddledet/models/pipeline/ppyoloe_crn_s_80e_smoking_visdrone.zip
+  batch_size: 8
+  threshold: 0.6
+  display_frames: 80
+  skip_frame_num: 2
+  enable: False
+
+ID_BASED_CLSACTION:
+  model_dir: https://bj.bcebos.com/v1/paddledet/models/pipeline/PPHGNet_tiny_calling_halfbody.zip
+  batch_size: 8
+  threshold: 0.8
+  display_frames: 80
+  skip_frame_num: 2
+  enable: False
+
+REID:
+  model_dir:  https://bj.bcebos.com/v1/paddledet/models/pipeline/reid_model.zip
+  batch_size: 16
+  enable: False

pipeline/config/tracker_config.yml

@@ -0,0 +1,28 @@
+# config of tracker for MOT SDE Detector, use 'JDETracker' as default.
+# The tracker of MOT JDE Detector (such as FairMOT) is exported together with the model.
+# Here 'min_box_area' and 'vertical_ratio' are set for pedestrian, you can modify for other objects tracking.
+
+type: OCSORTTracker # choose one tracker in ['JDETracker', 'OCSORTTracker']
+
+
+# BYTETracker
+JDETracker:
+  use_byte: True
+  det_thresh: 0.3
+  conf_thres: 0.6
+  low_conf_thres: 0.1
+  match_thres: 0.9
+  min_box_area: 0
+  vertical_ratio: 0 # 1.6 for pedestrian
+
+
+OCSORTTracker:
+  det_thresh: 0.4
+  max_age: 30
+  min_hits: 3
+  iou_threshold: 0.3
+  delta_t: 3
+  inertia: 0.2
+  vertical_ratio: 0
+  min_box_area: 0
+  use_byte: False

pipeline/datacollector.py

@@ -0,0 +1,132 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import copy
+from collections import Counter
+
+
+class Result(object):
+    def __init__(self):
+        self.res_dict = {
+            'det': dict(),
+            'mot': dict(),
+            'attr': dict(),
+            'kpt': dict(),
+            'video_action': dict(),
+            'skeleton_action': dict(),
+            'reid': dict(),
+            'det_action': dict(),
+            'cls_action': dict(),
+            'vehicleplate': dict(),
+            'vehicle_attr': dict()
+        }
+
+    def update(self, res, name):
+        self.res_dict[name].update(res)
+
+    def get(self, name):
+        if name in self.res_dict and len(self.res_dict[name]) > 0:
+            return self.res_dict[name]
+        return None
+
+    def clear(self, name):
+        self.res_dict[name].clear()
+
+
+class DataCollector(object):
+    """
+  DataCollector of Pipeline, collect results in every frames and assign it to each track ids.
+  mainly used in mtmct.
+  
+  data struct:
+  collector:
+    - [id1]: (all results of N frames)
+      - frames(list of int): Nx[int]
+      - rects(list of rect): Nx[rect(conf, xmin, ymin, xmax, ymax)]
+      - features(list of array(256,)): Nx[array(256,)]
+      - qualities(list of float): Nx[float]
+      - attrs(list of attr): refer to attrs for details
+      - kpts(list of kpts): refer to kpts for details
+      - skeleton_action(list of skeleton_action): refer to skeleton_action for details
+    ...
+    - [idN]
+  """
+
+    def __init__(self):
+        #id, frame, rect, score, label, attrs, kpts, skeleton_action
+        self.mots = {
+            "frames": [],
+            "rects": [],
+            "attrs": [],
+            "kpts": [],
+            "features": [],
+            "qualities": [],
+            "skeleton_action": [],
+            "vehicleplate": []
+        }
+        self.collector = {}
+
+    def append(self, frameid, Result):
+        mot_res = Result.get('mot')
+        attr_res = Result.get('attr')
+        kpt_res = Result.get('kpt')
+        skeleton_action_res = Result.get('skeleton_action')
+        reid_res = Result.get('reid')
+        vehicleplate_res = Result.get('vehicleplate')
+
+        rects = []
+        if reid_res is not None:
+            rects = reid_res['rects']
+        elif mot_res is not None:
+            rects = mot_res['boxes']
+
+        for idx, mot_item in enumerate(rects):
+            ids = int(mot_item[0])
+            if ids not in self.collector:
+                self.collector[ids] = copy.deepcopy(self.mots)
+            self.collector[ids]["frames"].append(frameid)
+            self.collector[ids]["rects"].append([mot_item[2:]])
+            if attr_res:
+                self.collector[ids]["attrs"].append(attr_res['output'][idx])
+            if kpt_res:
+                self.collector[ids]["kpts"].append([
+                    kpt_res['keypoint'][0][idx], kpt_res['keypoint'][1][idx]
+                ])
+            if skeleton_action_res and (idx + 1) in skeleton_action_res:
+                self.collector[ids]["skeleton_action"].append(
+                    skeleton_action_res[idx + 1])
+            else:
+                # action model generate result per X frames, Not available every frames
+                self.collector[ids]["skeleton_action"].append(None)
+            if reid_res:
+                self.collector[ids]["features"].append(reid_res['features'][
+                    idx])
+                self.collector[ids]["qualities"].append(reid_res['qualities'][
+                    idx])
+            if vehicleplate_res and vehicleplate_res['plate'][idx] != "":
+                self.collector[ids]["vehicleplate"].append(vehicleplate_res[
+                    'plate'][idx])
+
+    def get_res(self):
+        return self.collector
+
+    def get_carlp(self, trackid):
+        lps = self.collector[trackid]["vehicleplate"]
+        counter = Counter(lps)
+        carlp = counter.most_common()
+        if len(carlp) > 0:
+            return carlp[0][0]
+        else:
+            return None

pipeline/download.py

@@ -0,0 +1,340 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os, sys
+import os.path as osp
+import hashlib
+import requests
+import shutil
+import tqdm
+import time
+import tarfile
+import zipfile
+from paddle.utils.download import _get_unique_endpoints
+
+PPDET_WEIGHTS_DOWNLOAD_URL_PREFIX = 'https://paddledet.bj.bcebos.com/'
+
+DOWNLOAD_RETRY_LIMIT = 3
+
+WEIGHTS_HOME = osp.expanduser("~/.cache/paddle/infer_weights")
+
+MODEL_URL_MD5_DICT = {
+    'https://bj.bcebos.com/v1/paddledet/models/pipeline/ch_PP-OCRv3_det_infer.tar.gz':
+    '1b8eae0f098635699bd4e8bccf3067a7',
+    'https://bj.bcebos.com/v1/paddledet/models/pipeline/ch_PP-OCRv3_rec_infer.tar.gz':
+    '64fa0e0701efd93c7db52a9b685b3de6',
+    "https://bj.bcebos.com/v1/paddledet/models/pipeline/mot_ppyoloe_l_36e_ppvehicle.zip":
+    "3859d1a26e0c498285c2374b1a347013",
+    "https://bj.bcebos.com/v1/paddledet/models/pipeline/mot_ppyoloe_s_36e_ppvehicle.zip":
+    "4ed58b546be2a76d8ccbb138f64874ac",
+    "https://bj.bcebos.com/v1/paddledet/models/pipeline/dark_hrnet_w32_256x192.zip":
+    "a20d5f6ca087bff0e9f2b18df45a36f2",
+    "https://bj.bcebos.com/v1/paddledet/models/pipeline/PPLCNet_x1_0_person_attribute_945_infer.zip":
+    "1dfb161bf12bbc1365b2ed6866674483",
+    "https://videotag.bj.bcebos.com/PaddleVideo-release2.3/ppTSM_fight.zip":
+    "5d4609142501258608bf0a1445eedaba",
+    "https://bj.bcebos.com/v1/paddledet/models/pipeline/STGCN.zip":
+    "cf1c3c4bae90b975accb954d13129ea4",
+    "https://bj.bcebos.com/v1/paddledet/models/pipeline/ppyoloe_crn_s_80e_smoking_visdrone.zip":
+    "4cd12ae55be8f0eb2b90c08ac3b48218",
+    "https://bj.bcebos.com/v1/paddledet/models/pipeline/PPHGNet_tiny_calling_halfbody.zip":
+    "cf86b87ace97540dace6ef08e62b584a",
+    "https://bj.bcebos.com/v1/paddledet/models/pipeline/reid_model.zip":
+    "fdc4dac38393b8e2b5921c1e1fdd5315"
+}
+
+
+def is_url(path):
+    """
+    Whether path is URL.
+    Args:
+        path (string): URL string or not.
+    """
+    return path.startswith('http://') \
+            or path.startswith('https://') \
+            or path.startswith('ppdet://')
+
+
+def parse_url(url):
+    url = url.replace("ppdet://", PPDET_WEIGHTS_DOWNLOAD_URL_PREFIX)
+    return url
+
+
+def map_path(url, root_dir, path_depth=1):
+    # parse path after download to decompress under root_dir
+    assert path_depth > 0, "path_depth should be a positive integer"
+    dirname = url
+    for _ in range(path_depth):
+        dirname = osp.dirname(dirname)
+    fpath = osp.relpath(url, dirname)
+
+    zip_formats = ['.zip', '.tar', '.gz']
+    for zip_format in zip_formats:
+        fpath = fpath.replace(zip_format, '')
+    return osp.join(root_dir, fpath)
+
+
+def _md5check(fullname, md5sum=None):
+    if md5sum is None:
+        return True
+
+    md5 = hashlib.md5()
+    with open(fullname, 'rb') as f:
+        for chunk in iter(lambda: f.read(4096), b""):
+            md5.update(chunk)
+    calc_md5sum = md5.hexdigest()
+
+    if calc_md5sum != md5sum:
+        return False
+    return True
+
+
+def _check_exist_file_md5(filename, md5sum, url):
+    return _md5check(filename, md5sum)
+
+
+def _download(url, path, md5sum=None):
+    """
+    Download from url, save to path.
+    url (str): download url
+    path (str): download to given path
+    """
+    if not osp.exists(path):
+        os.makedirs(path)
+
+    fname = osp.split(url)[-1]
+    fullname = osp.join(path, fname)
+    retry_cnt = 0
+    while not (osp.exists(fullname) and _check_exist_file_md5(fullname, md5sum,
+                                                              url)):
+        if retry_cnt < DOWNLOAD_RETRY_LIMIT:
+            retry_cnt += 1
+        else:
+            raise RuntimeError("Download from {} failed. "
+                               "Retry limit reached".format(url))
+
+        # NOTE: windows path join may incur \, which is invalid in url
+        if sys.platform == "win32":
+            url = url.replace('\\', '/')
+
+        req = requests.get(url, stream=True)
+        if req.status_code != 200:
+            raise RuntimeError("Downloading from {} failed with code "
+                               "{}!".format(url, req.status_code))
+
+        # For protecting download interupted, download to
+        # tmp_fullname firstly, move tmp_fullname to fullname
+        # after download finished
+        tmp_fullname = fullname + "_tmp"
+        total_size = req.headers.get('content-length')
+        with open(tmp_fullname, 'wb') as f:
+            if total_size:
+                for chunk in tqdm.tqdm(
+                        req.iter_content(chunk_size=1024),
+                        total=(int(total_size) + 1023) // 1024,
+                        unit='KB'):
+                    f.write(chunk)
+            else:
+                for chunk in req.iter_content(chunk_size=1024):
+                    if chunk:
+                        f.write(chunk)
+        shutil.move(tmp_fullname, fullname)
+    return fullname
+
+
+def _download_dist(url, path, md5sum=None):
+    env = os.environ
+    if 'PADDLE_TRAINERS_NUM' in env and 'PADDLE_TRAINER_ID' in env:
+        trainer_id = int(env['PADDLE_TRAINER_ID'])
+        num_trainers = int(env['PADDLE_TRAINERS_NUM'])
+        if num_trainers <= 1:
+            return _download(url, path, md5sum)
+        else:
+            fname = osp.split(url)[-1]
+            fullname = osp.join(path, fname)
+            lock_path = fullname + '.download.lock'
+
+            if not osp.isdir(path):
+                os.makedirs(path)
+
+            if not osp.exists(fullname):
+                from paddle.distributed import ParallelEnv
+                unique_endpoints = _get_unique_endpoints(ParallelEnv()
+                                                         .trainer_endpoints[:])
+                with open(lock_path, 'w'):  # touch    
+                    os.utime(lock_path, None)
+                if ParallelEnv().current_endpoint in unique_endpoints:
+                    _download(url, path, md5sum)
+                    os.remove(lock_path)
+                else:
+                    while os.path.exists(lock_path):
+                        time.sleep(0.5)
+            return fullname
+    else:
+        return _download(url, path, md5sum)
+
+
+def _move_and_merge_tree(src, dst):
+    """
+    Move src directory to dst, if dst is already exists,
+    merge src to dst
+    """
+    if not osp.exists(dst):
+        shutil.move(src, dst)
+    elif osp.isfile(src):
+        shutil.move(src, dst)
+    else:
+        for fp in os.listdir(src):
+            src_fp = osp.join(src, fp)
+            dst_fp = osp.join(dst, fp)
+            if osp.isdir(src_fp):
+                if osp.isdir(dst_fp):
+                    _move_and_merge_tree(src_fp, dst_fp)
+                else:
+                    shutil.move(src_fp, dst_fp)
+            elif osp.isfile(src_fp) and \
+                    not osp.isfile(dst_fp):
+                shutil.move(src_fp, dst_fp)
+
+
+def _decompress(fname):
+    """
+    Decompress for zip and tar file
+    """
+
+    # For protecting decompressing interupted,
+    # decompress to fpath_tmp directory firstly, if decompress
+    # successed, move decompress files to fpath and delete
+    # fpath_tmp and remove download compress file.
+    fpath = osp.split(fname)[0]
+    fpath_tmp = osp.join(fpath, 'tmp')
+    if osp.isdir(fpath_tmp):
+        shutil.rmtree(fpath_tmp)
+        os.makedirs(fpath_tmp)
+
+    if fname.find('tar') >= 0:
+        with tarfile.open(fname) as tf:
+            tf.extractall(path=fpath_tmp)
+    elif fname.find('zip') >= 0:
+        with zipfile.ZipFile(fname) as zf:
+            zf.extractall(path=fpath_tmp)
+    elif fname.find('.txt') >= 0:
+        return
+    else:
+        raise TypeError("Unsupport compress file type {}".format(fname))
+
+    for f in os.listdir(fpath_tmp):
+        src_dir = osp.join(fpath_tmp, f)
+        dst_dir = osp.join(fpath, f)
+        _move_and_merge_tree(src_dir, dst_dir)
+
+    shutil.rmtree(fpath_tmp)
+    os.remove(fname)
+
+
+def _decompress_dist(fname):
+    env = os.environ
+    if 'PADDLE_TRAINERS_NUM' in env and 'PADDLE_TRAINER_ID' in env:
+        trainer_id = int(env['PADDLE_TRAINER_ID'])
+        num_trainers = int(env['PADDLE_TRAINERS_NUM'])
+        if num_trainers <= 1:
+            _decompress(fname)
+        else:
+            lock_path = fname + '.decompress.lock'
+            from paddle.distributed import ParallelEnv
+            unique_endpoints = _get_unique_endpoints(ParallelEnv()
+                                                     .trainer_endpoints[:])
+            # NOTE(dkp): _decompress_dist always performed after
+            # _download_dist, in _download_dist sub-trainers is waiting
+            # for download lock file release with sleeping, if decompress
+            # prograss is very fast and finished with in the sleeping gap
+            # time, e.g in tiny dataset such as coco_ce, spine_coco, main
+            # trainer may finish decompress and release lock file, so we
+            # only craete lock file in main trainer and all sub-trainer
+            # wait 1s for main trainer to create lock file, for 1s is
+            # twice as sleeping gap, this waiting time can keep all
+            # trainer pipeline in order
+            # **change this if you have more elegent methods**
+            if ParallelEnv().current_endpoint in unique_endpoints:
+                with open(lock_path, 'w'):  # touch    
+                    os.utime(lock_path, None)
+                _decompress(fname)
+                os.remove(lock_path)
+            else:
+                time.sleep(1)
+                while os.path.exists(lock_path):
+                    time.sleep(0.5)
+    else:
+        _decompress(fname)
+
+
+def get_path(url, root_dir=WEIGHTS_HOME, md5sum=None, check_exist=True):
+    """ Download from given url to root_dir.
+    if file or directory specified by url is exists under
+    root_dir, return the path directly, otherwise download
+    from url and decompress it, return the path.
+    url (str): download url
+    root_dir (str): root dir for downloading
+    md5sum (str): md5 sum of download package
+    """
+    # parse path after download to decompress under root_dir
+    fullpath = map_path(url, root_dir)
+
+    # For same zip file, decompressed directory name different
+    # from zip file name, rename by following map
+    decompress_name_map = {"ppTSM_fight": "ppTSM", }
+    for k, v in decompress_name_map.items():
+        if fullpath.find(k) >= 0:
+            fullpath = osp.join(osp.split(fullpath)[0], v)
+
+    if osp.exists(fullpath) and check_exist:
+        if not osp.isfile(fullpath) or \
+                _check_exist_file_md5(fullpath, md5sum, url):
+            return fullpath, True
+        else:
+            os.remove(fullpath)
+
+    fullname = _download_dist(url, root_dir, md5sum)
+
+    # new weights format which postfix is 'pdparams' not
+    # need to decompress
+    if osp.splitext(fullname)[-1] not in ['.pdparams', '.yml']:
+        _decompress_dist(fullname)
+
+    return fullpath, False
+
+
+def get_weights_path(url):
+    """Get weights path from WEIGHTS_HOME, if not exists,
+    download it from url.
+    """
+    url = parse_url(url)
+    md5sum = None
+    if url in MODEL_URL_MD5_DICT.keys():
+        md5sum = MODEL_URL_MD5_DICT[url]
+    path, _ = get_path(url, WEIGHTS_HOME, md5sum)
+    return path
+
+
+def auto_download_model(model_path):
+    # auto download
+    if is_url(model_path):
+        weight = get_weights_path(model_path)
+        return weight
+    return None
+
+
+if __name__ == "__main__":
+    model_path = "https://bj.bcebos.com/v1/paddledet/models/pipeline/mot_ppyoloe_l_36e_pipeline.zip"
+    auto_download_model(model_path)

pipeline/pipe_utils.py

@@ -0,0 +1,263 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import time
+import os
+import ast
+import glob
+import yaml
+import copy
+import numpy as np
+import subprocess as sp
+
+from python.keypoint_preprocess import EvalAffine, TopDownEvalAffine, expand_crop
+
+
+class Times(object):
+    def __init__(self):
+        self.time = 0.
+        # start time
+        self.st = 0.
+        # end time
+        self.et = 0.
+
+    def start(self):
+        self.st = time.time()
+
+    def end(self, repeats=1, accumulative=True):
+        self.et = time.time()
+        if accumulative:
+            self.time += (self.et - self.st) / repeats
+        else:
+            self.time = (self.et - self.st) / repeats
+
+    def reset(self):
+        self.time = 0.
+        self.st = 0.
+        self.et = 0.
+
+    def value(self):
+        return round(self.time, 4)
+
+
+class PipeTimer(Times):
+    def __init__(self):
+        super(PipeTimer, self).__init__()
+        self.total_time = Times()
+        self.module_time = {
+            'det': Times(),
+            'mot': Times(),
+            'attr': Times(),
+            'kpt': Times(),
+            'video_action': Times(),
+            'skeleton_action': Times(),
+            'reid': Times(),
+            'det_action': Times(),
+            'cls_action': Times(),
+            'vehicle_attr': Times(),
+            'vehicleplate': Times()
+        }
+        self.img_num = 0
+        self.track_num = 0
+
+    def get_total_time(self):
+        total_time = self.total_time.value()
+        total_time = round(total_time, 4)
+        average_latency = total_time / max(1, self.img_num)
+        qps = 0
+        if total_time > 0:
+            qps = 1 / average_latency
+        return total_time, average_latency, qps
+
+    def info(self):
+        total_time, average_latency, qps = self.get_total_time()
+        print("------------------ Inference Time Info ----------------------")
+        print("total_time(ms): {}, img_num: {}".format(total_time * 1000,
+                                                       self.img_num))
+
+        for k, v in self.module_time.items():
+            v_time = round(v.value(), 4)
+            if v_time > 0 and k in ['det', 'mot', 'video_action']:
+                print("{} time(ms): {}; per frame average time(ms): {}".format(
+                    k, v_time * 1000, v_time * 1000 / self.img_num))
+            elif v_time > 0:
+                print("{} time(ms): {}; per trackid average time(ms): {}".
+                      format(k, v_time * 1000, v_time * 1000 / self.track_num))
+
+        print("average latency time(ms): {:.2f}, QPS: {:2f}".format(
+            average_latency * 1000, qps))
+        return qps
+
+    def report(self, average=False):
+        dic = {}
+        dic['total'] = round(self.total_time.value() / max(1, self.img_num),
+                             4) if average else self.total_time.value()
+        dic['det'] = round(self.module_time['det'].value() /
+                           max(1, self.img_num),
+                           4) if average else self.module_time['det'].value()
+        dic['mot'] = round(self.module_time['mot'].value() /
+                           max(1, self.img_num),
+                           4) if average else self.module_time['mot'].value()
+        dic['attr'] = round(self.module_time['attr'].value() /
+                            max(1, self.img_num),
+                            4) if average else self.module_time['attr'].value()
+        dic['kpt'] = round(self.module_time['kpt'].value() /
+                           max(1, self.img_num),
+                           4) if average else self.module_time['kpt'].value()
+        dic['video_action'] = self.module_time['video_action'].value()
+        dic['skeleton_action'] = round(
+            self.module_time['skeleton_action'].value() / max(1, self.img_num),
+            4) if average else self.module_time['skeleton_action'].value()
+
+        dic['img_num'] = self.img_num
+        return dic
+
+
+class PushStream(object):
+    def __init__(self, pushurl="rtsp://127.0.0.1:8554/"):
+        self.command = ""
+        # 自行设置
+        self.pushurl = pushurl
+
+    def initcmd(self, fps, width, height):
+        self.command = [
+            'ffmpeg', '-y', '-f', 'rawvideo', '-vcodec', 'rawvideo',
+            '-pix_fmt', 'bgr24', '-s', "{}x{}".format(width, height), '-r',
+            str(fps), '-i', '-', '-pix_fmt', 'yuv420p', '-f', 'rtsp',
+            self.pushurl
+        ]
+        self.pipe = sp.Popen(self.command, stdin=sp.PIPE)
+
+
+def get_test_images(infer_dir, infer_img):
+    """
+    Get image path list in TEST mode
+    """
+    assert infer_img is not None or infer_dir is not None, \
+        "--infer_img or --infer_dir should be set"
+    assert infer_img is None or os.path.isfile(infer_img), \
+            "{} is not a file".format(infer_img)
+    assert infer_dir is None or os.path.isdir(infer_dir), \
+            "{} is not a directory".format(infer_dir)
+
+    # infer_img has a higher priority
+    if infer_img and os.path.isfile(infer_img):
+        return [infer_img]
+
+    images = set()
+    infer_dir = os.path.abspath(infer_dir)
+    assert os.path.isdir(infer_dir), \
+        "infer_dir {} is not a directory".format(infer_dir)
+    exts = ['jpg', 'jpeg', 'png', 'bmp']
+    exts += [ext.upper() for ext in exts]
+    for ext in exts:
+        images.update(glob.glob('{}/*.{}'.format(infer_dir, ext)))
+    images = list(images)
+
+    assert len(images) > 0, "no image found in {}".format(infer_dir)
+    print("Found {} inference images in total.".format(len(images)))
+
+    return images
+
+
+def crop_image_with_det(batch_input, det_res, thresh=0.3):
+    boxes = det_res['boxes']
+    score = det_res['boxes'][:, 1]
+    boxes_num = det_res['boxes_num']
+    start_idx = 0
+    crop_res = []
+    for b_id, input in enumerate(batch_input):
+        boxes_num_i = boxes_num[b_id]
+        if boxes_num_i == 0:
+            continue
+        boxes_i = boxes[start_idx:start_idx + boxes_num_i, :]
+        score_i = score[start_idx:start_idx + boxes_num_i]
+        res = []
+        for box, s in zip(boxes_i, score_i):
+            if s > thresh:
+                crop_image, new_box, ori_box = expand_crop(input, box)
+                if crop_image is not None:
+                    res.append(crop_image)
+        crop_res.append(res)
+    return crop_res
+
+
+def normal_crop(image, rect):
+    imgh, imgw, c = image.shape
+    label, conf, xmin, ymin, xmax, ymax = [int(x) for x in rect.tolist()]
+    org_rect = [xmin, ymin, xmax, ymax]
+    if label != 0:
+        return None, None, None
+    xmin = max(0, xmin)
+    ymin = max(0, ymin)
+    xmax = min(imgw, xmax)
+    ymax = min(imgh, ymax)
+    return image[ymin:ymax, xmin:xmax, :], [xmin, ymin, xmax, ymax], org_rect
+
+
+def crop_image_with_mot(input, mot_res, expand=True):
+    res = mot_res['boxes']
+    crop_res = []
+    new_bboxes = []
+    ori_bboxes = []
+    for box in res:
+        if expand:
+            crop_image, new_bbox, ori_bbox = expand_crop(input, box[1:])
+        else:
+            crop_image, new_bbox, ori_bbox = normal_crop(input, box[1:])
+        if crop_image is not None:
+            crop_res.append(crop_image)
+            new_bboxes.append(new_bbox)
+            ori_bboxes.append(ori_bbox)
+    return crop_res, new_bboxes, ori_bboxes
+
+
+def parse_mot_res(input):
+    mot_res = []
+    boxes, scores, ids = input[0]
+    for box, score, i in zip(boxes[0], scores[0], ids[0]):
+        xmin, ymin, w, h = box
+        res = [i, 0, score, xmin, ymin, xmin + w, ymin + h]
+        mot_res.append(res)
+    return {'boxes': np.array(mot_res)}
+
+
+def refine_keypoint_coordinary(kpts, bbox, coord_size):
+    """
+        This function is used to adjust coordinate values to a fixed scale.
+    """
+    tl = bbox[:, 0:2]
+    wh = bbox[:, 2:] - tl
+    tl = np.expand_dims(np.transpose(tl, (1, 0)), (2, 3))
+    wh = np.expand_dims(np.transpose(wh, (1, 0)), (2, 3))
+    target_w, target_h = coord_size
+    res = (kpts - tl) / wh * np.expand_dims(
+        np.array([[target_w], [target_h]]), (2, 3))
+    return res
+
+
+def parse_mot_keypoint(input, coord_size):
+    parsed_skeleton_with_mot = {}
+    ids = []
+    skeleton = []
+    for tracker_id, kpt_seq in input:
+        ids.append(tracker_id)
+        kpts = np.array(kpt_seq.kpts, dtype=np.float32)[:, :, :2]
+        kpts = np.expand_dims(np.transpose(kpts, [2, 0, 1]),
+                              -1)  #T, K, C -> C, T, K, 1
+        bbox = np.array(kpt_seq.bboxes, dtype=np.float32)
+        skeleton.append(refine_keypoint_coordinary(kpts, bbox, coord_size))
+    parsed_skeleton_with_mot["mot_id"] = ids
+    parsed_skeleton_with_mot["skeleton"] = skeleton
+    return parsed_skeleton_with_mot

pipeline/pipeline.py

@@ -0,0 +1,898 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import yaml
+import glob
+import cv2
+import numpy as np
+import math
+import paddle
+import sys
+import copy
+from collections import defaultdict
+from pipeline.datacollector import DataCollector, Result
+
+# add deploy path of PadleDetection to sys.path
+parent_path = os.path.abspath(os.path.join(__file__, *(['..'] * 2)))
+sys.path.insert(0, parent_path)
+
+from pipeline.cfg_utils import argsparser, merge_cfg
+from pipeline.pipe_utils import get_test_images, crop_image_with_det, crop_image_with_mot, parse_mot_res, parse_mot_keypoint
+from pipeline.pipe_utils import PushStream, PipeTimer
+
+from python.infer import Detector, DetectorPicoDet
+from python.keypoint_infer import KeyPointDetector
+from python.keypoint_postprocess import translate_to_ori_images
+from python.preprocess import decode_image, ShortSizeScale
+from python.visualize import visualize_box_mask, visualize_attr, visualize_pose, visualize_action, visualize_vehicleplate
+
+from pptracking.python.mot_sde_infer import SDE_Detector
+from pptracking.python.mot.visualize import plot_tracking_dict
+from pptracking.python.mot.utils import flow_statistic, update_object_info
+
+from pipeline.pphuman.attr_infer import AttrDetector
+from pipeline.pphuman.video_action_infer import VideoActionRecognizer
+from pipeline.pphuman.action_infer import SkeletonActionRecognizer, DetActionRecognizer, ClsActionRecognizer
+from pipeline.pphuman.action_utils import KeyPointBuff, ActionVisualHelper
+from pipeline.pphuman.reid import ReID
+from pipeline.pphuman.mtmct import mtmct_process
+
+from pipeline.download import auto_download_model
+
+
+class Pipeline(object):
+    """
+    Pipeline
+
+    Args:
+        args (argparse.Namespace): arguments in pipeline, which contains environment and runtime settings
+        cfg (dict): config of models in pipeline
+    """
+
+    def __init__(self, args, cfg):
+        self.multi_camera = False
+        reid_cfg = cfg.get('REID', False)
+        self.enable_mtmct = reid_cfg['enable'] if reid_cfg else False
+        self.is_video = False
+        self.output_dir = args.output_dir
+        self.vis_result = cfg['visual']
+        self.input = self._parse_input(args.image_file, args.video_file)
+
+        self.predictor = PipePredictor(args, cfg, self.is_video)
+        if self.is_video:
+            self.predictor.set_file_name(self.input)
+
+    def _parse_input(self, image, video_file):
+
+        # parse input as is_video and multi_camera
+
+        if image is not None:
+
+            input = image
+            self.is_video = False
+            self.multi_camera = False
+
+        elif video_file is not None:
+            input = video_file
+            self.is_video = True
+        else:
+            raise ValueError(
+                "Illegal Input, please set one of ['video_file', 'camera_id', 'image_file', 'image_dir']"
+            )
+
+        return input
+
+    def run_multithreads(self):
+
+        out = self.predictor.run(self.input)
+        return out
+
+    def run(self):
+        out = self.predictor.run(self.input)
+        return out
+
+
+def get_model_dir_with_list(cfg, args):
+    activate_list = args.avtivity_list
+    """ 
+        Auto download inference model if the model_path is a url link. 
+        Otherwise it will use the model_path directly.
+    """
+    for key in cfg.keys():
+        if type(cfg[key]) == dict and ((key in activate_list) or
+                                       "enable" not in cfg[key].keys()):
+            if "model_dir" in cfg[key].keys():
+                model_dir = cfg[key]["model_dir"]
+                downloaded_model_dir = auto_download_model(model_dir)
+                if downloaded_model_dir:
+                    model_dir = downloaded_model_dir
+                    cfg[key]["model_dir"] = model_dir
+                print(key, " model dir: ", model_dir)
+            elif key == "VEHICLE_PLATE":
+                det_model_dir = cfg[key]["det_model_dir"]
+                downloaded_det_model_dir = auto_download_model(det_model_dir)
+                if downloaded_det_model_dir:
+                    det_model_dir = downloaded_det_model_dir
+                    cfg[key]["det_model_dir"] = det_model_dir
+                print("det_model_dir model dir: ", det_model_dir)
+
+                rec_model_dir = cfg[key]["rec_model_dir"]
+                downloaded_rec_model_dir = auto_download_model(rec_model_dir)
+                if downloaded_rec_model_dir:
+                    rec_model_dir = downloaded_rec_model_dir
+                    cfg[key]["rec_model_dir"] = rec_model_dir
+                print("rec_model_dir model dir: ", rec_model_dir)
+
+            if (key == 'ID_BASED_DETACTION' and (key in activate_list)) or (key == 'SKELETON_ACTION' and (key in activate_list))or (key == 'ATTR' and (key in activate_list)) or (key =='ID_BASED_CLSACTION' and (key in activate_list)) or (key=='REID' and (key in activate_list)):
+                model_dir = cfg['MOT']["model_dir"]
+                downloaded_model_dir = auto_download_model(model_dir)
+                if downloaded_model_dir:
+                    model_dir = downloaded_model_dir
+                    cfg['MOT']["model_dir"] = model_dir
+                print("mot_model_dir model_dir: ", model_dir)
+
+
+def get_model_dir(cfg):
+    """ 
+        Auto download inference model if the model_path is a url link. 
+        Otherwise it will use the model_path directly.
+    """
+    for key in cfg.keys():
+        if type(cfg[key]) ==  dict and \
+            ("enable" in cfg[key].keys() and cfg[key]['enable']
+                or "enable" not in cfg[key].keys()):
+
+            if "model_dir" in cfg[key].keys():
+                model_dir = cfg[key]["model_dir"]
+                downloaded_model_dir = auto_download_model(model_dir)
+                if downloaded_model_dir:
+                    model_dir = downloaded_model_dir
+                    cfg[key]["model_dir"] = model_dir
+                print(key, " model dir: ", model_dir)
+            elif key == "VEHICLE_PLATE":
+                det_model_dir = cfg[key]["det_model_dir"]
+                downloaded_det_model_dir = auto_download_model(det_model_dir)
+                if downloaded_det_model_dir:
+                    det_model_dir = downloaded_det_model_dir
+                    cfg[key]["det_model_dir"] = det_model_dir
+                print("det_model_dir model dir: ", det_model_dir)
+
+                rec_model_dir = cfg[key]["rec_model_dir"]
+                downloaded_rec_model_dir = auto_download_model(rec_model_dir)
+                if downloaded_rec_model_dir:
+                    rec_model_dir = downloaded_rec_model_dir
+                    cfg[key]["rec_model_dir"] = rec_model_dir
+                print("rec_model_dir model dir: ", rec_model_dir)
+
+        elif key == "MOT":  # for idbased and skeletonbased actions
+            model_dir = cfg[key]["model_dir"]
+            downloaded_model_dir = auto_download_model(model_dir)
+            if downloaded_model_dir:
+                model_dir = downloaded_model_dir
+                cfg[key]["model_dir"] = model_dir
+            print("mot_model_dir model_dir: ", model_dir)
+
+
+class PipePredictor(object):
+    """
+    Predictor in single camera
+    
+    The pipeline for image input: 
+
+        1. Detection
+        2. Detection -> Attribute
+
+    The pipeline for video input: 
+
+        1. Tracking
+        2. Tracking -> Attribute
+        3. Tracking -> KeyPoint -> SkeletonAction Recognition
+        4. VideoAction Recognition
+
+    Args:
+        args (argparse.Namespace): arguments in pipeline, which contains environment and runtime settings
+        cfg (dict): config of models in pipeline
+        is_video (bool): whether the input is video, default as False
+        multi_camera (bool): whether to use multi camera in pipeline, 
+            default as False
+    """
+
+    def __init__(self, args, cfg, is_video=True, multi_camera=False):
+        # general module for pphuman and ppvehicle
+        activate_list = args.avtivity_list
+        self.with_mot = True if 'MOT' in activate_list else False
+        self.with_human_attr = True if 'ATTR' in activate_list else False
+        if self.with_mot:
+            print('Multi-Object Tracking enabled')
+        if self.with_human_attr:
+            print('Human Attribute Recognition enabled')
+
+        # only for pphuman
+        self.with_skeleton_action = True if 'SKELETON_ACTION' in activate_list else False
+
+        self.with_video_action = True if 'VIDEO_ACTION' in activate_list else False
+
+        self.with_idbased_detaction = True if 'ID_BASED_DETACTION' in activate_list else False
+
+        self.with_idbased_clsaction = True if 'ID_BASED_CLSACTION' in activate_list else False
+
+        self.with_mtmct = True if 'REID' in activate_list else False
+
+        if self.with_skeleton_action:
+            print('SkeletonAction Recognition enabled')
+        if self.with_video_action:
+            print('VideoAction Recognition enabled')
+        if self.with_idbased_detaction:
+            print('IDBASED Detection Action Recognition enabled')
+        if self.with_idbased_clsaction:
+            print('IDBASED Classification Action Recognition enabled')
+        if self.with_mtmct:
+            print("MTMCT enabled")
+
+        self.modebase = {
+            "framebased": False,
+            "videobased": False,
+            "idbased": False,
+            "skeletonbased": False
+        }
+
+        self.basemode = {
+            "MOT": "idbased",
+            "ATTR": "idbased",
+            "VIDEO_ACTION": "videobased",
+            "SKELETON_ACTION": "skeletonbased",
+            "ID_BASED_DETACTION": "idbased",
+            "ID_BASED_CLSACTION": "idbased",
+            "REID": "idbased",
+        }
+
+        self.is_video = is_video
+        self.multi_camera = multi_camera
+        self.cfg = cfg
+
+        self.output_dir = args.output_dir
+        self.draw_center_traj = True if 'draw_center_traj' in activate_list else False
+        self.secs_interval = args.secs_interval
+        self.do_entrance_counting = True if 'do_entrance_counting' in activate_list else False
+        self.do_break_in_counting = args.do_break_in_counting
+        self.region_type = args.region_type
+        self.region_polygon = args.region_polygon
+        self.illegal_parking_time = args.illegal_parking_time
+
+        self.warmup_frame = self.cfg['warmup_frame']
+        self.pipeline_res = Result()
+        self.pipe_timer = PipeTimer()
+        self.file_name = None
+        self.collector = DataCollector()
+
+        self.pushurl = args.pushurl
+
+        # auto download inference model
+        get_model_dir_with_list(self.cfg, args)
+
+        if self.with_human_attr:
+            attr_cfg = self.cfg['ATTR']
+            basemode = self.basemode['ATTR']
+            self.modebase[basemode] = True
+            self.attr_predictor = AttrDetector.init_with_cfg(args, attr_cfg)
+
+        if not is_video:
+            det_cfg = self.cfg['DET']
+            model_dir = det_cfg['model_dir']
+            batch_size = det_cfg['batch_size']
+            self.det_predictor = Detector(
+                model_dir, args.device, args.run_mode, batch_size,
+                args.trt_min_shape, args.trt_max_shape, args.trt_opt_shape,
+                args.trt_calib_mode, args.cpu_threads, args.enable_mkldnn)
+        else:
+            if self.with_idbased_detaction:
+                idbased_detaction_cfg = self.cfg['ID_BASED_DETACTION']
+                basemode = self.basemode['ID_BASED_DETACTION']
+                self.modebase[basemode] = True
+
+                self.det_action_predictor = DetActionRecognizer.init_with_cfg(
+                    args, idbased_detaction_cfg)
+                self.det_action_visual_helper = ActionVisualHelper(1)
+
+            if self.with_idbased_clsaction:
+                idbased_clsaction_cfg = self.cfg['ID_BASED_CLSACTION']
+                basemode = self.basemode['ID_BASED_CLSACTION']
+                self.modebase[basemode] = True
+
+                self.cls_action_predictor = ClsActionRecognizer.init_with_cfg(
+                    args, idbased_clsaction_cfg)
+                self.cls_action_visual_helper = ActionVisualHelper(1)
+
+            if self.with_skeleton_action:
+                skeleton_action_cfg = self.cfg['SKELETON_ACTION']
+                display_frames = skeleton_action_cfg['display_frames']
+                self.coord_size = skeleton_action_cfg['coord_size']
+                basemode = self.basemode['SKELETON_ACTION']
+                self.modebase[basemode] = True
+                skeleton_action_frames = skeleton_action_cfg['max_frames']
+
+                self.skeleton_action_predictor = SkeletonActionRecognizer.init_with_cfg(
+                    args, skeleton_action_cfg)
+                self.skeleton_action_visual_helper = ActionVisualHelper(
+                    display_frames)
+
+                kpt_cfg = self.cfg['KPT']
+                kpt_model_dir = kpt_cfg['model_dir']
+                kpt_batch_size = kpt_cfg['batch_size']
+                self.kpt_predictor = KeyPointDetector(
+                    kpt_model_dir,
+                    args.device,
+                    args.run_mode,
+                    kpt_batch_size,
+                    args.trt_min_shape,
+                    args.trt_max_shape,
+                    args.trt_opt_shape,
+                    args.trt_calib_mode,
+                    args.cpu_threads,
+                    args.enable_mkldnn,
+                    use_dark=False)
+                self.kpt_buff = KeyPointBuff(skeleton_action_frames)
+
+            if self.with_mtmct:
+                reid_cfg = self.cfg['REID']
+                basemode = self.basemode['REID']
+                self.modebase[basemode] = True
+                self.reid_predictor = ReID.init_with_cfg(args, reid_cfg)
+
+            if self.with_mot or self.modebase["idbased"] or self.modebase[
+                    "skeletonbased"]:
+                mot_cfg = self.cfg['MOT']
+                model_dir = mot_cfg['model_dir']
+                tracker_config = mot_cfg['tracker_config']
+                batch_size = mot_cfg['batch_size']
+                skip_frame_num = mot_cfg.get('skip_frame_num', -1)
+                basemode = self.basemode['MOT']
+                self.modebase[basemode] = True
+                self.mot_predictor = SDE_Detector(
+                    model_dir,
+                    tracker_config,
+                    args.device,
+                    args.run_mode,
+                    batch_size,
+                    args.trt_min_shape,
+                    args.trt_max_shape,
+                    args.trt_opt_shape,
+                    args.trt_calib_mode,
+                    args.cpu_threads,
+                    args.enable_mkldnn,
+                    skip_frame_num=skip_frame_num,
+                    draw_center_traj=self.draw_center_traj,
+                    secs_interval=self.secs_interval,
+                    do_entrance_counting=self.do_entrance_counting,
+                    do_break_in_counting=self.do_break_in_counting,
+                    region_type=self.region_type,
+                    region_polygon=self.region_polygon)
+
+            if self.with_video_action:
+                video_action_cfg = self.cfg['VIDEO_ACTION']
+                basemode = self.basemode['VIDEO_ACTION']
+                self.modebase[basemode] = True
+                self.video_action_predictor = VideoActionRecognizer.init_with_cfg(
+                    args, video_action_cfg)
+
+    def set_file_name(self, path):
+        if path is not None:
+            if "." in path:
+                self.file_name = path.split(".")[-2]
+        else:
+            # use camera id
+            self.file_name = None
+
+    def get_result(self):
+        return self.collector.get_res()
+
+    def run(self, input, thread_idx=0):
+
+        if self.is_video:
+            out = self.predict_video(input, thread_idx=thread_idx)
+            return out
+        else:
+            out = self.predict_image(input)
+            return out
+
+    def predict_image(self, input):
+        # det
+        # det -> attr
+        batch_input = [decode_image(input, {})[0]]
+        batch_input[0] = cv2.cvtColor(batch_input[0], cv2.COLOR_BGR2RGB)
+        # det output format: class, score, xmin, ymin, xmax, ymax
+        det_res = self.det_predictor.predict_image(batch_input, visual=False)
+        det_res = self.det_predictor.filter_box(det_res,
+                                                self.cfg['crop_thresh'])
+
+        self.pipeline_res.update(det_res, 'det')
+
+        if self.with_human_attr:
+            crop_inputs = crop_image_with_det(batch_input, det_res)
+            attr_res_list = []
+
+            for crop_input in crop_inputs:
+                attr_res = self.attr_predictor.predict_image(
+                    crop_input, visual=False)
+                attr_res_list.extend(attr_res['output'])
+
+            attr_res = {'output': attr_res_list}
+            self.pipeline_res.update(attr_res, 'attr')
+
+        return self.visualize_image(batch_input, self.pipeline_res)
+
+    def predict_video(self, video_file, thread_idx=0):
+        # mot
+        # mot -> attr
+        # mot -> pose -> action
+        capture = cv2.VideoCapture(video_file)
+
+        # Get Video info : resolution, fps, frame count
+        width = int(capture.get(cv2.CAP_PROP_FRAME_WIDTH))
+        height = int(capture.get(cv2.CAP_PROP_FRAME_HEIGHT))
+        fps = int(capture.get(cv2.CAP_PROP_FPS))
+        frame_count = int(capture.get(cv2.CAP_PROP_FRAME_COUNT))
+        print("video fps: %d, frame_count: %d" % (fps, frame_count))
+
+        video_out_name = 'output' if self.file_name is None else self.file_name
+        out_path = video_out_name + "_output.mp4"
+        fourcc = cv2.VideoWriter_fourcc(*'mp4v')
+        writer = cv2.VideoWriter(out_path, fourcc, fps, (width, height))
+
+        frame_id = 0
+
+        entrance, records, center_traj = None, None, None
+        if self.draw_center_traj:
+            center_traj = [{}]
+        id_set = set()
+        interval_id_set = set()
+        in_id_list = list()
+        out_id_list = list()
+        prev_center = dict()
+        records = list()
+        if self.do_entrance_counting or self.do_break_in_counting or self.illegal_parking_time != -1:
+            if self.region_type == 'horizontal':
+                entrance = [0, height / 2., width, height / 2.]
+            elif self.region_type == 'vertical':
+                entrance = [width / 2, 0., width / 2, height]
+            elif self.region_type == 'custom':
+                entrance = []
+                assert len(
+                    self.region_polygon
+                ) % 2 == 0, "region_polygon should be pairs of coords points when do break_in counting."
+                assert len(
+                    self.region_polygon
+                ) > 6, 'region_type is custom, region_polygon should be at least 3 pairs of point coords.'
+
+                for i in range(0, len(self.region_polygon), 2):
+                    entrance.append(
+                        [self.region_polygon[i], self.region_polygon[i + 1]])
+                entrance.append([width, height])
+            else:
+                raise ValueError("region_type:{} unsupported.".format(
+                    self.region_type))
+
+        video_fps = fps
+
+        video_action_imgs = []
+
+        if self.with_video_action:
+            short_size = self.cfg["VIDEO_ACTION"]["short_size"]
+            scale = ShortSizeScale(short_size)
+
+        object_in_region_info = {
+        }  # store info for vehicle parking in region       
+        illegal_parking_dict = None
+
+        while (1):
+            if frame_id % 10 == 0:
+                print('Thread: {}; frame id: {}'.format(thread_idx, frame_id))
+
+            ret, frame = capture.read()
+            if not ret:
+                break
+            frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+            if frame_id > self.warmup_frame:
+                self.pipe_timer.total_time.start()
+
+            if self.modebase["idbased"] or self.modebase["skeletonbased"]:
+                if frame_id > self.warmup_frame:
+                    self.pipe_timer.module_time['mot'].start()
+
+                mot_skip_frame_num = self.mot_predictor.skip_frame_num
+                reuse_det_result = False
+                if mot_skip_frame_num > 1 and frame_id > 0 and frame_id % mot_skip_frame_num > 0:
+                    reuse_det_result = True
+                res = self.mot_predictor.predict_image(
+                    [copy.deepcopy(frame_rgb)],
+                    visual=False,
+                    reuse_det_result=reuse_det_result)
+
+                # mot output format: id, class, score, xmin, ymin, xmax, ymax
+                mot_res = parse_mot_res(res)
+                if frame_id > self.warmup_frame:
+                    self.pipe_timer.module_time['mot'].end()
+                    self.pipe_timer.track_num += len(mot_res['boxes'])
+
+                if frame_id % 10 == 0:
+                    print("Thread: {}; trackid number: {}".format(
+                        thread_idx, len(mot_res['boxes'])))
+
+                # flow_statistic only support single class MOT
+                boxes, scores, ids = res[0]  # batch size = 1 in MOT
+                mot_result = (frame_id + 1, boxes[0], scores[0],
+                              ids[0])  # single class
+                statistic = flow_statistic(
+                    mot_result,
+                    self.secs_interval,
+                    self.do_entrance_counting,
+                    self.do_break_in_counting,
+                    self.region_type,
+                    video_fps,
+                    entrance,
+                    id_set,
+                    interval_id_set,
+                    in_id_list,
+                    out_id_list,
+                    prev_center,
+                    records,
+                    ids2names=self.mot_predictor.pred_config.labels)
+                records = statistic['records']
+
+                if self.illegal_parking_time != -1:
+                    object_in_region_info, illegal_parking_dict = update_object_info(
+                        object_in_region_info, mot_result, self.region_type,
+                        entrance, video_fps, self.illegal_parking_time)
+                    if len(illegal_parking_dict) != 0:
+                        # build relationship between id and plate
+                        for key, value in illegal_parking_dict.items():
+                            plate = self.collector.get_carlp(key)
+                            illegal_parking_dict[key]['plate'] = plate
+
+                # nothing detected
+                if len(mot_res['boxes']) == 0:
+                    frame_id += 1
+                    if frame_id > self.warmup_frame:
+                        self.pipe_timer.img_num += 1
+                        self.pipe_timer.total_time.end()
+                    if self.cfg['visual']:
+                        _, _, fps = self.pipe_timer.get_total_time()
+                        im = self.visualize_video(frame, mot_res, frame_id,
+                                                  fps, entrance, records,
+                                                  center_traj)  # visualize
+                        if len(self.pushurl) > 0:
+                            pushstream.pipe.stdin.write(im.tobytes())
+                        else:
+                            writer.write(im)
+                            if self.file_name is None:  # use camera_id
+                                cv2.imshow('Paddle-Pipeline', im)
+                                if cv2.waitKey(1) & 0xFF == ord('q'):
+                                    break
+                    continue
+
+                self.pipeline_res.update(mot_res, 'mot')
+                crop_input, new_bboxes, ori_bboxes = crop_image_with_mot(
+                    frame_rgb, mot_res)
+
+                if self.with_human_attr:
+                    if frame_id > self.warmup_frame:
+                        self.pipe_timer.module_time['attr'].start()
+                    attr_res = self.attr_predictor.predict_image(
+                        crop_input, visual=False)
+                    if frame_id > self.warmup_frame:
+                        self.pipe_timer.module_time['attr'].end()
+                    self.pipeline_res.update(attr_res, 'attr')
+
+                if self.with_idbased_detaction:
+                    if frame_id > self.warmup_frame:
+                        self.pipe_timer.module_time['det_action'].start()
+                    det_action_res = self.det_action_predictor.predict(
+                        crop_input, mot_res)
+                    if frame_id > self.warmup_frame:
+                        self.pipe_timer.module_time['det_action'].end()
+                    self.pipeline_res.update(det_action_res, 'det_action')
+
+                    if self.cfg['visual']:
+                        self.det_action_visual_helper.update(det_action_res)
+
+                if self.with_idbased_clsaction:
+                    if frame_id > self.warmup_frame:
+                        self.pipe_timer.module_time['cls_action'].start()
+                    cls_action_res = self.cls_action_predictor.predict_with_mot(
+                        crop_input, mot_res)
+                    if frame_id > self.warmup_frame:
+                        self.pipe_timer.module_time['cls_action'].end()
+                    self.pipeline_res.update(cls_action_res, 'cls_action')
+
+                    if self.cfg['visual']:
+                        self.cls_action_visual_helper.update(cls_action_res)
+
+                if self.with_skeleton_action:
+                    if frame_id > self.warmup_frame:
+                        self.pipe_timer.module_time['kpt'].start()
+                    kpt_pred = self.kpt_predictor.predict_image(
+                        crop_input, visual=False)
+                    keypoint_vector, score_vector = translate_to_ori_images(
+                        kpt_pred, np.array(new_bboxes))
+                    kpt_res = {}
+                    kpt_res['keypoint'] = [
+                        keypoint_vector.tolist(), score_vector.tolist()
+                    ] if len(keypoint_vector) > 0 else [[], []]
+                    kpt_res['bbox'] = ori_bboxes
+                    if frame_id > self.warmup_frame:
+                        self.pipe_timer.module_time['kpt'].end()
+
+                    self.pipeline_res.update(kpt_res, 'kpt')
+
+                    self.kpt_buff.update(kpt_res,
+                                         mot_res)  # collect kpt output
+                    state = self.kpt_buff.get_state(
+                    )  # whether frame num is enough or lost tracker
+
+                    skeleton_action_res = {}
+                    if state:
+                        if frame_id > self.warmup_frame:
+                            self.pipe_timer.module_time[
+                                'skeleton_action'].start()
+                        collected_keypoint = self.kpt_buff.get_collected_keypoint(
+                        )  # reoragnize kpt output with ID
+                        skeleton_action_input = parse_mot_keypoint(
+                            collected_keypoint, self.coord_size)
+                        skeleton_action_res = self.skeleton_action_predictor.predict_skeleton_with_mot(
+                            skeleton_action_input)
+                        if frame_id > self.warmup_frame:
+                            self.pipe_timer.module_time['skeleton_action'].end(
+                            )
+                        self.pipeline_res.update(skeleton_action_res,
+                                                 'skeleton_action')
+
+                    if self.cfg['visual']:
+                        self.skeleton_action_visual_helper.update(
+                            skeleton_action_res)
+
+                if self.with_mtmct and frame_id % 10 == 0:
+                    crop_input, img_qualities, rects = self.reid_predictor.crop_image_with_mot(
+                        frame_rgb, mot_res)
+                    if frame_id > self.warmup_frame:
+                        self.pipe_timer.module_time['reid'].start()
+                    reid_res = self.reid_predictor.predict_batch(crop_input)
+
+                    if frame_id > self.warmup_frame:
+                        self.pipe_timer.module_time['reid'].end()
+
+                    reid_res_dict = {
+                        'features': reid_res,
+                        "qualities": img_qualities,
+                        "rects": rects
+                    }
+                    self.pipeline_res.update(reid_res_dict, 'reid')
+                else:
+                    self.pipeline_res.clear('reid')
+
+            if self.with_video_action:
+                # get the params
+                frame_len = self.cfg["VIDEO_ACTION"]["frame_len"]
+                sample_freq = self.cfg["VIDEO_ACTION"]["sample_freq"]
+
+                if sample_freq * frame_len > frame_count:  # video is too short
+                    sample_freq = int(frame_count / frame_len)
+
+                # filter the warmup frames
+                if frame_id > self.warmup_frame:
+                    self.pipe_timer.module_time['video_action'].start()
+
+                # collect frames
+                if frame_id % sample_freq == 0:
+                    # Scale image
+                    scaled_img = scale(frame_rgb)
+                    video_action_imgs.append(scaled_img)
+
+                # the number of collected frames is enough to predict video action
+                if len(video_action_imgs) == frame_len:
+                    classes, scores = self.video_action_predictor.predict(
+                        video_action_imgs)
+                    if frame_id > self.warmup_frame:
+                        self.pipe_timer.module_time['video_action'].end()
+
+                    video_action_res = {
+                        "class": classes[0],
+                        "score": scores[0]
+                    }
+                    self.pipeline_res.update(video_action_res, 'video_action')
+
+                    print("video_action_res:", video_action_res)
+
+                    video_action_imgs.clear()  # next clip
+
+            self.collector.append(frame_id, self.pipeline_res)
+
+            if frame_id > self.warmup_frame:
+                self.pipe_timer.img_num += 1
+                self.pipe_timer.total_time.end()
+            frame_id += 1
+
+            if self.cfg['visual']:
+                _, _, fps = self.pipe_timer.get_total_time()
+
+                im = self.visualize_video(frame, self.pipeline_res,
+                                          self.collector, frame_id, fps,
+                                          entrance, records, center_traj,
+                                          self.illegal_parking_time != -1,
+                                          illegal_parking_dict)  # visualize
+                if len(self.pushurl) > 0:
+                    pushstream.pipe.stdin.write(im.tobytes())
+                else:
+                    writer.write(im)
+        if self.cfg['visual'] and len(self.pushurl) == 0:
+            writer.release()
+
+        return out_path
+
+    def visualize_video(self,
+                        image,
+                        result,
+                        collector,
+                        frame_id,
+                        fps,
+                        entrance=None,
+                        records=None,
+                        center_traj=None,
+                        do_illegal_parking_recognition=False,
+                        illegal_parking_dict=None):
+        mot_res = copy.deepcopy(result.get('mot'))
+        if mot_res is not None:
+            ids = mot_res['boxes'][:, 0]
+            scores = mot_res['boxes'][:, 2]
+            boxes = mot_res['boxes'][:, 3:]
+            boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
+            boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
+        else:
+            boxes = np.zeros([0, 4])
+            ids = np.zeros([0])
+            scores = np.zeros([0])
+
+        # single class, still need to be defaultdict type for ploting
+        num_classes = 1
+        online_tlwhs = defaultdict(list)
+        online_scores = defaultdict(list)
+        online_ids = defaultdict(list)
+        online_tlwhs[0] = boxes
+        online_scores[0] = scores
+        online_ids[0] = ids
+
+        if mot_res is not None:
+            image = plot_tracking_dict(
+                image,
+                num_classes,
+                online_tlwhs,
+                online_ids,
+                online_scores,
+                frame_id=frame_id,
+                fps=fps,
+                ids2names=self.mot_predictor.pred_config.labels,
+                do_entrance_counting=self.do_entrance_counting,
+                do_break_in_counting=self.do_break_in_counting,
+                do_illegal_parking_recognition=do_illegal_parking_recognition,
+                illegal_parking_dict=illegal_parking_dict,
+                entrance=entrance,
+                records=records,
+                center_traj=center_traj)
+
+        human_attr_res = result.get('attr')
+        if human_attr_res is not None:
+            boxes = mot_res['boxes'][:, 1:]
+            human_attr_res = human_attr_res['output']
+            image = visualize_attr(image, human_attr_res, boxes)
+            image = np.array(image)
+
+        kpt_res = result.get('kpt')
+        if kpt_res is not None:
+            image = visualize_pose(
+                image,
+                kpt_res,
+                visual_thresh=self.cfg['kpt_thresh'],
+                returnimg=True)
+
+        video_action_res = result.get('video_action')
+        if video_action_res is not None:
+            video_action_score = None
+            if video_action_res and video_action_res["class"] == 1:
+                video_action_score = video_action_res["score"]
+            mot_boxes = None
+            if mot_res:
+                mot_boxes = mot_res['boxes']
+            image = visualize_action(
+                image,
+                mot_boxes,
+                action_visual_collector=None,
+                action_text="SkeletonAction",
+                video_action_score=video_action_score,
+                video_action_text="Fight")
+
+        visual_helper_for_display = []
+        action_to_display = []
+
+        skeleton_action_res = result.get('skeleton_action')
+        if skeleton_action_res is not None:
+            visual_helper_for_display.append(
+                self.skeleton_action_visual_helper)
+            action_to_display.append("Falling")
+
+        det_action_res = result.get('det_action')
+        if det_action_res is not None:
+            visual_helper_for_display.append(self.det_action_visual_helper)
+            action_to_display.append("Smoking")
+
+        cls_action_res = result.get('cls_action')
+        if cls_action_res is not None:
+            visual_helper_for_display.append(self.cls_action_visual_helper)
+            action_to_display.append("Calling")
+
+        if len(visual_helper_for_display) > 0:
+            image = visualize_action(image, mot_res['boxes'],
+                                     visual_helper_for_display,
+                                     action_to_display)
+
+        return image
+
+    def visualize_image(self, images, result):
+
+        det_res = result.get('det')
+        human_attr_res = result.get('attr')
+        vehicle_attr_res = result.get('vehicle_attr')
+        vehicleplate_res = result.get('vehicleplate')
+
+        if det_res is not None:
+            det_res_i = {}
+            boxes_num_i = det_res['boxes_num'][0]
+            det_res_i['boxes'] = det_res['boxes'][0:0 + boxes_num_i, :]
+            im = visualize_box_mask(
+                images[0],
+                det_res_i,
+                labels=['target'],
+                threshold=self.cfg['crop_thresh'])
+            im = np.ascontiguousarray(np.copy(im))
+            im = cv2.cvtColor(im, cv2.COLOR_RGB2BGR)
+        if human_attr_res is not None:
+            human_attr_res_i = human_attr_res['output'][0:0 + boxes_num_i]
+            im = visualize_attr(im, human_attr_res_i, det_res_i['boxes'])
+
+        return im
+
+
+def pp_humanv2(input_date, avtivity_list):
+
+    paddle.enable_static()
+
+    # parse params from command
+    parser = argsparser()
+    FLAGS = parser.parse_args()
+    FLAGS.device = FLAGS.device.upper()
+    assert FLAGS.device in ['CPU', 'GPU', 'XPU'
+                            ], "device should be CPU, GPU or XPU"
+
+    cfg = merge_cfg(FLAGS)  # use command params to update config
+
+    if isinstance(input_date, str):
+        FLAGS.video_file = input_date
+    else:
+        FLAGS.image_file = input_date
+
+    FLAGS.avtivity_list = avtivity_list
+
+    pipeline = Pipeline(FLAGS, cfg)
+    out = pipeline.run_multithreads()
+
+    return out

pipeline/pphuman/action_infer.py

@@ -0,0 +1,693 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import yaml
+import glob
+
+import cv2
+import numpy as np
+import math
+import paddle
+import sys
+
+# add deploy path of PadleDetection to sys.path
+parent_path = os.path.abspath(os.path.join(__file__, *(['..'] * 2)))
+sys.path.insert(0, parent_path)
+
+from paddle.inference import Config, create_predictor
+from python.utils import argsparser, Timer, get_current_memory_mb
+from python.benchmark_utils import PaddleInferBenchmark
+from python.infer import Detector, print_arguments
+from attr_infer import AttrDetector
+
+
+class SkeletonActionRecognizer(Detector):
+    """
+    Args:
+        model_dir (str): root path of model.pdiparams, model.pdmodel and infer_cfg.yml
+        device (str): Choose the device you want to run, it can be: CPU/GPU/XPU, default is CPU
+        run_mode (str): mode of running(paddle/trt_fp32/trt_fp16)
+        batch_size (int): size of pre batch in inference
+        trt_min_shape (int): min shape for dynamic shape in trt
+        trt_max_shape (int): max shape for dynamic shape in trt
+        trt_opt_shape (int): opt shape for dynamic shape in trt
+        trt_calib_mode (bool): If the model is produced by TRT offline quantitative
+            calibration, trt_calib_mode need to set True
+        cpu_threads (int): cpu threads
+        enable_mkldnn (bool): whether to open MKLDNN
+        threshold (float): The threshold of score for visualization
+        window_size(int): Temporal size of skeleton feature.
+        random_pad (bool): Whether do random padding when frame length < window_size.
+    """
+
+    def __init__(self,
+                 model_dir,
+                 device='CPU',
+                 run_mode='paddle',
+                 batch_size=1,
+                 trt_min_shape=1,
+                 trt_max_shape=1280,
+                 trt_opt_shape=640,
+                 trt_calib_mode=False,
+                 cpu_threads=1,
+                 enable_mkldnn=False,
+                 output_dir='output',
+                 threshold=0.5,
+                 window_size=100,
+                 random_pad=False):
+        assert batch_size == 1, "SkeletonActionRecognizer only support batch_size=1 now."
+        super(SkeletonActionRecognizer, self).__init__(
+            model_dir=model_dir,
+            device=device,
+            run_mode=run_mode,
+            batch_size=batch_size,
+            trt_min_shape=trt_min_shape,
+            trt_max_shape=trt_max_shape,
+            trt_opt_shape=trt_opt_shape,
+            trt_calib_mode=trt_calib_mode,
+            cpu_threads=cpu_threads,
+            enable_mkldnn=enable_mkldnn,
+            output_dir=output_dir,
+            threshold=threshold,
+            delete_shuffle_pass=True)
+
+    @classmethod
+    def init_with_cfg(cls, args, cfg):
+        return cls(model_dir=cfg['model_dir'],
+                   batch_size=cfg['batch_size'],
+                   window_size=cfg['max_frames'],
+                   device=args.device,
+                   run_mode=args.run_mode,
+                   trt_min_shape=args.trt_min_shape,
+                   trt_max_shape=args.trt_max_shape,
+                   trt_opt_shape=args.trt_opt_shape,
+                   trt_calib_mode=args.trt_calib_mode,
+                   cpu_threads=args.cpu_threads,
+                   enable_mkldnn=args.enable_mkldnn)
+
+    def predict(self, repeats=1):
+        '''
+        Args:
+            repeats (int): repeat number for prediction
+        Returns:
+            results (dict): 
+        '''
+        # model prediction
+        output_names = self.predictor.get_output_names()
+        for i in range(repeats):
+            self.predictor.run()
+            output_tensor = self.predictor.get_output_handle(output_names[0])
+            np_output = output_tensor.copy_to_cpu()
+        result = dict(output=np_output)
+        return result
+
+    def predict_skeleton(self, skeleton_list, run_benchmark=False, repeats=1):
+        results = []
+        for i, skeleton in enumerate(skeleton_list):
+            if run_benchmark:
+                # preprocess
+                inputs = self.preprocess(skeleton)  # warmup
+                self.det_times.preprocess_time_s.start()
+                inputs = self.preprocess(skeleton)
+                self.det_times.preprocess_time_s.end()
+
+                # model prediction
+                result = self.predict(repeats=repeats)  # warmup
+                self.det_times.inference_time_s.start()
+                result = self.predict(repeats=repeats)
+                self.det_times.inference_time_s.end(repeats=repeats)
+
+                # postprocess
+                result_warmup = self.postprocess(inputs, result)  # warmup
+                self.det_times.postprocess_time_s.start()
+                result = self.postprocess(inputs, result)
+                self.det_times.postprocess_time_s.end()
+                self.det_times.img_num += len(skeleton)
+
+                cm, gm, gu = get_current_memory_mb()
+                self.cpu_mem += cm
+                self.gpu_mem += gm
+                self.gpu_util += gu
+            else:
+                # preprocess
+                self.det_times.preprocess_time_s.start()
+                inputs = self.preprocess(skeleton)
+                self.det_times.preprocess_time_s.end()
+
+                # model prediction
+                self.det_times.inference_time_s.start()
+                result = self.predict()
+                self.det_times.inference_time_s.end()
+
+                # postprocess
+                self.det_times.postprocess_time_s.start()
+                result = self.postprocess(inputs, result)
+                self.det_times.postprocess_time_s.end()
+                self.det_times.img_num += len(skeleton)
+
+            results.append(result)
+        return results
+
+    def predict_skeleton_with_mot(self, skeleton_with_mot,
+                                  run_benchmark=False):
+        """
+            skeleton_with_mot (dict): includes individual skeleton sequences, which shape is [C, T, K, 1]
+                                      and its corresponding track id.
+        """
+
+        skeleton_list = skeleton_with_mot["skeleton"]
+        mot_id = skeleton_with_mot["mot_id"]
+        act_res = self.predict_skeleton(
+            skeleton_list, run_benchmark, repeats=1)
+        results = list(zip(mot_id, act_res))
+        return results
+
+    def preprocess(self, data):
+        preprocess_ops = []
+        for op_info in self.pred_config.preprocess_infos:
+            new_op_info = op_info.copy()
+            op_type = new_op_info.pop('type')
+            preprocess_ops.append(eval(op_type)(**new_op_info))
+
+        input_lst = []
+        data = action_preprocess(data, preprocess_ops)
+        input_lst.append(data)
+        input_names = self.predictor.get_input_names()
+        inputs = {}
+        inputs['data_batch_0'] = np.stack(input_lst, axis=0).astype('float32')
+
+        for i in range(len(input_names)):
+            input_tensor = self.predictor.get_input_handle(input_names[i])
+            input_tensor.copy_from_cpu(inputs[input_names[i]])
+
+        return inputs
+
+    def postprocess(self, inputs, result):
+        # postprocess output of predictor
+        output_logit = result['output'][0]
+        classes = np.argpartition(output_logit, -1)[-1:]
+        classes = classes[np.argsort(-output_logit[classes])]
+        scores = output_logit[classes]
+        result = {'class': classes, 'score': scores}
+        return result
+
+
+def action_preprocess(input, preprocess_ops):
+    """
+    input (str | numpy.array): if input is str, it should be a legal file path with numpy array saved.
+                               Otherwise it should be numpy.array as direct input.
+    return (numpy.array) 
+    """
+    if isinstance(input, str):
+        assert os.path.isfile(input) is not None, "{0} not exists".format(
+            input)
+        data = np.load(input)
+    else:
+        data = input
+    for operator in preprocess_ops:
+        data = operator(data)
+    return data
+
+
+class AutoPadding(object):
+    """
+    Sample or Padding frame skeleton feature.
+    Args:
+        window_size (int): Temporal size of skeleton feature.
+        random_pad (bool): Whether do random padding when frame length < window size. Default: False.
+    """
+
+    def __init__(self, window_size=100, random_pad=False):
+        self.window_size = window_size
+        self.random_pad = random_pad
+
+    def get_frame_num(self, data):
+        C, T, V, M = data.shape
+        for i in range(T - 1, -1, -1):
+            tmp = np.sum(data[:, i, :, :])
+            if tmp > 0:
+                T = i + 1
+                break
+        return T
+
+    def __call__(self, results):
+        data = results
+
+        C, T, V, M = data.shape
+        T = self.get_frame_num(data)
+        if T == self.window_size:
+            data_pad = data[:, :self.window_size, :, :]
+        elif T < self.window_size:
+            begin = random.randint(
+                0, self.window_size - T) if self.random_pad else 0
+            data_pad = np.zeros((C, self.window_size, V, M))
+            data_pad[:, begin:begin + T, :, :] = data[:, :T, :, :]
+        else:
+            if self.random_pad:
+                index = np.random.choice(
+                    T, self.window_size, replace=False).astype('int64')
+            else:
+                index = np.linspace(0, T, self.window_size).astype("int64")
+            data_pad = data[:, index, :, :]
+
+        return data_pad
+
+
+def get_test_skeletons(input_file):
+    assert input_file is not None, "--action_file can not be None"
+    input_data = np.load(input_file)
+    if input_data.ndim == 4:
+        return [input_data]
+    elif input_data.ndim == 5:
+        output = list(
+            map(lambda x: np.squeeze(x, 0),
+                np.split(input_data, input_data.shape[0], 0)))
+        return output
+    else:
+        raise ValueError(
+            "Now only support input with shape: (N, C, T, K, M) or (C, T, K, M)"
+        )
+
+
+class DetActionRecognizer(object):
+    """
+    Args:
+        model_dir (str): root path of model.pdiparams, model.pdmodel and infer_cfg.yml
+        device (str): Choose the device you want to run, it can be: CPU/GPU/XPU, default is CPU
+        run_mode (str): mode of running(paddle/trt_fp32/trt_fp16)
+        batch_size (int): size of pre batch in inference
+        trt_min_shape (int): min shape for dynamic shape in trt
+        trt_max_shape (int): max shape for dynamic shape in trt
+        trt_opt_shape (int): opt shape for dynamic shape in trt
+        trt_calib_mode (bool): If the model is produced by TRT offline quantitative
+            calibration, trt_calib_mode need to set True
+        cpu_threads (int): cpu threads
+        enable_mkldnn (bool): whether to open MKLDNN
+        threshold (float): The threshold of score for action feature object detection.
+        display_frames (int): The duration for corresponding detected action.
+        skip_frame_num (int): The number of frames for interval prediction. A skipped frame will 
+            reuse the result of its last frame. If it is set to 0, no frame will be skipped. Default
+            is 0.
+
+    """
+
+    def __init__(self,
+                 model_dir,
+                 device='CPU',
+                 run_mode='paddle',
+                 batch_size=1,
+                 trt_min_shape=1,
+                 trt_max_shape=1280,
+                 trt_opt_shape=640,
+                 trt_calib_mode=False,
+                 cpu_threads=1,
+                 enable_mkldnn=False,
+                 output_dir='output',
+                 threshold=0.5,
+                 display_frames=20,
+                 skip_frame_num=0):
+        super(DetActionRecognizer, self).__init__()
+        self.detector = Detector(
+            model_dir=model_dir,
+            device=device,
+            run_mode=run_mode,
+            batch_size=batch_size,
+            trt_min_shape=trt_min_shape,
+            trt_max_shape=trt_max_shape,
+            trt_opt_shape=trt_opt_shape,
+            trt_calib_mode=trt_calib_mode,
+            cpu_threads=cpu_threads,
+            enable_mkldnn=enable_mkldnn,
+            output_dir=output_dir,
+            threshold=threshold)
+        self.threshold = threshold
+        self.frame_life = display_frames
+        self.result_history = {}
+        self.skip_frame_num = skip_frame_num
+        self.skip_frame_cnt = 0
+        self.id_in_last_frame = []
+
+    @classmethod
+    def init_with_cfg(cls, args, cfg):
+        return cls(model_dir=cfg['model_dir'],
+                   batch_size=cfg['batch_size'],
+                   threshold=cfg['threshold'],
+                   display_frames=cfg['display_frames'],
+                   skip_frame_num=cfg['skip_frame_num'],
+                   device=args.device,
+                   run_mode=args.run_mode,
+                   trt_min_shape=args.trt_min_shape,
+                   trt_max_shape=args.trt_max_shape,
+                   trt_opt_shape=args.trt_opt_shape,
+                   trt_calib_mode=args.trt_calib_mode,
+                   cpu_threads=args.cpu_threads,
+                   enable_mkldnn=args.enable_mkldnn)
+
+    def predict(self, images, mot_result):
+        if self.skip_frame_cnt == 0 or (not self.check_id_is_same(mot_result)):
+            det_result = self.detector.predict_image(images, visual=False)
+            result = self.postprocess(det_result, mot_result)
+        else:
+            result = self.reuse_result(mot_result)
+
+        self.skip_frame_cnt += 1
+        if self.skip_frame_cnt >= self.skip_frame_num:
+            self.skip_frame_cnt = 0
+
+        return result
+
+    def postprocess(self, det_result, mot_result):
+        np_boxes_num = det_result['boxes_num']
+        if np_boxes_num[0] <= 0:
+            return [[], []]
+
+        mot_bboxes = mot_result.get('boxes')
+
+        cur_box_idx = 0
+        mot_id = []
+        act_res = []
+        for idx in range(len(mot_bboxes)):
+            tracker_id = mot_bboxes[idx, 0]
+
+            # Current now,  class 0 is positive, class 1 is negative.
+            action_ret = {'class': 1.0, 'score': -1.0}
+            box_num = np_boxes_num[idx]
+            boxes = det_result['boxes'][cur_box_idx:cur_box_idx + box_num]
+            cur_box_idx += box_num
+            isvalid = (boxes[:, 1] > self.threshold) & (boxes[:, 0] == 0)
+            valid_boxes = boxes[isvalid, :]
+
+            if valid_boxes.shape[0] >= 1:
+                action_ret['class'] = valid_boxes[0, 0]
+                action_ret['score'] = valid_boxes[0, 1]
+                self.result_history[
+                    tracker_id] = [0, self.frame_life, valid_boxes[0, 1]]
+            else:
+                history_det, life_remain, history_score = self.result_history.get(
+                    tracker_id, [1, self.frame_life, -1.0])
+                action_ret['class'] = history_det
+                action_ret['score'] = -1.0
+                life_remain -= 1
+                if life_remain <= 0 and tracker_id in self.result_history:
+                    del (self.result_history[tracker_id])
+                elif tracker_id in self.result_history:
+                    self.result_history[tracker_id][1] = life_remain
+                else:
+                    self.result_history[tracker_id] = [
+                        history_det, life_remain, history_score
+                    ]
+
+            mot_id.append(tracker_id)
+            act_res.append(action_ret)
+        result = list(zip(mot_id, act_res))
+        self.id_in_last_frame = mot_id
+
+        return result
+
+    def check_id_is_same(self, mot_result):
+        mot_bboxes = mot_result.get('boxes')
+        for idx in range(len(mot_bboxes)):
+            tracker_id = mot_bboxes[idx, 0]
+            if tracker_id not in self.id_in_last_frame:
+                return False
+        return True
+
+    def reuse_result(self, mot_result):
+        # This function reusing previous results of the same ID directly.
+        mot_bboxes = mot_result.get('boxes')
+
+        mot_id = []
+        act_res = []
+
+        for idx in range(len(mot_bboxes)):
+            tracker_id = mot_bboxes[idx, 0]
+            history_cls, life_remain, history_score = self.result_history.get(
+                tracker_id, [1, 0, -1.0])
+
+            life_remain -= 1
+            if tracker_id in self.result_history:
+                self.result_history[tracker_id][1] = life_remain
+
+            action_ret = {'class': history_cls, 'score': history_score}
+            mot_id.append(tracker_id)
+            act_res.append(action_ret)
+
+        result = list(zip(mot_id, act_res))
+        self.id_in_last_frame = mot_id
+
+        return result
+
+
+class ClsActionRecognizer(AttrDetector):
+    """
+    Args:
+        model_dir (str): root path of model.pdiparams, model.pdmodel and infer_cfg.yml
+        device (str): Choose the device you want to run, it can be: CPU/GPU/XPU, default is CPU
+        run_mode (str): mode of running(paddle/trt_fp32/trt_fp16)
+        batch_size (int): size of pre batch in inference
+        trt_min_shape (int): min shape for dynamic shape in trt
+        trt_max_shape (int): max shape for dynamic shape in trt
+        trt_opt_shape (int): opt shape for dynamic shape in trt
+        trt_calib_mode (bool): If the model is produced by TRT offline quantitative
+            calibration, trt_calib_mode need to set True
+        cpu_threads (int): cpu threads
+        enable_mkldnn (bool): whether to open MKLDNN
+        threshold (float): The threshold of score for action feature object detection.
+        display_frames (int): The duration for corresponding detected action. 
+        skip_frame_num (int): The number of frames for interval prediction. A skipped frame will 
+            reuse the result of its last frame. If it is set to 0, no frame will be skipped. Default
+            is 0.
+    """
+
+    def __init__(self,
+                 model_dir,
+                 device='CPU',
+                 run_mode='paddle',
+                 batch_size=1,
+                 trt_min_shape=1,
+                 trt_max_shape=1280,
+                 trt_opt_shape=640,
+                 trt_calib_mode=False,
+                 cpu_threads=1,
+                 enable_mkldnn=False,
+                 output_dir='output',
+                 threshold=0.5,
+                 display_frames=80,
+                 skip_frame_num=0):
+        super(ClsActionRecognizer, self).__init__(
+            model_dir=model_dir,
+            device=device,
+            run_mode=run_mode,
+            batch_size=batch_size,
+            trt_min_shape=trt_min_shape,
+            trt_max_shape=trt_max_shape,
+            trt_opt_shape=trt_opt_shape,
+            trt_calib_mode=trt_calib_mode,
+            cpu_threads=cpu_threads,
+            enable_mkldnn=enable_mkldnn,
+            output_dir=output_dir,
+            threshold=threshold)
+        self.threshold = threshold
+        self.frame_life = display_frames
+        self.result_history = {}
+        self.skip_frame_num = skip_frame_num
+        self.skip_frame_cnt = 0
+        self.id_in_last_frame = []
+
+    @classmethod
+    def init_with_cfg(cls, args, cfg):
+        return cls(model_dir=cfg['model_dir'],
+                   batch_size=cfg['batch_size'],
+                   threshold=cfg['threshold'],
+                   display_frames=cfg['display_frames'],
+                   skip_frame_num=cfg['skip_frame_num'],
+                   device=args.device,
+                   run_mode=args.run_mode,
+                   trt_min_shape=args.trt_min_shape,
+                   trt_max_shape=args.trt_max_shape,
+                   trt_opt_shape=args.trt_opt_shape,
+                   trt_calib_mode=args.trt_calib_mode,
+                   cpu_threads=args.cpu_threads,
+                   enable_mkldnn=args.enable_mkldnn)
+
+    def predict_with_mot(self, images, mot_result):
+        if self.skip_frame_cnt == 0 or (not self.check_id_is_same(mot_result)):
+            images = self.crop_half_body(images)
+            cls_result = self.predict_image(images, visual=False)["output"]
+            result = self.match_action_with_id(cls_result, mot_result)
+        else:
+            result = self.reuse_result(mot_result)
+
+        self.skip_frame_cnt += 1
+        if self.skip_frame_cnt >= self.skip_frame_num:
+            self.skip_frame_cnt = 0
+
+        return result
+
+    def crop_half_body(self, images):
+        crop_images = []
+        for image in images:
+            h = image.shape[0]
+            crop_images.append(image[:h // 2 + 1, :, :])
+        return crop_images
+
+    def postprocess(self, inputs, result):
+        # postprocess output of predictor
+        im_results = result['output']
+        batch_res = []
+        for res in im_results:
+            action_res = res.tolist()
+            for cid, score in enumerate(action_res):
+                action_res[cid] = score
+            batch_res.append(action_res)
+        result = {'output': batch_res}
+        return result
+
+    def match_action_with_id(self, cls_result, mot_result):
+        mot_bboxes = mot_result.get('boxes')
+
+        mot_id = []
+        act_res = []
+
+        for idx in range(len(mot_bboxes)):
+            tracker_id = mot_bboxes[idx, 0]
+
+            cls_id_res = 1
+            cls_score_res = -1.0
+            for cls_id in range(len(cls_result[idx])):
+                score = cls_result[idx][cls_id]
+                if score > cls_score_res:
+                    cls_id_res = cls_id
+                    cls_score_res = score
+
+            # Current now,  class 0 is positive, class 1 is negative.
+            if cls_id_res == 1 or (cls_id_res == 0 and
+                                   cls_score_res < self.threshold):
+                history_cls, life_remain, history_score = self.result_history.get(
+                    tracker_id, [1, self.frame_life, -1.0])
+                cls_id_res = history_cls
+                cls_score_res = 1 - cls_score_res
+                life_remain -= 1
+                if life_remain <= 0 and tracker_id in self.result_history:
+                    del (self.result_history[tracker_id])
+                elif tracker_id in self.result_history:
+                    self.result_history[tracker_id][1] = life_remain
+                else:
+                    self.result_history[tracker_id] = [
+                        cls_id_res, life_remain, cls_score_res
+                    ]
+            else:
+                self.result_history[tracker_id] = [
+                    cls_id_res, self.frame_life, cls_score_res
+                ]
+
+            action_ret = {'class': cls_id_res, 'score': cls_score_res}
+            mot_id.append(tracker_id)
+            act_res.append(action_ret)
+        result = list(zip(mot_id, act_res))
+        self.id_in_last_frame = mot_id
+
+        return result
+
+    def check_id_is_same(self, mot_result):
+        mot_bboxes = mot_result.get('boxes')
+        for idx in range(len(mot_bboxes)):
+            tracker_id = mot_bboxes[idx, 0]
+            if tracker_id not in self.id_in_last_frame:
+                return False
+        return True
+
+    def reuse_result(self, mot_result):
+        # This function reusing previous results of the same ID directly.
+        mot_bboxes = mot_result.get('boxes')
+
+        mot_id = []
+        act_res = []
+
+        for idx in range(len(mot_bboxes)):
+            tracker_id = mot_bboxes[idx, 0]
+            history_cls, life_remain, history_score = self.result_history.get(
+                tracker_id, [1, 0, -1.0])
+
+            life_remain -= 1
+            if tracker_id in self.result_history:
+                self.result_history[tracker_id][1] = life_remain
+
+            action_ret = {'class': history_cls, 'score': history_score}
+            mot_id.append(tracker_id)
+            act_res.append(action_ret)
+
+        result = list(zip(mot_id, act_res))
+        self.id_in_last_frame = mot_id
+
+        return result
+
+
+def main():
+    detector = SkeletonActionRecognizer(
+        FLAGS.model_dir,
+        device=FLAGS.device,
+        run_mode=FLAGS.run_mode,
+        batch_size=FLAGS.batch_size,
+        trt_min_shape=FLAGS.trt_min_shape,
+        trt_max_shape=FLAGS.trt_max_shape,
+        trt_opt_shape=FLAGS.trt_opt_shape,
+        trt_calib_mode=FLAGS.trt_calib_mode,
+        cpu_threads=FLAGS.cpu_threads,
+        enable_mkldnn=FLAGS.enable_mkldnn,
+        threshold=FLAGS.threshold,
+        output_dir=FLAGS.output_dir,
+        window_size=FLAGS.window_size,
+        random_pad=FLAGS.random_pad)
+    # predict from numpy array
+    input_list = get_test_skeletons(FLAGS.action_file)
+    detector.predict_skeleton(input_list, FLAGS.run_benchmark, repeats=10)
+    if not FLAGS.run_benchmark:
+        detector.det_times.info(average=True)
+    else:
+        mems = {
+            'cpu_rss_mb': detector.cpu_mem / len(input_list),
+            'gpu_rss_mb': detector.gpu_mem / len(input_list),
+            'gpu_util': detector.gpu_util * 100 / len(input_list)
+        }
+
+        perf_info = detector.det_times.report(average=True)
+        model_dir = FLAGS.model_dir
+        mode = FLAGS.run_mode
+        model_info = {
+            'model_name': model_dir.strip('/').split('/')[-1],
+            'precision': mode.split('_')[-1]
+        }
+        data_info = {
+            'batch_size': FLAGS.batch_size,
+            'shape': "dynamic_shape",
+            'data_num': perf_info['img_num']
+        }
+        det_log = PaddleInferBenchmark(detector.config, model_info, data_info,
+                                       perf_info, mems)
+        det_log('SkeletonAction')
+
+
+if __name__ == '__main__':
+    paddle.enable_static()
+    parser = argsparser()
+    FLAGS = parser.parse_args()
+    print_arguments(FLAGS)
+    FLAGS.device = FLAGS.device.upper()
+    assert FLAGS.device in ['CPU', 'GPU', 'XPU'
+                            ], "device should be CPU, GPU or XPU"
+    assert not FLAGS.use_gpu, "use_gpu has been deprecated, please use --device"
+
+    main()

pipeline/pphuman/action_utils.py

@@ -0,0 +1,114 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+class KeyPointSequence(object):
+    def __init__(self, max_size=100):
+        self.frames = 0
+        self.kpts = []
+        self.bboxes = []
+        self.max_size = max_size
+
+    def save(self, kpt, bbox):
+        self.kpts.append(kpt)
+        self.bboxes.append(bbox)
+        self.frames += 1
+        if self.frames == self.max_size:
+            return True
+        return False
+
+
+class KeyPointBuff(object):
+    def __init__(self, max_size=100):
+        self.flag_track_interrupt = False
+        self.keypoint_saver = dict()
+        self.max_size = max_size
+        self.id_to_pop = set()
+        self.flag_to_pop = False
+
+    def get_state(self):
+        return self.flag_to_pop
+
+    def update(self, kpt_res, mot_res):
+        kpts = kpt_res.get('keypoint')[0]
+        bboxes = kpt_res.get('bbox')
+        mot_bboxes = mot_res.get('boxes')
+        updated_id = set()
+
+        for idx in range(len(kpts)):
+            tracker_id = mot_bboxes[idx, 0]
+            updated_id.add(tracker_id)
+
+            kpt_seq = self.keypoint_saver.get(tracker_id,
+                                              KeyPointSequence(self.max_size))
+            is_full = kpt_seq.save(kpts[idx], bboxes[idx])
+            self.keypoint_saver[tracker_id] = kpt_seq
+
+            #Scene1: result should be popped when frames meet max size
+            if is_full:
+                self.id_to_pop.add(tracker_id)
+                self.flag_to_pop = True
+
+        #Scene2: result of a lost tracker should be popped
+        interrupted_id = set(self.keypoint_saver.keys()) - updated_id
+        if len(interrupted_id) > 0:
+            self.flag_to_pop = True
+            self.id_to_pop.update(interrupted_id)
+
+    def get_collected_keypoint(self):
+        """
+            Output (List): List of keypoint results for Skeletonbased Recognition task, where 
+                           the format of each element is [tracker_id, KeyPointSequence of tracker_id]
+        """
+        output = []
+        for tracker_id in self.id_to_pop:
+            output.append([tracker_id, self.keypoint_saver[tracker_id]])
+            del (self.keypoint_saver[tracker_id])
+        self.flag_to_pop = False
+        self.id_to_pop.clear()
+        return output
+
+
+class ActionVisualHelper(object):
+    def __init__(self, frame_life=20):
+        self.frame_life = frame_life
+        self.action_history = {}
+
+    def get_visualize_ids(self):
+        id_detected = self.check_detected()
+        return id_detected
+
+    def check_detected(self):
+        id_detected = set()
+        deperate_id = []
+        for mot_id in self.action_history:
+            self.action_history[mot_id]["life_remain"] -= 1
+            if int(self.action_history[mot_id]["class"]) == 0:
+                id_detected.add(mot_id)
+            if self.action_history[mot_id]["life_remain"] == 0:
+                deperate_id.append(mot_id)
+        for mot_id in deperate_id:
+            del (self.action_history[mot_id])
+        return id_detected
+
+    def update(self, action_res_list):
+        for mot_id, action_res in action_res_list:
+            if mot_id in self.action_history:
+                if int(action_res["class"]) != 0 and int(self.action_history[
+                        mot_id]["class"]) == 0:
+                    continue
+            action_info = self.action_history.get(mot_id, {})
+            action_info["class"] = action_res["class"]
+            action_info["life_remain"] = self.frame_life
+            self.action_history[mot_id] = action_info

pipeline/pphuman/attr_infer.py

@@ -0,0 +1,348 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import yaml
+import glob
+from functools import reduce
+
+import cv2
+import numpy as np
+import math
+import paddle
+from paddle.inference import Config
+from paddle.inference import create_predictor
+
+import sys
+# add deploy path of PadleDetection to sys.path
+parent_path = os.path.abspath(os.path.join(__file__, *(['..'])))
+sys.path.insert(0, parent_path)
+
+from python.benchmark_utils import PaddleInferBenchmark
+from python.preprocess import preprocess, Resize, NormalizeImage, Permute, PadStride, LetterBoxResize, WarpAffine
+from python.visualize import visualize_attr
+from python.utils import argsparser, Timer, get_current_memory_mb
+from python.infer import Detector, get_test_images, print_arguments, load_predictor
+
+from PIL import Image, ImageDraw, ImageFont
+
+
+class AttrDetector(Detector):
+    """
+    Args:
+        model_dir (str): root path of model.pdiparams, model.pdmodel and infer_cfg.yml
+        device (str): Choose the device you want to run, it can be: CPU/GPU/XPU, default is CPU
+        run_mode (str): mode of running(paddle/trt_fp32/trt_fp16)
+        batch_size (int): size of pre batch in inference
+        trt_min_shape (int): min shape for dynamic shape in trt
+        trt_max_shape (int): max shape for dynamic shape in trt
+        trt_opt_shape (int): opt shape for dynamic shape in trt
+        trt_calib_mode (bool): If the model is produced by TRT offline quantitative
+            calibration, trt_calib_mode need to set True
+        cpu_threads (int): cpu threads
+        enable_mkldnn (bool): whether to open MKLDNN
+        output_dir (str): The path of output
+        threshold (float): The threshold of score for visualization
+    """
+
+    def __init__(
+            self,
+            model_dir,
+            device='CPU',
+            run_mode='paddle',
+            batch_size=1,
+            trt_min_shape=1,
+            trt_max_shape=1280,
+            trt_opt_shape=640,
+            trt_calib_mode=False,
+            cpu_threads=1,
+            enable_mkldnn=False,
+            output_dir='output',
+            threshold=0.5, ):
+        super(AttrDetector, self).__init__(
+            model_dir=model_dir,
+            device=device,
+            run_mode=run_mode,
+            batch_size=batch_size,
+            trt_min_shape=trt_min_shape,
+            trt_max_shape=trt_max_shape,
+            trt_opt_shape=trt_opt_shape,
+            trt_calib_mode=trt_calib_mode,
+            cpu_threads=cpu_threads,
+            enable_mkldnn=enable_mkldnn,
+            output_dir=output_dir,
+            threshold=threshold, )
+
+    @classmethod
+    def init_with_cfg(cls, args, cfg):
+        return cls(model_dir=cfg['model_dir'],
+                   batch_size=cfg['batch_size'],
+                   device=args.device,
+                   run_mode=args.run_mode,
+                   trt_min_shape=args.trt_min_shape,
+                   trt_max_shape=args.trt_max_shape,
+                   trt_opt_shape=args.trt_opt_shape,
+                   trt_calib_mode=args.trt_calib_mode,
+                   cpu_threads=args.cpu_threads,
+                   enable_mkldnn=args.enable_mkldnn)
+
+    def get_label(self):
+        return self.pred_config.labels
+
+    def postprocess(self, inputs, result):
+        # postprocess output of predictor
+        im_results = result['output']
+
+        labels = self.pred_config.labels
+        age_list = ['AgeLess18', 'Age18-60', 'AgeOver60']
+        direct_list = ['Front', 'Side', 'Back']
+        bag_list = ['HandBag', 'ShoulderBag', 'Backpack']
+        upper_list = ['UpperStride', 'UpperLogo', 'UpperPlaid', 'UpperSplice']
+        lower_list = [
+            'LowerStripe', 'LowerPattern', 'LongCoat', 'Trousers', 'Shorts',
+            'Skirt&Dress'
+        ]
+        glasses_threshold = 0.3
+        hold_threshold = 0.6
+        batch_res = []
+        for res in im_results:
+            res = res.tolist()
+            label_res = []
+            # gender 
+            gender = 'Female' if res[22] > self.threshold else 'Male'
+            label_res.append(gender)
+            # age
+            age = age_list[np.argmax(res[19:22])]
+            label_res.append(age)
+            # direction 
+            direction = direct_list[np.argmax(res[23:])]
+            label_res.append(direction)
+            # glasses
+            glasses = 'Glasses: '
+            if res[1] > glasses_threshold:
+                glasses += 'True'
+            else:
+                glasses += 'False'
+            label_res.append(glasses)
+            # hat
+            hat = 'Hat: '
+            if res[0] > self.threshold:
+                hat += 'True'
+            else:
+                hat += 'False'
+            label_res.append(hat)
+            # hold obj
+            hold_obj = 'HoldObjectsInFront: '
+            if res[18] > hold_threshold:
+                hold_obj += 'True'
+            else:
+                hold_obj += 'False'
+            label_res.append(hold_obj)
+            # bag
+            bag = bag_list[np.argmax(res[15:18])]
+            bag_score = res[15 + np.argmax(res[15:18])]
+            bag_label = bag if bag_score > self.threshold else 'No bag'
+            label_res.append(bag_label)
+            # upper
+            upper_label = 'Upper:'
+            sleeve = 'LongSleeve' if res[3] > res[2] else 'ShortSleeve'
+            upper_label += ' {}'.format(sleeve)
+            upper_res = res[4:8]
+            if np.max(upper_res) > self.threshold:
+                upper_label += ' {}'.format(upper_list[np.argmax(upper_res)])
+            label_res.append(upper_label)
+            # lower
+            lower_res = res[8:14]
+            lower_label = 'Lower: '
+            has_lower = False
+            for i, l in enumerate(lower_res):
+                if l > self.threshold:
+                    lower_label += ' {}'.format(lower_list[i])
+                    has_lower = True
+            if not has_lower:
+                lower_label += ' {}'.format(lower_list[np.argmax(lower_res)])
+
+            label_res.append(lower_label)
+            # shoe
+            shoe = 'Boots' if res[14] > self.threshold else 'No boots'
+            label_res.append(shoe)
+
+            batch_res.append(label_res)
+        result = {'output': batch_res}
+        return result
+
+    def predict(self, repeats=1):
+        '''
+        Args:
+            repeats (int): repeats number for prediction
+        Returns:
+            result (dict): include 'boxes': np.ndarray: shape:[N,6], N: number of box,
+                            matix element:[class, score, x_min, y_min, x_max, y_max]
+                            MaskRCNN's result include 'masks': np.ndarray:
+                            shape: [N, im_h, im_w]
+        '''
+        # model prediction
+        for i in range(repeats):
+            self.predictor.run()
+            output_names = self.predictor.get_output_names()
+            output_tensor = self.predictor.get_output_handle(output_names[0])
+            np_output = output_tensor.copy_to_cpu()
+        result = dict(output=np_output)
+        return result
+
+    def predict_image(self,
+                      image_list,
+                      run_benchmark=False,
+                      repeats=1,
+                      visual=True):
+        batch_loop_cnt = math.ceil(float(len(image_list)) / self.batch_size)
+        results = []
+        for i in range(batch_loop_cnt):
+            start_index = i * self.batch_size
+            end_index = min((i + 1) * self.batch_size, len(image_list))
+            batch_image_list = image_list[start_index:end_index]
+            if run_benchmark:
+                # preprocess
+                inputs = self.preprocess(batch_image_list)  # warmup
+                self.det_times.preprocess_time_s.start()
+                inputs = self.preprocess(batch_image_list)
+                self.det_times.preprocess_time_s.end()
+
+                # model prediction
+                result = self.predict(repeats=repeats)  # warmup
+                self.det_times.inference_time_s.start()
+                result = self.predict(repeats=repeats)
+                self.det_times.inference_time_s.end(repeats=repeats)
+
+                # postprocess
+                result_warmup = self.postprocess(inputs, result)  # warmup
+                self.det_times.postprocess_time_s.start()
+                result = self.postprocess(inputs, result)
+                self.det_times.postprocess_time_s.end()
+                self.det_times.img_num += len(batch_image_list)
+
+                cm, gm, gu = get_current_memory_mb()
+                self.cpu_mem += cm
+                self.gpu_mem += gm
+                self.gpu_util += gu
+            else:
+                # preprocess
+                self.det_times.preprocess_time_s.start()
+                inputs = self.preprocess(batch_image_list)
+                self.det_times.preprocess_time_s.end()
+
+                # model prediction
+                self.det_times.inference_time_s.start()
+                result = self.predict()
+                self.det_times.inference_time_s.end()
+
+                # postprocess
+                self.det_times.postprocess_time_s.start()
+                result = self.postprocess(inputs, result)
+                self.det_times.postprocess_time_s.end()
+                self.det_times.img_num += len(batch_image_list)
+
+                if visual:
+                    visualize(
+                        batch_image_list, result, output_dir=self.output_dir)
+
+            results.append(result)
+            if visual:
+                print('Test iter {}'.format(i))
+
+        results = self.merge_batch_result(results)
+        return results
+
+    def merge_batch_result(self, batch_result):
+        if len(batch_result) == 1:
+            return batch_result[0]
+        res_key = batch_result[0].keys()
+        results = {k: [] for k in res_key}
+        for res in batch_result:
+            for k, v in res.items():
+                results[k].extend(v)
+        return results
+
+
+def visualize(image_list, batch_res, output_dir='output'):
+
+    # visualize the predict result
+    batch_res = batch_res['output']
+    for image_file, res in zip(image_list, batch_res):
+        im = visualize_attr(image_file, [res])
+        if not os.path.exists(output_dir):
+            os.makedirs(output_dir)
+        img_name = os.path.split(image_file)[-1]
+        out_path = os.path.join(output_dir, img_name)
+        cv2.imwrite(out_path, im)
+        print("save result to: " + out_path)
+
+
+def main():
+    detector = AttrDetector(
+        FLAGS.model_dir,
+        device=FLAGS.device,
+        run_mode=FLAGS.run_mode,
+        batch_size=FLAGS.batch_size,
+        trt_min_shape=FLAGS.trt_min_shape,
+        trt_max_shape=FLAGS.trt_max_shape,
+        trt_opt_shape=FLAGS.trt_opt_shape,
+        trt_calib_mode=FLAGS.trt_calib_mode,
+        cpu_threads=FLAGS.cpu_threads,
+        enable_mkldnn=FLAGS.enable_mkldnn,
+        threshold=FLAGS.threshold,
+        output_dir=FLAGS.output_dir)
+
+    # predict from image
+    if FLAGS.image_dir is None and FLAGS.image_file is not None:
+        assert FLAGS.batch_size == 1, "batch_size should be 1, when image_file is not None"
+    img_list = get_test_images(FLAGS.image_dir, FLAGS.image_file)
+    detector.predict_image(img_list, FLAGS.run_benchmark, repeats=10)
+    if not FLAGS.run_benchmark:
+        detector.det_times.info(average=True)
+    else:
+        mems = {
+            'cpu_rss_mb': detector.cpu_mem / len(img_list),
+            'gpu_rss_mb': detector.gpu_mem / len(img_list),
+            'gpu_util': detector.gpu_util * 100 / len(img_list)
+        }
+
+        perf_info = detector.det_times.report(average=True)
+        model_dir = FLAGS.model_dir
+        mode = FLAGS.run_mode
+        model_info = {
+            'model_name': model_dir.strip('/').split('/')[-1],
+            'precision': mode.split('_')[-1]
+        }
+        data_info = {
+            'batch_size': FLAGS.batch_size,
+            'shape': "dynamic_shape",
+            'data_num': perf_info['img_num']
+        }
+        det_log = PaddleInferBenchmark(detector.config, model_info, data_info,
+                                       perf_info, mems)
+        det_log('Attr')
+
+
+if __name__ == '__main__':
+    paddle.enable_static()
+    parser = argsparser()
+    FLAGS = parser.parse_args()
+    print_arguments(FLAGS)
+    FLAGS.device = FLAGS.device.upper()
+    assert FLAGS.device in ['CPU', 'GPU', 'XPU'
+                            ], "device should be CPU, GPU or XPU"
+    assert not FLAGS.use_gpu, "use_gpu has been deprecated, please use --device"
+
+    main()

pipeline/pphuman/mtmct.py

@@ -0,0 +1,381 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from pptracking.python.mot.visualize import plot_tracking
+from python.visualize import visualize_attr
+import os
+import re
+import cv2
+import gc
+import numpy as np
+try:
+    from sklearn import preprocessing
+    from sklearn.cluster import AgglomerativeClustering
+except:
+    print(
+        'Warning: Unable to use MTMCT in PP-Human, please install sklearn, for example: `pip install sklearn`'
+    )
+    pass
+import pandas as pd
+from tqdm import tqdm
+from functools import reduce
+import warnings
+warnings.filterwarnings("ignore")
+
+
+def gen_restxt(output_dir_filename, map_tid, cid_tid_dict):
+    pattern = re.compile(r'c(\d)_t(\d)')
+    f_w = open(output_dir_filename, 'w')
+    for key, res in cid_tid_dict.items():
+        cid, tid = pattern.search(key).groups()
+        cid = int(cid) + 1
+        rects = res["rects"]
+        frames = res["frames"]
+        for idx, bbox in enumerate(rects):
+            bbox[0][3:] -= bbox[0][1:3]
+            fid = frames[idx] + 1
+            rect = [max(int(x), 0) for x in bbox[0][1:]]
+            if key in map_tid:
+                new_tid = map_tid[key]
+                f_w.write(
+                    str(cid) + ' ' + str(new_tid) + ' ' + str(fid) + ' ' +
+                    ' '.join(map(str, rect)) + '\n')
+    print('gen_res: write file in {}'.format(output_dir_filename))
+    f_w.close()
+
+
+def get_mtmct_matching_results(pred_mtmct_file,
+                               secs_interval=0.5,
+                               video_fps=20):
+    res = np.loadtxt(pred_mtmct_file)  # 'cid, tid, fid, x1, y1, w, h, -1, -1'
+    camera_ids = list(map(int, np.unique(res[:, 0])))
+
+    res = res[:, :7]
+    # each line in res: 'cid, tid, fid, x1, y1, w, h'
+
+    camera_tids = []
+    camera_results = dict()
+    for c_id in camera_ids:
+        camera_results[c_id] = res[res[:, 0] == c_id]
+        tids = np.unique(camera_results[c_id][:, 1])
+        tids = list(map(int, tids))
+        camera_tids.append(tids)
+
+    # select common tids throughout each video
+    common_tids = reduce(np.intersect1d, camera_tids)
+
+    # get mtmct matching results by cid_tid_fid_results[c_id][t_id][f_id]
+    cid_tid_fid_results = dict()
+    cid_tid_to_fids = dict()
+    interval = int(secs_interval * video_fps)  # preferably less than 10
+    for c_id in camera_ids:
+        cid_tid_fid_results[c_id] = dict()
+        cid_tid_to_fids[c_id] = dict()
+        for t_id in common_tids:
+            tid_mask = camera_results[c_id][:, 1] == t_id
+            cid_tid_fid_results[c_id][t_id] = dict()
+
+            camera_trackid_results = camera_results[c_id][tid_mask]
+            fids = np.unique(camera_trackid_results[:, 2])
+            fids = fids[fids % interval == 0]
+            fids = list(map(int, fids))
+            cid_tid_to_fids[c_id][t_id] = fids
+
+            for f_id in fids:
+                st_frame = f_id
+                ed_frame = f_id + interval
+
+                st_mask = camera_trackid_results[:, 2] >= st_frame
+                ed_mask = camera_trackid_results[:, 2] < ed_frame
+                frame_mask = np.logical_and(st_mask, ed_mask)
+                cid_tid_fid_results[c_id][t_id][f_id] = camera_trackid_results[
+                    frame_mask]
+
+    return camera_results, cid_tid_fid_results
+
+
+def save_mtmct_vis_results(camera_results,
+                           captures,
+                           output_dir,
+                           multi_res=None):
+    # camera_results: 'cid, tid, fid, x1, y1, w, h'
+    camera_ids = list(camera_results.keys())
+
+    import shutil
+    save_dir = os.path.join(output_dir, 'mtmct_vis')
+    if os.path.exists(save_dir):
+        shutil.rmtree(save_dir)
+    os.makedirs(save_dir)
+
+    for idx, video_file in enumerate(captures):
+        capture = cv2.VideoCapture(video_file)
+        cid = camera_ids[idx]
+        basename = os.path.basename(video_file)
+        video_out_name = "vis_" + basename
+        out_path = os.path.join(save_dir, video_out_name)
+        print("Start visualizing output video: {}".format(out_path))
+
+        # Get Video info : resolution, fps, frame count
+        width = int(capture.get(cv2.CAP_PROP_FRAME_WIDTH))
+        height = int(capture.get(cv2.CAP_PROP_FRAME_HEIGHT))
+        fps = int(capture.get(cv2.CAP_PROP_FPS))
+        frame_count = int(capture.get(cv2.CAP_PROP_FRAME_COUNT))
+        fourcc = cv2.VideoWriter_fourcc(*'mp4v')
+        writer = cv2.VideoWriter(out_path, fourcc, fps, (width, height))
+        frame_id = 0
+        while (1):
+            if frame_id % 50 == 0:
+                print('frame id: ', frame_id)
+            ret, frame = capture.read()
+            frame_id += 1
+            if not ret:
+                if frame_id == 1:
+                    print("video read failed!")
+                break
+            frame_results = camera_results[cid][camera_results[cid][:, 2] ==
+                                                frame_id]
+            boxes = frame_results[:, -4:]
+            ids = frame_results[:, 1]
+            image = plot_tracking(
+                frame, boxes, ids, frame_id=frame_id, fps=fps)
+
+            # add attr vis
+            if multi_res:
+                tid_list = multi_res.keys()  # c0_t1, c0_t2...
+                all_attr_result = [multi_res[i]["attrs"]
+                                   for i in tid_list]  # all cid_tid result
+                if any(
+                        all_attr_result
+                ):  # at least one cid_tid[attrs] is not None will goes to attrs_vis
+                    attr_res = []
+                    for k in tid_list:
+                        if (frame_id - 1) >= len(multi_res[k]['attrs']):
+                            t_attr = None
+                        else:
+                            t_attr = multi_res[k]['attrs'][frame_id - 1]
+                            attr_res.append(t_attr)
+                    image = visualize_attr(
+                        image, attr_res, boxes, is_mtmct=True)
+
+            writer.write(image)
+        writer.release()
+
+
+def get_euclidean(x, y, **kwargs):
+    m = x.shape[0]
+    n = y.shape[0]
+    distmat = (np.power(x, 2).sum(axis=1, keepdims=True).repeat(
+        n, axis=1) + np.power(y, 2).sum(axis=1, keepdims=True).repeat(
+            m, axis=1).T)
+    distmat -= np.dot(2 * x, y.T)
+    return distmat
+
+
+def cosine_similarity(x, y, eps=1e-12):
+    """
+    Computes cosine similarity between two tensors.
+    Value == 1 means the same vector
+    Value == 0 means perpendicular vectors
+    """
+    x_n, y_n = np.linalg.norm(
+        x, axis=1, keepdims=True), np.linalg.norm(
+            y, axis=1, keepdims=True)
+    x_norm = x / np.maximum(x_n, eps * np.ones_like(x_n))
+    y_norm = y / np.maximum(y_n, eps * np.ones_like(y_n))
+    sim_mt = np.dot(x_norm, y_norm.T)
+    return sim_mt
+
+
+def get_cosine(x, y, eps=1e-12):
+    """
+    Computes cosine distance between two tensors.
+    The cosine distance is the inverse cosine similarity
+    -> cosine_distance = abs(-cosine_distance) to make it
+    similar in behaviour to euclidean distance
+    """
+    sim_mt = cosine_similarity(x, y, eps)
+    return sim_mt
+
+
+def get_dist_mat(x, y, func_name="euclidean"):
+    if func_name == "cosine":
+        dist_mat = get_cosine(x, y)
+    elif func_name == "euclidean":
+        dist_mat = get_euclidean(x, y)
+    print("Using {} as distance function during evaluation".format(func_name))
+    return dist_mat
+
+
+def intracam_ignore(st_mask, cid_tids):
+    count = len(cid_tids)
+    for i in range(count):
+        for j in range(count):
+            if cid_tids[i][1] == cid_tids[j][1]:
+                st_mask[i, j] = 0.
+    return st_mask
+
+
+def get_sim_matrix_new(cid_tid_dict, cid_tids):
+    # Note: camera independent get_sim_matrix function,
+    # which is different from the one in camera_utils.py.
+    count = len(cid_tids)
+
+    q_arr = np.array(
+        [cid_tid_dict[cid_tids[i]]['mean_feat'] for i in range(count)])
+    g_arr = np.array(
+        [cid_tid_dict[cid_tids[i]]['mean_feat'] for i in range(count)])
+    #compute distmat
+    distmat = get_dist_mat(q_arr, g_arr, func_name="cosine")
+
+    #mask the element which belongs to same video
+    st_mask = np.ones((count, count), dtype=np.float32)
+    st_mask = intracam_ignore(st_mask, cid_tids)
+
+    sim_matrix = distmat * st_mask
+    np.fill_diagonal(sim_matrix, 0.)
+    return 1. - sim_matrix
+
+
+def get_match(cluster_labels):
+    cluster_dict = dict()
+    cluster = list()
+    for i, l in enumerate(cluster_labels):
+        if l in list(cluster_dict.keys()):
+            cluster_dict[l].append(i)
+        else:
+            cluster_dict[l] = [i]
+    for idx in cluster_dict:
+        cluster.append(cluster_dict[idx])
+    return cluster
+
+
+def get_cid_tid(cluster_labels, cid_tids):
+    cluster = list()
+    for labels in cluster_labels:
+        cid_tid_list = list()
+        for label in labels:
+            cid_tid_list.append(cid_tids[label])
+        cluster.append(cid_tid_list)
+    return cluster
+
+
+def get_labels(cid_tid_dict, cid_tids):
+    #compute cost matrix between features
+    cost_matrix = get_sim_matrix_new(cid_tid_dict, cid_tids)
+
+    #cluster all the features
+    cluster1 = AgglomerativeClustering(
+        n_clusters=None,
+        distance_threshold=0.5,
+        affinity='precomputed',
+        linkage='complete')
+    cluster_labels1 = cluster1.fit_predict(cost_matrix)
+    labels = get_match(cluster_labels1)
+
+    sub_cluster = get_cid_tid(labels, cid_tids)
+    return labels
+
+
+def sub_cluster(cid_tid_dict):
+    '''
+    cid_tid_dict: all camera_id and track_id
+    '''
+    #get all keys
+    cid_tids = sorted([key for key in cid_tid_dict.keys()])
+
+    #cluster all trackid
+    clu = get_labels(cid_tid_dict, cid_tids)
+
+    #relabel every cluster groups
+    new_clu = list()
+    for c_list in clu:
+        new_clu.append([cid_tids[c] for c in c_list])
+    cid_tid_label = dict()
+    for i, c_list in enumerate(new_clu):
+        for c in c_list:
+            cid_tid_label[c] = i + 1
+    return cid_tid_label
+
+
+def distill_idfeat(mot_res):
+    qualities_list = mot_res["qualities"]
+    feature_list = mot_res["features"]
+    rects = mot_res["rects"]
+
+    qualities_new = []
+    feature_new = []
+    #filter rect less than 100*20
+    for idx, rect in enumerate(rects):
+        conf, xmin, ymin, xmax, ymax = rect[0]
+        if (xmax - xmin) * (ymax - ymin) and (xmax > xmin) > 2000:
+            qualities_new.append(qualities_list[idx])
+            feature_new.append(feature_list[idx])
+    #take all features if available rect is less than 2
+    if len(qualities_new) < 2:
+        qualities_new = qualities_list
+        feature_new = feature_list
+
+    #if available frames number is more than 200, take one frame data per 20 frames
+    skipf = 1
+    if len(qualities_new) > 20:
+        skipf = 2
+    quality_skip = np.array(qualities_new[::skipf])
+    feature_skip = np.array(feature_new[::skipf])
+
+    #sort features with image qualities, take the most trustworth features
+    topk_argq = np.argsort(quality_skip)[::-1]
+    if (quality_skip > 0.6).sum() > 1:
+        topk_feat = feature_skip[topk_argq[quality_skip > 0.6]]
+    else:
+        topk_feat = feature_skip[topk_argq]
+
+    #get final features by mean or cluster, at most take five
+    mean_feat = np.mean(topk_feat[:5], axis=0)
+    return mean_feat
+
+
+def res2dict(multi_res):
+    cid_tid_dict = {}
+    for cid, c_res in enumerate(multi_res):
+        for tid, res in c_res.items():
+            key = "c" + str(cid) + "_t" + str(tid)
+            if key not in cid_tid_dict:
+                if len(res["features"]) == 0:
+                    continue
+                cid_tid_dict[key] = res
+                cid_tid_dict[key]['mean_feat'] = distill_idfeat(res)
+    return cid_tid_dict
+
+
+def mtmct_process(multi_res, captures, mtmct_vis=True, output_dir="output"):
+    cid_tid_dict = res2dict(multi_res)
+    if len(cid_tid_dict) == 0:
+        print("no tracking result found, mtmct will be skiped.")
+        return
+    map_tid = sub_cluster(cid_tid_dict)
+
+    if not os.path.exists(output_dir):
+        os.mkdir(output_dir)
+    pred_mtmct_file = os.path.join(output_dir, 'mtmct_result.txt')
+    gen_restxt(pred_mtmct_file, map_tid, cid_tid_dict)
+
+    if mtmct_vis:
+        camera_results, cid_tid_fid_res = get_mtmct_matching_results(
+            pred_mtmct_file)
+
+        save_mtmct_vis_results(
+            camera_results,
+            captures,
+            output_dir=output_dir,
+            multi_res=cid_tid_dict)

pipeline/pphuman/reid.py

@@ -0,0 +1,204 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+import cv2
+import numpy as np
+# add deploy path of PadleDetection to sys.path
+parent_path = os.path.abspath(os.path.join(__file__, *(['..'] * 2)))
+sys.path.insert(0, parent_path)
+
+from python.infer import PredictConfig
+from pptracking.python.det_infer import load_predictor
+from python.utils import Timer
+
+
+class ReID(object):
+    """
+    ReID of SDE methods
+
+    Args:
+        pred_config (object): config of model, defined by `Config(model_dir)`
+        model_dir (str): root path of model.pdiparams, model.pdmodel and infer_cfg.yml
+        device (str): Choose the device you want to run, it can be: CPU/GPU/XPU, default is CPU
+        run_mode (str): mode of running(paddle/trt_fp32/trt_fp16)
+        batch_size (int): size of per batch in inference, default 50 means at most
+            50 sub images can be made a batch and send into ReID model
+        trt_min_shape (int): min shape for dynamic shape in trt
+        trt_max_shape (int): max shape for dynamic shape in trt
+        trt_opt_shape (int): opt shape for dynamic shape in trt
+        trt_calib_mode (bool): If the model is produced by TRT offline quantitative
+            calibration, trt_calib_mode need to set True
+        cpu_threads (int): cpu threads
+        enable_mkldnn (bool): whether to open MKLDNN
+    """
+
+    def __init__(self,
+                 model_dir,
+                 device='CPU',
+                 run_mode='paddle',
+                 batch_size=50,
+                 trt_min_shape=1,
+                 trt_max_shape=1088,
+                 trt_opt_shape=608,
+                 trt_calib_mode=False,
+                 cpu_threads=4,
+                 enable_mkldnn=False):
+        self.pred_config = self.set_config(model_dir)
+        self.predictor, self.config = load_predictor(
+            model_dir,
+            run_mode=run_mode,
+            batch_size=batch_size,
+            min_subgraph_size=self.pred_config.min_subgraph_size,
+            device=device,
+            use_dynamic_shape=self.pred_config.use_dynamic_shape,
+            trt_min_shape=trt_min_shape,
+            trt_max_shape=trt_max_shape,
+            trt_opt_shape=trt_opt_shape,
+            trt_calib_mode=trt_calib_mode,
+            cpu_threads=cpu_threads,
+            enable_mkldnn=enable_mkldnn)
+        self.det_times = Timer()
+        self.cpu_mem, self.gpu_mem, self.gpu_util = 0, 0, 0
+        self.batch_size = batch_size
+        self.input_wh = (128, 256)
+
+    @classmethod
+    def init_with_cfg(cls, args, cfg):
+        return cls(model_dir=cfg['model_dir'],
+                   batch_size=cfg['batch_size'],
+                   device=args.device,
+                   run_mode=args.run_mode,
+                   trt_min_shape=args.trt_min_shape,
+                   trt_max_shape=args.trt_max_shape,
+                   trt_opt_shape=args.trt_opt_shape,
+                   trt_calib_mode=args.trt_calib_mode,
+                   cpu_threads=args.cpu_threads,
+                   enable_mkldnn=args.enable_mkldnn)
+
+    def set_config(self, model_dir):
+        return PredictConfig(model_dir)
+
+    def check_img_quality(self, crop, bbox, xyxy):
+        if crop is None:
+            return None
+        #eclipse
+        eclipse_quality = 1.0
+        inner_rect = np.zeros(xyxy.shape)
+        inner_rect[:, :2] = np.maximum(xyxy[:, :2], bbox[None, :2])
+        inner_rect[:, 2:] = np.minimum(xyxy[:, 2:], bbox[None, 2:])
+        wh_array = inner_rect[:, 2:] - inner_rect[:, :2]
+        filt = np.logical_and(wh_array[:, 0] > 0, wh_array[:, 1] > 0)
+        wh_array = wh_array[filt]
+        if wh_array.shape[0] > 1:
+            eclipse_ratio = wh_array / (bbox[2:] - bbox[:2])
+            eclipse_area_ratio = eclipse_ratio[:, 0] * eclipse_ratio[:, 1]
+            ear_lst = eclipse_area_ratio.tolist()
+            ear_lst.sort(reverse=True)
+            eclipse_quality = 1.0 - ear_lst[1]
+        bbox_wh = (bbox[2:] - bbox[:2])
+        height_quality = bbox_wh[1] / (bbox_wh[0] * 2)
+        eclipse_quality = min(eclipse_quality, height_quality)
+
+        #definition
+        cropgray = cv2.cvtColor(crop, cv2.COLOR_BGR2GRAY)
+        definition = int(cv2.Laplacian(cropgray, cv2.CV_64F, ksize=3).var())
+        brightness = int(cropgray.mean())
+        bd_quality = min(1., brightness / 50.)
+
+        eclipse_weight = 0.7
+        return eclipse_quality * eclipse_weight + bd_quality * (1 -
+                                                                eclipse_weight)
+
+    def normal_crop(self, image, rect):
+        imgh, imgw, c = image.shape
+        label, conf, xmin, ymin, xmax, ymax = [int(x) for x in rect.tolist()]
+        xmin = max(0, xmin)
+        ymin = max(0, ymin)
+        xmax = min(imgw, xmax)
+        ymax = min(imgh, ymax)
+        if label != 0 or xmax <= xmin or ymax <= ymin:
+            print("Warning! label missed!!")
+            return None, None, None
+        return image[ymin:ymax, xmin:xmax, :]
+
+    def crop_image_with_mot(self, image, mot_res):
+        res = mot_res['boxes']
+        crop_res = []
+        img_quality = []
+        rects = []
+        for box in res:
+            crop_image = self.normal_crop(image, box[1:])
+            quality_item = self.check_img_quality(crop_image, box[3:],
+                                                  res[:, 3:])
+            if crop_image is not None:
+                crop_res.append(crop_image)
+                img_quality.append(quality_item)
+                rects.append(box)
+        return crop_res, img_quality, rects
+
+    def preprocess(self,
+                   imgs,
+                   mean=[0.485, 0.456, 0.406],
+                   std=[0.229, 0.224, 0.225]):
+        im_batch = []
+        for img in imgs:
+            img = cv2.resize(img, self.input_wh)
+            img = img.astype('float32') / 255.
+            img -= np.array(mean)
+            img /= np.array(std)
+            im_batch.append(img.transpose((2, 0, 1)))
+        inputs = {}
+        inputs['x'] = np.array(im_batch).astype('float32')
+        return inputs
+
+    def predict(self, crops, repeats=1, add_timer=True, seq_name=''):
+        # preprocess
+        if add_timer:
+            self.det_times.preprocess_time_s.start()
+        inputs = self.preprocess(crops)
+        input_names = self.predictor.get_input_names()
+        for i in range(len(input_names)):
+            input_tensor = self.predictor.get_input_handle(input_names[i])
+            input_tensor.copy_from_cpu(inputs[input_names[i]])
+
+        if add_timer:
+            self.det_times.preprocess_time_s.end()
+            self.det_times.inference_time_s.start()
+
+        # model prediction
+        for i in range(repeats):
+            self.predictor.run()
+            output_names = self.predictor.get_output_names()
+            feature_tensor = self.predictor.get_output_handle(output_names[0])
+            pred_embs = feature_tensor.copy_to_cpu()
+        if add_timer:
+            self.det_times.inference_time_s.end(repeats=repeats)
+            self.det_times.postprocess_time_s.start()
+
+        if add_timer:
+            self.det_times.postprocess_time_s.end()
+            self.det_times.img_num += 1
+        return pred_embs
+
+    def predict_batch(self, imgs, batch_size=4):
+        batch_feat = []
+        for b in range(0, len(imgs), batch_size):
+            b_end = min(len(imgs), b + batch_size)
+            batch_imgs = imgs[b:b_end]
+            feat = self.predict(batch_imgs)
+            batch_feat.extend(feat.tolist())
+
+        return batch_feat

pipeline/pphuman/video_action_infer.py

@@ -0,0 +1,310 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import yaml
+import glob
+
+import cv2
+import numpy as np
+import math
+import paddle
+import sys
+import paddle.nn.functional as F
+
+# add deploy path of PadleDetection to sys.path
+parent_path = os.path.abspath(os.path.join(__file__, *(['..'] * 2)))
+sys.path.insert(0, parent_path)
+
+from paddle.inference import Config, create_predictor
+from python.utils import argsparser, Timer, get_current_memory_mb
+from python.benchmark_utils import PaddleInferBenchmark
+from python.infer import Detector, print_arguments
+from video_action_preprocess import VideoDecoder, Sampler, Scale, CenterCrop, Normalization, Image2Array
+
+
+def softmax(x):
+    f_x = np.exp(x) / np.sum(np.exp(x))
+    return f_x
+
+
+class VideoActionRecognizer(object):
+    """
+    Args:
+        model_dir (str): root path of model.pdiparams, model.pdmodel and infer_cfg.yml
+        device (str): Choose the device you want to run, it can be: CPU/GPU/XPU, default is CPU
+        run_mode (str): mode of running(paddle/trt_fp32/trt_fp16)
+        batch_size (int): size of pre batch in inference
+        trt_min_shape (int): min shape for dynamic shape in trt
+        trt_max_shape (int): max shape for dynamic shape in trt
+        trt_opt_shape (int): opt shape for dynamic shape in trt
+        trt_calib_mode (bool): If the model is produced by TRT offline quantitative
+            calibration, trt_calib_mode need to set True
+        cpu_threads (int): cpu threads
+        enable_mkldnn (bool): whether to open MKLDNN
+    """
+
+    def __init__(self,
+                 model_dir,
+                 device='CPU',
+                 run_mode='paddle',
+                 num_seg=8,
+                 seg_len=1,
+                 short_size=256,
+                 target_size=224,
+                 top_k=1,
+                 batch_size=1,
+                 trt_min_shape=1,
+                 trt_max_shape=1280,
+                 trt_opt_shape=640,
+                 trt_calib_mode=False,
+                 cpu_threads=1,
+                 enable_mkldnn=False,
+                 ir_optim=True):
+
+        self.num_seg = num_seg
+        self.seg_len = seg_len
+        self.short_size = short_size
+        self.target_size = target_size
+        self.top_k = top_k
+
+        assert batch_size == 1, "VideoActionRecognizer only support batch_size=1 now."
+
+        self.model_dir = model_dir
+        self.device = device
+        self.run_mode = run_mode
+        self.batch_size = batch_size
+        self.trt_min_shape = trt_min_shape
+        self.trt_max_shape = trt_max_shape
+        self.trt_opt_shape = trt_opt_shape
+        self.trt_calib_mode = trt_calib_mode
+        self.cpu_threads = cpu_threads
+        self.enable_mkldnn = enable_mkldnn
+        self.ir_optim = ir_optim
+
+        self.recognize_times = Timer()
+
+        model_file_path = glob.glob(os.path.join(model_dir, "*.pdmodel"))[0]
+        params_file_path = glob.glob(os.path.join(model_dir, "*.pdiparams"))[0]
+        self.config = Config(model_file_path, params_file_path)
+
+        if device == "GPU" or device == "gpu":
+            self.config.enable_use_gpu(8000, 0)
+        else:
+            self.config.disable_gpu()
+        if self.enable_mkldnn:
+            # cache 10 different shapes for mkldnn to avoid memory leak
+            self.config.set_mkldnn_cache_capacity(10)
+            self.config.enable_mkldnn()
+
+        self.config.switch_ir_optim(self.ir_optim)  # default true
+
+        precision_map = {
+            'trt_int8': Config.Precision.Int8,
+            'trt_fp32': Config.Precision.Float32,
+            'trt_fp16': Config.Precision.Half
+        }
+        if run_mode in precision_map.keys():
+            self.config.enable_tensorrt_engine(
+                max_batch_size=8, precision_mode=precision_map[run_mode])
+
+        self.config.enable_memory_optim()
+        # use zero copy
+        self.config.switch_use_feed_fetch_ops(False)
+
+        self.predictor = create_predictor(self.config)
+
+    @classmethod
+    def init_with_cfg(cls, args, cfg):
+        return cls(model_dir=cfg['model_dir'],
+                   short_size=cfg['short_size'],
+                   target_size=cfg['target_size'],
+                   batch_size=cfg['batch_size'],
+                   device=args.device,
+                   run_mode=args.run_mode,
+                   trt_min_shape=args.trt_min_shape,
+                   trt_max_shape=args.trt_max_shape,
+                   trt_opt_shape=args.trt_opt_shape,
+                   trt_calib_mode=args.trt_calib_mode,
+                   cpu_threads=args.cpu_threads,
+                   enable_mkldnn=args.enable_mkldnn)
+
+    def preprocess_batch(self, file_list):
+        batched_inputs = []
+        for file in file_list:
+            inputs = self.preprocess(file)
+            batched_inputs.append(inputs)
+        batched_inputs = [
+            np.concatenate([item[i] for item in batched_inputs])
+            for i in range(len(batched_inputs[0]))
+        ]
+        self.input_file = file_list
+        return batched_inputs
+
+    def get_timer(self):
+        return self.recognize_times
+
+    def predict(self, input):
+        '''
+        Args:
+            input (str) or (list): video file path or image data list
+        Returns:
+            results (dict): 
+        '''
+
+        input_names = self.predictor.get_input_names()
+        input_tensor = self.predictor.get_input_handle(input_names[0])
+
+        output_names = self.predictor.get_output_names()
+        output_tensor = self.predictor.get_output_handle(output_names[0])
+
+        # preprocess
+        self.recognize_times.preprocess_time_s.start()
+        if type(input) == str:
+            inputs = self.preprocess_video(input)
+        else:
+            inputs = self.preprocess_frames(input)
+        self.recognize_times.preprocess_time_s.end()
+
+        inputs = np.expand_dims(
+            inputs, axis=0).repeat(
+                self.batch_size, axis=0).copy()
+
+        input_tensor.copy_from_cpu(inputs)
+
+        # model prediction
+        self.recognize_times.inference_time_s.start()
+        self.predictor.run()
+        self.recognize_times.inference_time_s.end()
+
+        output = output_tensor.copy_to_cpu()
+
+        # postprocess
+        self.recognize_times.postprocess_time_s.start()
+        classes, scores = self.postprocess(output)
+        self.recognize_times.postprocess_time_s.end()
+
+        return classes, scores
+
+    def preprocess_frames(self, frame_list):
+        """
+        frame_list: list, frame list
+        return: list
+        """
+
+        results = {}
+        results['frames_len'] = len(frame_list)
+        results["imgs"] = frame_list
+
+        img_mean = [0.485, 0.456, 0.406]
+        img_std = [0.229, 0.224, 0.225]
+        ops = [
+            CenterCrop(self.target_size), Image2Array(),
+            Normalization(img_mean, img_std)
+        ]
+        for op in ops:
+            results = op(results)
+
+        res = np.expand_dims(results['imgs'], axis=0).copy()
+        return [res]
+
+    def preprocess_video(self, input_file):
+        """
+        input_file: str, file path
+        return: list
+        """
+        assert os.path.isfile(input_file) is not None, "{0} not exists".format(
+            input_file)
+
+        results = {'filename': input_file}
+        img_mean = [0.485, 0.456, 0.406]
+        img_std = [0.229, 0.224, 0.225]
+        ops = [
+            VideoDecoder(), Sampler(
+                self.num_seg, self.seg_len, valid_mode=True),
+            Scale(self.short_size), CenterCrop(self.target_size),
+            Image2Array(), Normalization(img_mean, img_std)
+        ]
+        for op in ops:
+            results = op(results)
+
+        res = np.expand_dims(results['imgs'], axis=0).copy()
+        return [res]
+
+    def postprocess(self, output):
+        output = output.flatten()  # numpy.ndarray
+        output = softmax(output)
+        classes = np.argpartition(output, -self.top_k)[-self.top_k:]
+        classes = classes[np.argsort(-output[classes])]
+        scores = output[classes]
+        return classes, scores
+
+
+def main():
+    if not FLAGS.run_benchmark:
+        assert FLAGS.batch_size == 1
+        assert FLAGS.use_fp16 is False
+    else:
+        assert FLAGS.use_gpu is True
+
+    recognizer = VideoActionRecognizer(
+        FLAGS.model_dir,
+        short_size=FLAGS.short_size,
+        target_size=FLAGS.target_size,
+        device=FLAGS.device,
+        run_mode=FLAGS.run_mode,
+        batch_size=FLAGS.batch_size,
+        trt_min_shape=FLAGS.trt_min_shape,
+        trt_max_shape=FLAGS.trt_max_shape,
+        trt_opt_shape=FLAGS.trt_opt_shape,
+        trt_calib_mode=FLAGS.trt_calib_mode,
+        cpu_threads=FLAGS.cpu_threads,
+        enable_mkldnn=FLAGS.enable_mkldnn, )
+
+    if not FLAGS.run_benchmark:
+        classes, scores = recognizer.predict(FLAGS.video_file)
+        print("Current video file: {}".format(FLAGS.video_file))
+        print("\ttop-1 class: {0}".format(classes[0]))
+        print("\ttop-1 score: {0}".format(scores[0]))
+    else:
+        cm, gm, gu = get_current_memory_mb()
+        mems = {'cpu_rss_mb': cm, 'gpu_rss_mb': gm, 'gpu_util': gu * 100}
+
+        perf_info = recognizer.recognize_times.report()
+        model_dir = FLAGS.model_dir
+        mode = FLAGS.run_mode
+        model_info = {
+            'model_name': model_dir.strip('/').split('/')[-1],
+            'precision': mode.split('_')[-1]
+        }
+        data_info = {
+            'batch_size': FLAGS.batch_size,
+            'shape': "dynamic_shape",
+            'data_num': perf_info['img_num']
+        }
+        recognize_log = PaddleInferBenchmark(recognizer.config, model_info,
+                                             data_info, perf_info, mems)
+        recognize_log('Fight')
+
+
+if __name__ == '__main__':
+    paddle.enable_static()
+    parser = argsparser()
+    FLAGS = parser.parse_args()
+    print_arguments(FLAGS)
+    FLAGS.device = FLAGS.device.upper()
+    assert FLAGS.device in ['CPU', 'GPU', 'XPU'
+                            ], "device should be CPU, GPU or XPU"
+
+    main()

pipeline/pphuman/video_action_preprocess.py

@@ -0,0 +1,545 @@
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import cv2
+import numpy as np
+from collections.abc import Sequence
+from PIL import Image
+import paddle
+
+
+class Sampler(object):
+    """
+    Sample frames id.
+    NOTE: Use PIL to read image here, has diff with CV2
+    Args:
+        num_seg(int): number of segments.
+        seg_len(int): number of sampled frames in each segment.
+        valid_mode(bool): True or False.
+    Returns:
+        frames_idx: the index of sampled #frames.
+    """
+
+    def __init__(self,
+                 num_seg,
+                 seg_len,
+                 frame_interval=None,
+                 valid_mode=True,
+                 dense_sample=False,
+                 linspace_sample=False,
+                 use_pil=True):
+        self.num_seg = num_seg
+        self.seg_len = seg_len
+        self.frame_interval = frame_interval
+        self.valid_mode = valid_mode
+        self.dense_sample = dense_sample
+        self.linspace_sample = linspace_sample
+        self.use_pil = use_pil
+
+    def _get(self, frames_idx, results):
+        data_format = results['format']
+
+        if data_format == "frame":
+            frame_dir = results['frame_dir']
+            imgs = []
+            for idx in frames_idx:
+                img = Image.open(
+                    os.path.join(frame_dir, results['suffix'].format(
+                        idx))).convert('RGB')
+                imgs.append(img)
+
+        elif data_format == "video":
+            if results['backend'] == 'cv2':
+                frames = np.array(results['frames'])
+                imgs = []
+                for idx in frames_idx:
+                    imgbuf = frames[idx]
+                    img = Image.fromarray(imgbuf, mode='RGB')
+                    imgs.append(img)
+            elif results['backend'] == 'decord':
+                container = results['frames']
+                if self.use_pil:
+                    frames_select = container.get_batch(frames_idx)
+                    # dearray_to_img
+                    np_frames = frames_select.asnumpy()
+                    imgs = []
+                    for i in range(np_frames.shape[0]):
+                        imgbuf = np_frames[i]
+                        imgs.append(Image.fromarray(imgbuf, mode='RGB'))
+                else:
+                    if frames_idx.ndim != 1:
+                        frames_idx = np.squeeze(frames_idx)
+                    frame_dict = {
+                        idx: container[idx].asnumpy()
+                        for idx in np.unique(frames_idx)
+                    }
+                    imgs = [frame_dict[idx] for idx in frames_idx]
+            elif results['backend'] == 'pyav':
+                imgs = []
+                frames = np.array(results['frames'])
+                for idx in frames_idx:
+                    imgbuf = frames[idx]
+                    imgs.append(imgbuf)
+                imgs = np.stack(imgs)  # thwc
+            else:
+                raise NotImplementedError
+        else:
+            raise NotImplementedError
+        results['imgs'] = imgs  # all image data
+        return results
+
+    def _get_train_clips(self, num_frames):
+        ori_seg_len = self.seg_len * self.frame_interval
+        avg_interval = (num_frames - ori_seg_len + 1) // self.num_seg
+
+        if avg_interval > 0:
+            base_offsets = np.arange(self.num_seg) * avg_interval
+            clip_offsets = base_offsets + np.random.randint(
+                avg_interval, size=self.num_seg)
+        elif num_frames > max(self.num_seg, ori_seg_len):
+            clip_offsets = np.sort(
+                np.random.randint(
+                    num_frames - ori_seg_len + 1, size=self.num_seg))
+        elif avg_interval == 0:
+            ratio = (num_frames - ori_seg_len + 1.0) / self.num_seg
+            clip_offsets = np.around(np.arange(self.num_seg) * ratio)
+        else:
+            clip_offsets = np.zeros((self.num_seg, ), dtype=np.int)
+        return clip_offsets
+
+    def _get_test_clips(self, num_frames):
+        ori_seg_len = self.seg_len * self.frame_interval
+        avg_interval = (num_frames - ori_seg_len + 1) / float(self.num_seg)
+        if num_frames > ori_seg_len - 1:
+            base_offsets = np.arange(self.num_seg) * avg_interval
+            clip_offsets = (base_offsets + avg_interval / 2.0).astype(np.int)
+        else:
+            clip_offsets = np.zeros((self.num_seg, ), dtype=np.int)
+        return clip_offsets
+
+    def __call__(self, results):
+        """
+        Args:
+            frames_len: length of frames.
+        return:
+            sampling id.
+        """
+        frames_len = int(results['frames_len'])  # total number of frames
+
+        frames_idx = []
+        if self.frame_interval is not None:
+            assert isinstance(self.frame_interval, int)
+            if not self.valid_mode:
+                offsets = self._get_train_clips(frames_len)
+            else:
+                offsets = self._get_test_clips(frames_len)
+
+            offsets = offsets[:, None] + np.arange(self.seg_len)[
+                None, :] * self.frame_interval
+            offsets = np.concatenate(offsets)
+
+            offsets = offsets.reshape((-1, self.seg_len))
+            offsets = np.mod(offsets, frames_len)
+            offsets = np.concatenate(offsets)
+
+            if results['format'] == 'video':
+                frames_idx = offsets
+            elif results['format'] == 'frame':
+                frames_idx = list(offsets + 1)
+            else:
+                raise NotImplementedError
+
+            return self._get(frames_idx, results)
+
+        print("self.frame_interval:", self.frame_interval)
+
+        if self.linspace_sample:  # default if False
+            if 'start_idx' in results and 'end_idx' in results:
+                offsets = np.linspace(results['start_idx'], results['end_idx'],
+                                      self.num_seg)
+            else:
+                offsets = np.linspace(0, frames_len - 1, self.num_seg)
+            offsets = np.clip(offsets, 0, frames_len - 1).astype(np.int64)
+            if results['format'] == 'video':
+                frames_idx = list(offsets)
+                frames_idx = [x % frames_len for x in frames_idx]
+            elif results['format'] == 'frame':
+                frames_idx = list(offsets + 1)
+            else:
+                raise NotImplementedError
+            return self._get(frames_idx, results)
+
+        average_dur = int(frames_len / self.num_seg)
+
+        print("results['format']:", results['format'])
+
+        if self.dense_sample:  # For ppTSM, default is False
+            if not self.valid_mode:  # train
+                sample_pos = max(1, 1 + frames_len - 64)
+                t_stride = 64 // self.num_seg
+                start_idx = 0 if sample_pos == 1 else np.random.randint(
+                    0, sample_pos - 1)
+                offsets = [(idx * t_stride + start_idx) % frames_len + 1
+                           for idx in range(self.num_seg)]
+                frames_idx = offsets
+            else:
+                sample_pos = max(1, 1 + frames_len - 64)
+                t_stride = 64 // self.num_seg
+                start_list = np.linspace(0, sample_pos - 1, num=10, dtype=int)
+                offsets = []
+                for start_idx in start_list.tolist():
+                    offsets += [(idx * t_stride + start_idx) % frames_len + 1
+                                for idx in range(self.num_seg)]
+                frames_idx = offsets
+        else:
+            for i in range(self.num_seg):
+                idx = 0
+                if not self.valid_mode:
+                    if average_dur >= self.seg_len:
+                        idx = random.randint(0, average_dur - self.seg_len)
+                        idx += i * average_dur
+                    elif average_dur >= 1:
+                        idx += i * average_dur
+                    else:
+                        idx = i
+                else:
+                    if average_dur >= self.seg_len:
+                        idx = (average_dur - 1) // 2
+                        idx += i * average_dur
+                    elif average_dur >= 1:
+                        idx += i * average_dur
+                    else:
+                        idx = i
+
+                for jj in range(idx, idx + self.seg_len):
+                    if results['format'] == 'video':
+                        frames_idx.append(int(jj % frames_len))
+                    elif results['format'] == 'frame':
+                        frames_idx.append(jj + 1)
+
+                    elif results['format'] == 'MRI':
+                        frames_idx.append(jj)
+                    else:
+                        raise NotImplementedError
+
+        return self._get(frames_idx, results)
+
+
+class Scale(object):
+    """
+    Scale images.
+    Args:
+        short_size(float | int): Short size of an image will be scaled to the short_size.
+        fixed_ratio(bool): Set whether to zoom according to a fixed ratio. default: True
+        do_round(bool): Whether to round up when calculating the zoom ratio. default: False
+        backend(str): Choose pillow or cv2 as the graphics processing backend. default: 'pillow'
+    """
+
+    def __init__(self,
+                 short_size,
+                 fixed_ratio=True,
+                 keep_ratio=None,
+                 do_round=False,
+                 backend='pillow'):
+        self.short_size = short_size
+        assert (fixed_ratio and not keep_ratio) or (
+            not fixed_ratio
+        ), "fixed_ratio and keep_ratio cannot be true at the same time"
+        self.fixed_ratio = fixed_ratio
+        self.keep_ratio = keep_ratio
+        self.do_round = do_round
+
+        assert backend in [
+            'pillow', 'cv2'
+        ], "Scale's backend must be pillow or cv2, but get {backend}"
+
+        self.backend = backend
+
+    def __call__(self, results):
+        """
+        Performs resize operations.
+        Args:
+            imgs (Sequence[PIL.Image]): List where each item is a PIL.Image.
+            For example, [PIL.Image0, PIL.Image1, PIL.Image2, ...]
+        return:
+            resized_imgs: List where each item is a PIL.Image after scaling.
+        """
+        imgs = results['imgs']
+        resized_imgs = []
+        for i in range(len(imgs)):
+            img = imgs[i]
+            if isinstance(img, np.ndarray):
+                h, w, _ = img.shape
+            elif isinstance(img, Image.Image):
+                w, h = img.size
+            else:
+                raise NotImplementedError
+
+            if w <= h:
+                ow = self.short_size
+                if self.fixed_ratio:  # default is True
+                    oh = int(self.short_size * 4.0 / 3.0)
+                elif not self.keep_ratio:  # no
+                    oh = self.short_size
+                else:
+                    scale_factor = self.short_size / w
+                    oh = int(h * float(scale_factor) +
+                             0.5) if self.do_round else int(
+                                 h * self.short_size / w)
+                    ow = int(w * float(scale_factor) +
+                             0.5) if self.do_round else int(
+                                 w * self.short_size / h)
+            else:
+                oh = self.short_size
+                if self.fixed_ratio:
+                    ow = int(self.short_size * 4.0 / 3.0)
+                elif not self.keep_ratio:  # no
+                    ow = self.short_size
+                else:
+                    scale_factor = self.short_size / h
+                    oh = int(h * float(scale_factor) +
+                             0.5) if self.do_round else int(
+                                 h * self.short_size / w)
+                    ow = int(w * float(scale_factor) +
+                             0.5) if self.do_round else int(
+                                 w * self.short_size / h)
+
+            if type(img) == np.ndarray:
+                img = Image.fromarray(img, mode='RGB')
+
+            if self.backend == 'pillow':
+                resized_imgs.append(img.resize((ow, oh), Image.BILINEAR))
+            elif self.backend == 'cv2' and (self.keep_ratio is not None):
+                resized_imgs.append(
+                    cv2.resize(
+                        img, (ow, oh), interpolation=cv2.INTER_LINEAR))
+            else:
+                resized_imgs.append(
+                    Image.fromarray(
+                        cv2.resize(
+                            np.asarray(img), (ow, oh),
+                            interpolation=cv2.INTER_LINEAR)))
+        results['imgs'] = resized_imgs
+        return results
+
+
+class CenterCrop(object):
+    """
+    Center crop images
+    Args:
+        target_size(int): Center crop a square with the target_size from an image.
+        do_round(bool): Whether to round up the coordinates of the upper left corner of the cropping area. default: True
+    """
+
+    def __init__(self, target_size, do_round=True, backend='pillow'):
+        self.target_size = target_size
+        self.do_round = do_round
+        self.backend = backend
+
+    def __call__(self, results):
+        """
+        Performs Center crop operations.
+        Args:
+            imgs: List where each item is a PIL.Image.
+            For example, [PIL.Image0, PIL.Image1, PIL.Image2, ...]
+        return:
+            ccrop_imgs: List where each item is a PIL.Image after Center crop.
+        """
+        imgs = results['imgs']
+        ccrop_imgs = []
+        th, tw = self.target_size, self.target_size
+        if isinstance(imgs, paddle.Tensor):
+            h, w = imgs.shape[-2:]
+            x1 = int(round((w - tw) / 2.0)) if self.do_round else (w - tw) // 2
+            y1 = int(round((h - th) / 2.0)) if self.do_round else (h - th) // 2
+            ccrop_imgs = imgs[:, :, y1:y1 + th, x1:x1 + tw]
+        else:
+            for img in imgs:
+                if self.backend == 'pillow':
+                    w, h = img.size
+                elif self.backend == 'cv2':
+                    h, w, _ = img.shape
+                else:
+                    raise NotImplementedError
+                assert (w >= self.target_size) and (h >= self.target_size), \
+                    "image width({}) and height({}) should be larger than crop size".format(
+                        w, h, self.target_size)
+                x1 = int(round((w - tw) / 2.0)) if self.do_round else (
+                    w - tw) // 2
+                y1 = int(round((h - th) / 2.0)) if self.do_round else (
+                    h - th) // 2
+                if self.backend == 'cv2':
+                    ccrop_imgs.append(img[y1:y1 + th, x1:x1 + tw])
+                elif self.backend == 'pillow':
+                    ccrop_imgs.append(img.crop((x1, y1, x1 + tw, y1 + th)))
+        results['imgs'] = ccrop_imgs
+        return results
+
+
+class Image2Array(object):
+    """
+    transfer PIL.Image to Numpy array and transpose dimensions from 'dhwc' to 'dchw'.
+    Args:
+        transpose: whether to transpose or not, default True, False for slowfast.
+    """
+
+    def __init__(self, transpose=True, data_format='tchw'):
+        assert data_format in [
+            'tchw', 'cthw'
+        ], "Target format must in ['tchw', 'cthw'], but got {data_format}"
+        self.transpose = transpose
+        self.data_format = data_format
+
+    def __call__(self, results):
+        """
+        Performs Image to NumpyArray operations.
+        Args:
+            imgs: List where each item is a PIL.Image.
+            For example, [PIL.Image0, PIL.Image1, PIL.Image2, ...]
+        return:
+            np_imgs: Numpy array.
+        """
+        imgs = results['imgs']
+        if 'backend' in results and results[
+                'backend'] == 'pyav':  # [T,H,W,C] in [0, 1]
+            if self.transpose:
+                if self.data_format == 'tchw':
+                    t_imgs = imgs.transpose((0, 3, 1, 2))  # tchw
+                else:
+                    t_imgs = imgs.transpose((3, 0, 1, 2))  # cthw
+            results['imgs'] = t_imgs
+        else:
+            t_imgs = np.stack(imgs).astype('float32')
+            if self.transpose:
+                if self.data_format == 'tchw':
+                    t_imgs = t_imgs.transpose(0, 3, 1, 2)  # tchw
+                else:
+                    t_imgs = t_imgs.transpose(3, 0, 1, 2)  # cthw
+            results['imgs'] = t_imgs
+        return results
+
+
+class VideoDecoder(object):
+    """
+    Decode mp4 file to frames.
+    Args:
+        filepath: the file path of mp4 file
+    """
+
+    def __init__(self,
+                 backend='cv2',
+                 mode='train',
+                 sampling_rate=32,
+                 num_seg=8,
+                 num_clips=1,
+                 target_fps=30):
+
+        self.backend = backend
+        # params below only for TimeSformer
+        self.mode = mode
+        self.sampling_rate = sampling_rate
+        self.num_seg = num_seg
+        self.num_clips = num_clips
+        self.target_fps = target_fps
+
+    def __call__(self, results):
+        """
+        Perform mp4 decode operations.
+        return:
+            List where each item is a numpy array after decoder.
+        """
+        file_path = results['filename']
+        results['format'] = 'video'
+        results['backend'] = self.backend
+
+        if self.backend == 'cv2':  # here
+            cap = cv2.VideoCapture(file_path)
+            videolen = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
+
+            sampledFrames = []
+            for i in range(videolen):
+                ret, frame = cap.read()
+                # maybe first frame is empty
+                if ret == False:
+                    continue
+                img = frame[:, :, ::-1]
+                sampledFrames.append(img)
+            results['frames'] = sampledFrames
+            results['frames_len'] = len(sampledFrames)
+
+        elif self.backend == 'decord':
+            container = de.VideoReader(file_path)
+            frames_len = len(container)
+            results['frames'] = container
+            results['frames_len'] = frames_len
+        else:
+            raise NotImplementedError
+        return results
+
+
+class Normalization(object):
+    """
+    Normalization.
+    Args:
+        mean(Sequence[float]): mean values of different channels.
+        std(Sequence[float]): std values of different channels.
+        tensor_shape(list): size of mean, default [3,1,1]. For slowfast, [1,1,1,3]
+    """
+
+    def __init__(self, mean, std, tensor_shape=[3, 1, 1], inplace=False):
+        if not isinstance(mean, Sequence):
+            raise TypeError(
+                'Mean must be list, tuple or np.ndarray, but got {type(mean)}')
+        if not isinstance(std, Sequence):
+            raise TypeError(
+                'Std must be list, tuple or np.ndarray, but got {type(std)}')
+
+        self.inplace = inplace
+        if not inplace:
+            self.mean = np.array(mean).reshape(tensor_shape).astype(np.float32)
+            self.std = np.array(std).reshape(tensor_shape).astype(np.float32)
+        else:
+            self.mean = np.array(mean, dtype=np.float32)
+            self.std = np.array(std, dtype=np.float32)
+
+    def __call__(self, results):
+        """
+        Performs normalization operations.
+        Args:
+            imgs: Numpy array.
+        return:
+            np_imgs: Numpy array after normalization.
+        """
+
+        if self.inplace:  # default is False
+            n = len(results['imgs'])
+            h, w, c = results['imgs'][0].shape
+            norm_imgs = np.empty((n, h, w, c), dtype=np.float32)
+            for i, img in enumerate(results['imgs']):
+                norm_imgs[i] = img
+
+            for img in norm_imgs:  # [n,h,w,c]
+                mean = np.float64(self.mean.reshape(1, -1))  # [1, 3]
+                stdinv = 1 / np.float64(self.std.reshape(1, -1))  # [1, 3]
+                cv2.subtract(img, mean, img)
+                cv2.multiply(img, stdinv, img)
+        else:
+            imgs = results['imgs']
+            norm_imgs = imgs / 255.0
+            norm_imgs -= self.mean
+            norm_imgs /= self.std
+            if 'backend' in results and results['backend'] == 'pyav':
+                norm_imgs = paddle.to_tensor(norm_imgs, dtype=paddle.float32)
+        results['imgs'] = norm_imgs
+        return results

pptracking/python/det_infer.py

@@ -0,0 +1,594 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import yaml
+import glob
+from functools import reduce
+
+import cv2
+import numpy as np
+import math
+
+import paddle
+from paddle.inference import Config
+from paddle.inference import create_predictor
+
+import sys
+# add deploy path of PadleDetection to sys.path
+parent_path = os.path.abspath(os.path.join(__file__, *(['..'])))
+sys.path.insert(0, parent_path)
+
+from benchmark_utils import PaddleInferBenchmark
+from picodet_postprocess import PicoDetPostProcess
+from preprocess import preprocess, Resize, NormalizeImage, Permute, PadStride, LetterBoxResize, Pad, decode_image
+from mot.visualize import visualize_box_mask
+from mot_utils import argsparser, Timer, get_current_memory_mb
+
+# Global dictionary
+SUPPORT_MODELS = {
+    'YOLO',
+    'PicoDet',
+    'JDE',
+    'FairMOT',
+    'DeepSORT',
+    'StrongBaseline',
+}
+
+
+def bench_log(detector, img_list, model_info, batch_size=1, name=None):
+    mems = {
+        'cpu_rss_mb': detector.cpu_mem / len(img_list),
+        'gpu_rss_mb': detector.gpu_mem / len(img_list),
+        'gpu_util': detector.gpu_util * 100 / len(img_list)
+    }
+    perf_info = detector.det_times.report(average=True)
+    data_info = {
+        'batch_size': batch_size,
+        'shape': "dynamic_shape",
+        'data_num': perf_info['img_num']
+    }
+    log = PaddleInferBenchmark(detector.config, model_info, data_info,
+                               perf_info, mems)
+    log(name)
+
+
+class Detector(object):
+    """
+    Args:
+        pred_config (object): config of model, defined by `Config(model_dir)`
+        model_dir (str): root path of model.pdiparams, model.pdmodel and infer_cfg.yml
+        device (str): Choose the device you want to run, it can be: CPU/GPU/XPU, default is CPU
+        run_mode (str): mode of running(paddle/trt_fp32/trt_fp16)
+        batch_size (int): size of pre batch in inference
+        trt_min_shape (int): min shape for dynamic shape in trt
+        trt_max_shape (int): max shape for dynamic shape in trt
+        trt_opt_shape (int): opt shape for dynamic shape in trt
+        trt_calib_mode (bool): If the model is produced by TRT offline quantitative
+            calibration, trt_calib_mode need to set True
+        cpu_threads (int): cpu threads
+        enable_mkldnn (bool): whether to open MKLDNN
+        output_dir (str): The path of output
+        threshold (float): The threshold of score for visualization
+    """
+
+    def __init__(
+            self,
+            model_dir,
+            device='CPU',
+            run_mode='paddle',
+            batch_size=1,
+            trt_min_shape=1,
+            trt_max_shape=1280,
+            trt_opt_shape=640,
+            trt_calib_mode=False,
+            cpu_threads=1,
+            enable_mkldnn=False,
+            output_dir='output',
+            threshold=0.5, ):
+        self.pred_config = self.set_config(model_dir)
+        self.predictor, self.config = load_predictor(
+            model_dir,
+            run_mode=run_mode,
+            batch_size=batch_size,
+            min_subgraph_size=self.pred_config.min_subgraph_size,
+            device=device,
+            use_dynamic_shape=self.pred_config.use_dynamic_shape,
+            trt_min_shape=trt_min_shape,
+            trt_max_shape=trt_max_shape,
+            trt_opt_shape=trt_opt_shape,
+            trt_calib_mode=trt_calib_mode,
+            cpu_threads=cpu_threads,
+            enable_mkldnn=enable_mkldnn)
+        self.det_times = Timer()
+        self.cpu_mem, self.gpu_mem, self.gpu_util = 0, 0, 0
+        self.batch_size = batch_size
+        self.output_dir = output_dir
+        self.threshold = threshold
+
+    def set_config(self, model_dir):
+        return PredictConfig(model_dir)
+
+    def preprocess(self, image_list):
+        preprocess_ops = []
+        for op_info in self.pred_config.preprocess_infos:
+            new_op_info = op_info.copy()
+            op_type = new_op_info.pop('type')
+            preprocess_ops.append(eval(op_type)(**new_op_info))
+
+        input_im_lst = []
+        input_im_info_lst = []
+        for im_path in image_list:
+            im, im_info = preprocess(im_path, preprocess_ops)
+            input_im_lst.append(im)
+            input_im_info_lst.append(im_info)
+        inputs = create_inputs(input_im_lst, input_im_info_lst)
+        input_names = self.predictor.get_input_names()
+        for i in range(len(input_names)):
+            input_tensor = self.predictor.get_input_handle(input_names[i])
+            input_tensor.copy_from_cpu(inputs[input_names[i]])
+
+        return inputs
+
+    def postprocess(self, inputs, result):
+        # postprocess output of predictor
+        np_boxes_num = result['boxes_num']
+        if np_boxes_num[0] <= 0:
+            print('[WARNNING] No object detected.')
+            result = {'boxes': np.zeros([0, 6]), 'boxes_num': [0]}
+        result = {k: v for k, v in result.items() if v is not None}
+        return result
+
+    def predict(self, repeats=1):
+        '''
+        Args:
+            repeats (int): repeats number for prediction
+        Returns:
+            result (dict): include 'boxes': np.ndarray: shape:[N,6], N: number of box,
+                            matix element:[class, score, x_min, y_min, x_max, y_max]
+        '''
+        # model prediction
+        np_boxes, np_boxes_num = None, None
+        for i in range(repeats):
+            self.predictor.run()
+            output_names = self.predictor.get_output_names()
+            boxes_tensor = self.predictor.get_output_handle(output_names[0])
+            np_boxes = boxes_tensor.copy_to_cpu()
+            boxes_num = self.predictor.get_output_handle(output_names[1])
+            np_boxes_num = boxes_num.copy_to_cpu()
+        result = dict(boxes=np_boxes, boxes_num=np_boxes_num)
+        return result
+
+    def merge_batch_result(self, batch_result):
+        if len(batch_result) == 1:
+            return batch_result[0]
+        res_key = batch_result[0].keys()
+        results = {k: [] for k in res_key}
+        for res in batch_result:
+            for k, v in res.items():
+                results[k].append(v)
+        for k, v in results.items():
+            results[k] = np.concatenate(v)
+        return results
+
+    def get_timer(self):
+        return self.det_times
+
+    def predict_image(self,
+                      image_list,
+                      run_benchmark=False,
+                      repeats=1,
+                      visual=True):
+        batch_loop_cnt = math.ceil(float(len(image_list)) / self.batch_size)
+        results = []
+        for i in range(batch_loop_cnt):
+            start_index = i * self.batch_size
+            end_index = min((i + 1) * self.batch_size, len(image_list))
+            batch_image_list = image_list[start_index:end_index]
+            if run_benchmark:
+                # preprocess
+                inputs = self.preprocess(batch_image_list)  # warmup
+                self.det_times.preprocess_time_s.start()
+                inputs = self.preprocess(batch_image_list)
+                self.det_times.preprocess_time_s.end()
+
+                # model prediction
+                result = self.predict(repeats=repeats)  # warmup
+                self.det_times.inference_time_s.start()
+                result = self.predict(repeats=repeats)
+                self.det_times.inference_time_s.end(repeats=repeats)
+
+                # postprocess
+                result_warmup = self.postprocess(inputs, result)  # warmup
+                self.det_times.postprocess_time_s.start()
+                result = self.postprocess(inputs, result)
+                self.det_times.postprocess_time_s.end()
+                self.det_times.img_num += len(batch_image_list)
+
+                cm, gm, gu = get_current_memory_mb()
+                self.cpu_mem += cm
+                self.gpu_mem += gm
+                self.gpu_util += gu
+            else:
+                # preprocess
+                self.det_times.preprocess_time_s.start()
+                inputs = self.preprocess(batch_image_list)
+                self.det_times.preprocess_time_s.end()
+
+                # model prediction
+                self.det_times.inference_time_s.start()
+                result = self.predict()
+                self.det_times.inference_time_s.end()
+
+                # postprocess
+                self.det_times.postprocess_time_s.start()
+                result = self.postprocess(inputs, result)
+                self.det_times.postprocess_time_s.end()
+                self.det_times.img_num += len(batch_image_list)
+
+                if visual:
+                    visualize(
+                        batch_image_list,
+                        result,
+                        self.pred_config.labels,
+                        output_dir=self.output_dir,
+                        threshold=self.threshold)
+
+            results.append(result)
+            if visual:
+                print('Test iter {}'.format(i))
+
+        results = self.merge_batch_result(results)
+        return results
+
+    def predict_video(self, video_file, camera_id):
+        video_out_name = 'output.mp4'
+        if camera_id != -1:
+            capture = cv2.VideoCapture(camera_id)
+        else:
+            capture = cv2.VideoCapture(video_file)
+            video_out_name = os.path.split(video_file)[-1]
+        # Get Video info : resolution, fps, frame count
+        width = int(capture.get(cv2.CAP_PROP_FRAME_WIDTH))
+        height = int(capture.get(cv2.CAP_PROP_FRAME_HEIGHT))
+        fps = int(capture.get(cv2.CAP_PROP_FPS))
+        frame_count = int(capture.get(cv2.CAP_PROP_FRAME_COUNT))
+        print("fps: %d, frame_count: %d" % (fps, frame_count))
+
+        if not os.path.exists(self.output_dir):
+            os.makedirs(self.output_dir)
+        out_path = os.path.join(self.output_dir, video_out_name)
+        fourcc = cv2.VideoWriter_fourcc(*'mp4v')
+        writer = cv2.VideoWriter(out_path, fourcc, fps, (width, height))
+        index = 1
+        while (1):
+            ret, frame = capture.read()
+            if not ret:
+                break
+            print('detect frame: %d' % (index))
+            index += 1
+            results = self.predict_image([frame], visual=False)
+
+            im = visualize_box_mask(
+                frame,
+                results,
+                self.pred_config.labels,
+                threshold=self.threshold)
+            im = np.array(im)
+            writer.write(im)
+            if camera_id != -1:
+                cv2.imshow('Mask Detection', im)
+                if cv2.waitKey(1) & 0xFF == ord('q'):
+                    break
+        writer.release()
+
+
+def create_inputs(imgs, im_info):
+    """generate input for different model type
+    Args:
+        imgs (list(numpy)): list of images (np.ndarray)
+        im_info (list(dict)): list of image info
+    Returns:
+        inputs (dict): input of model
+    """
+    inputs = {}
+
+    im_shape = []
+    scale_factor = []
+    if len(imgs) == 1:
+        inputs['image'] = np.array((imgs[0], )).astype('float32')
+        inputs['im_shape'] = np.array(
+            (im_info[0]['im_shape'], )).astype('float32')
+        inputs['scale_factor'] = np.array(
+            (im_info[0]['scale_factor'], )).astype('float32')
+        return inputs
+
+    for e in im_info:
+        im_shape.append(np.array((e['im_shape'], )).astype('float32'))
+        scale_factor.append(np.array((e['scale_factor'], )).astype('float32'))
+
+    inputs['im_shape'] = np.concatenate(im_shape, axis=0)
+    inputs['scale_factor'] = np.concatenate(scale_factor, axis=0)
+
+    imgs_shape = [[e.shape[1], e.shape[2]] for e in imgs]
+    max_shape_h = max([e[0] for e in imgs_shape])
+    max_shape_w = max([e[1] for e in imgs_shape])
+    padding_imgs = []
+    for img in imgs:
+        im_c, im_h, im_w = img.shape[:]
+        padding_im = np.zeros(
+            (im_c, max_shape_h, max_shape_w), dtype=np.float32)
+        padding_im[:, :im_h, :im_w] = img
+        padding_imgs.append(padding_im)
+    inputs['image'] = np.stack(padding_imgs, axis=0)
+    return inputs
+
+
+class PredictConfig():
+    """set config of preprocess, postprocess and visualize
+    Args:
+        model_dir (str): root path of model.yml
+    """
+
+    def __init__(self, model_dir):
+        # parsing Yaml config for Preprocess
+        deploy_file = os.path.join(model_dir, 'infer_cfg.yml')
+        with open(deploy_file) as f:
+            yml_conf = yaml.safe_load(f)
+        self.check_model(yml_conf)
+        self.arch = yml_conf['arch']
+        self.preprocess_infos = yml_conf['Preprocess']
+        self.min_subgraph_size = yml_conf['min_subgraph_size']
+        self.labels = yml_conf['label_list']
+        self.mask = False
+        self.use_dynamic_shape = yml_conf['use_dynamic_shape']
+        if 'mask' in yml_conf:
+            self.mask = yml_conf['mask']
+        self.tracker = None
+        if 'tracker' in yml_conf:
+            self.tracker = yml_conf['tracker']
+        if 'NMS' in yml_conf:
+            self.nms = yml_conf['NMS']
+        if 'fpn_stride' in yml_conf:
+            self.fpn_stride = yml_conf['fpn_stride']
+        self.print_config()
+
+    def check_model(self, yml_conf):
+        """
+        Raises:
+            ValueError: loaded model not in supported model type 
+        """
+        for support_model in SUPPORT_MODELS:
+            if support_model in yml_conf['arch']:
+                return True
+        raise ValueError("Unsupported arch: {}, expect {}".format(yml_conf[
+            'arch'], SUPPORT_MODELS))
+
+    def print_config(self):
+        print('-----------  Model Configuration -----------')
+        print('%s: %s' % ('Model Arch', self.arch))
+        print('%s: ' % ('Transform Order'))
+        for op_info in self.preprocess_infos:
+            print('--%s: %s' % ('transform op', op_info['type']))
+        print('--------------------------------------------')
+
+
+def load_predictor(model_dir,
+                   run_mode='paddle',
+                   batch_size=1,
+                   device='CPU',
+                   min_subgraph_size=3,
+                   use_dynamic_shape=False,
+                   trt_min_shape=1,
+                   trt_max_shape=1280,
+                   trt_opt_shape=640,
+                   trt_calib_mode=False,
+                   cpu_threads=1,
+                   enable_mkldnn=False):
+    """set AnalysisConfig, generate AnalysisPredictor
+    Args:
+        model_dir (str): root path of __model__ and __params__
+        device (str): Choose the device you want to run, it can be: CPU/GPU/XPU, default is CPU
+        run_mode (str): mode of running(paddle/trt_fp32/trt_fp16/trt_int8)
+        use_dynamic_shape (bool): use dynamic shape or not
+        trt_min_shape (int): min shape for dynamic shape in trt
+        trt_max_shape (int): max shape for dynamic shape in trt
+        trt_opt_shape (int): opt shape for dynamic shape in trt
+        trt_calib_mode (bool): If the model is produced by TRT offline quantitative
+            calibration, trt_calib_mode need to set True
+    Returns:
+        predictor (PaddlePredictor): AnalysisPredictor
+    Raises:
+        ValueError: predict by TensorRT need device == 'GPU'.
+    """
+    if device != 'GPU' and run_mode != 'paddle':
+        raise ValueError(
+            "Predict by TensorRT mode: {}, expect device=='GPU', but device == {}"
+            .format(run_mode, device))
+    infer_model = os.path.join(model_dir, 'model.pdmodel')
+    infer_params = os.path.join(model_dir, 'model.pdiparams')
+    if not os.path.exists(infer_model):
+        infer_model = os.path.join(model_dir, 'inference.pdmodel')
+        infer_params = os.path.join(model_dir, 'inference.pdiparams')
+        if not os.path.exists(infer_model):
+            raise ValueError("Cannot find any inference model in dir: {},".
+                             format(model_dir))
+    config = Config(infer_model, infer_params)
+    if device == 'GPU':
+        # initial GPU memory(M), device ID
+        config.enable_use_gpu(200, 0)
+        # optimize graph and fuse op
+        config.switch_ir_optim(True)
+    elif device == 'XPU':
+        config.enable_lite_engine()
+        config.enable_xpu(10 * 1024 * 1024)
+    else:
+        config.disable_gpu()
+        config.set_cpu_math_library_num_threads(cpu_threads)
+        if enable_mkldnn:
+            try:
+                # cache 10 different shapes for mkldnn to avoid memory leak
+                config.set_mkldnn_cache_capacity(10)
+                config.enable_mkldnn()
+            except Exception as e:
+                print(
+                    "The current environment does not support `mkldnn`, so disable mkldnn."
+                )
+                pass
+
+    precision_map = {
+        'trt_int8': Config.Precision.Int8,
+        'trt_fp32': Config.Precision.Float32,
+        'trt_fp16': Config.Precision.Half
+    }
+    if run_mode in precision_map.keys():
+        config.enable_tensorrt_engine(
+            workspace_size=1 << 25,
+            max_batch_size=batch_size,
+            min_subgraph_size=min_subgraph_size,
+            precision_mode=precision_map[run_mode],
+            use_static=False,
+            use_calib_mode=trt_calib_mode)
+
+        if use_dynamic_shape:
+            min_input_shape = {
+                'image': [batch_size, 3, trt_min_shape, trt_min_shape]
+            }
+            max_input_shape = {
+                'image': [batch_size, 3, trt_max_shape, trt_max_shape]
+            }
+            opt_input_shape = {
+                'image': [batch_size, 3, trt_opt_shape, trt_opt_shape]
+            }
+            config.set_trt_dynamic_shape_info(min_input_shape, max_input_shape,
+                                              opt_input_shape)
+            print('trt set dynamic shape done!')
+
+    # disable print log when predict
+    config.disable_glog_info()
+    # enable shared memory
+    config.enable_memory_optim()
+    # disable feed, fetch OP, needed by zero_copy_run
+    config.switch_use_feed_fetch_ops(False)
+    predictor = create_predictor(config)
+    return predictor, config
+
+
+def get_test_images(infer_dir, infer_img):
+    """
+    Get image path list in TEST mode
+    """
+    assert infer_img is not None or infer_dir is not None, \
+        "--infer_img or --infer_dir should be set"
+    assert infer_img is None or os.path.isfile(infer_img), \
+            "{} is not a file".format(infer_img)
+    assert infer_dir is None or os.path.isdir(infer_dir), \
+            "{} is not a directory".format(infer_dir)
+
+    # infer_img has a higher priority
+    if infer_img and os.path.isfile(infer_img):
+        return [infer_img]
+
+    images = set()
+    infer_dir = os.path.abspath(infer_dir)
+    assert os.path.isdir(infer_dir), \
+        "infer_dir {} is not a directory".format(infer_dir)
+    exts = ['jpg', 'jpeg', 'png', 'bmp']
+    exts += [ext.upper() for ext in exts]
+    for ext in exts:
+        images.update(glob.glob('{}/*.{}'.format(infer_dir, ext)))
+    images = list(images)
+
+    assert len(images) > 0, "no image found in {}".format(infer_dir)
+    print("Found {} inference images in total.".format(len(images)))
+
+    return images
+
+
+def visualize(image_list, result, labels, output_dir='output/', threshold=0.5):
+    # visualize the predict result
+    start_idx = 0
+    for idx, image_file in enumerate(image_list):
+        im_bboxes_num = result['boxes_num'][idx]
+        im_results = {}
+        if 'boxes' in result:
+            im_results['boxes'] = result['boxes'][start_idx:start_idx +
+                                                  im_bboxes_num, :]
+        start_idx += im_bboxes_num
+        im = visualize_box_mask(
+            image_file, im_results, labels, threshold=threshold)
+        img_name = os.path.split(image_file)[-1]
+        if not os.path.exists(output_dir):
+            os.makedirs(output_dir)
+        out_path = os.path.join(output_dir, img_name)
+        im.save(out_path, quality=95)
+        print("save result to: " + out_path)
+
+
+def print_arguments(args):
+    print('-----------  Running Arguments -----------')
+    for arg, value in sorted(vars(args).items()):
+        print('%s: %s' % (arg, value))
+    print('------------------------------------------')
+
+
+def main():
+    deploy_file = os.path.join(FLAGS.model_dir, 'infer_cfg.yml')
+    with open(deploy_file) as f:
+        yml_conf = yaml.safe_load(f)
+    arch = yml_conf['arch']
+    detector_func = 'Detector'
+    detector = eval(detector_func)(FLAGS.model_dir,
+                                   device=FLAGS.device,
+                                   run_mode=FLAGS.run_mode,
+                                   batch_size=FLAGS.batch_size,
+                                   trt_min_shape=FLAGS.trt_min_shape,
+                                   trt_max_shape=FLAGS.trt_max_shape,
+                                   trt_opt_shape=FLAGS.trt_opt_shape,
+                                   trt_calib_mode=FLAGS.trt_calib_mode,
+                                   cpu_threads=FLAGS.cpu_threads,
+                                   enable_mkldnn=FLAGS.enable_mkldnn,
+                                   threshold=FLAGS.threshold,
+                                   output_dir=FLAGS.output_dir)
+
+    # predict from video file or camera video stream
+    if FLAGS.video_file is not None or FLAGS.camera_id != -1:
+        detector.predict_video(FLAGS.video_file, FLAGS.camera_id)
+    else:
+        # predict from image
+        if FLAGS.image_dir is None and FLAGS.image_file is not None:
+            assert FLAGS.batch_size == 1, "batch_size should be 1, when image_file is not None"
+        img_list = get_test_images(FLAGS.image_dir, FLAGS.image_file)
+        detector.predict_image(img_list, FLAGS.run_benchmark, repeats=10)
+        if not FLAGS.run_benchmark:
+            detector.det_times.info(average=True)
+        else:
+            mode = FLAGS.run_mode
+            model_dir = FLAGS.model_dir
+            model_info = {
+                'model_name': model_dir.strip('/').split('/')[-1],
+                'precision': mode.split('_')[-1]
+            }
+            bench_log(detector, img_list, model_info, name='DET')
+
+
+if __name__ == '__main__':
+    paddle.enable_static()
+    parser = argsparser()
+    FLAGS = parser.parse_args()
+    print_arguments(FLAGS)
+    FLAGS.device = FLAGS.device.upper()
+    assert FLAGS.device in ['CPU', 'GPU', 'XPU'
+                            ], "device should be CPU, GPU or XPU"
+    main()

pptracking/python/mot/__init__.py

@@ -0,0 +1,25 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from . import matching
+from . import tracker
+from . import motion
+from . import utils
+from . import mtmct
+
+from .matching import *
+from .tracker import *
+from .motion import *
+from .utils import *
+from .mtmct import *

pptracking/python/mot/matching/__init__.py

@@ -0,0 +1,21 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from . import jde_matching
+from . import deepsort_matching
+from . import ocsort_matching
+
+from .jde_matching import *
+from .deepsort_matching import *
+from .ocsort_matching import *

pptracking/python/mot/matching/deepsort_matching.py

@@ -0,0 +1,382 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+This code is based on https://github.com/nwojke/deep_sort/tree/master/deep_sort
+"""
+
+import numpy as np
+from scipy.optimize import linear_sum_assignment
+from ..motion import kalman_filter
+
+INFTY_COST = 1e+5
+
+__all__ = [
+    'iou_1toN',
+    'iou_cost',
+    '_nn_euclidean_distance',
+    '_nn_cosine_distance',
+    'NearestNeighborDistanceMetric',
+    'min_cost_matching',
+    'matching_cascade',
+    'gate_cost_matrix',
+]
+
+
+def iou_1toN(bbox, candidates):
+    """
+    Computer intersection over union (IoU) by one box to N candidates.
+
+    Args:
+        bbox (ndarray): A bounding box in format `(top left x, top left y, width, height)`.
+            candidates (ndarray): A matrix of candidate bounding boxes (one per row) in the
+            same format as `bbox`.
+
+    Returns:
+        ious (ndarray): The intersection over union in [0, 1] between the `bbox`
+            and each candidate. A higher score means a larger fraction of the
+            `bbox` is occluded by the candidate.
+    """
+    bbox_tl = bbox[:2]
+    bbox_br = bbox[:2] + bbox[2:]
+    candidates_tl = candidates[:, :2]
+    candidates_br = candidates[:, :2] + candidates[:, 2:]
+
+    tl = np.c_[np.maximum(bbox_tl[0], candidates_tl[:, 0])[:, np.newaxis],
+               np.maximum(bbox_tl[1], candidates_tl[:, 1])[:, np.newaxis]]
+    br = np.c_[np.minimum(bbox_br[0], candidates_br[:, 0])[:, np.newaxis],
+               np.minimum(bbox_br[1], candidates_br[:, 1])[:, np.newaxis]]
+    wh = np.maximum(0., br - tl)
+
+    area_intersection = wh.prod(axis=1)
+    area_bbox = bbox[2:].prod()
+    area_candidates = candidates[:, 2:].prod(axis=1)
+    ious = area_intersection / (
+        area_bbox + area_candidates - area_intersection)
+    return ious
+
+
+def iou_cost(tracks, detections, track_indices=None, detection_indices=None):
+    """
+    IoU distance metric.
+
+    Args:
+        tracks (list[Track]): A list of tracks.
+        detections (list[Detection]): A list of detections.
+        track_indices (Optional[list[int]]): A list of indices to tracks that
+            should be matched. Defaults to all `tracks`.
+        detection_indices (Optional[list[int]]): A list of indices to detections
+            that should be matched. Defaults to all `detections`.
+
+    Returns:
+        cost_matrix (ndarray): A cost matrix of shape len(track_indices), 
+            len(detection_indices) where entry (i, j) is 
+            `1 - iou(tracks[track_indices[i]], detections[detection_indices[j]])`.
+    """
+    if track_indices is None:
+        track_indices = np.arange(len(tracks))
+    if detection_indices is None:
+        detection_indices = np.arange(len(detections))
+
+    cost_matrix = np.zeros((len(track_indices), len(detection_indices)))
+    for row, track_idx in enumerate(track_indices):
+        if tracks[track_idx].time_since_update > 1:
+            cost_matrix[row, :] = 1e+5
+            continue
+
+        bbox = tracks[track_idx].to_tlwh()
+        candidates = np.asarray(
+            [detections[i].tlwh for i in detection_indices])
+        cost_matrix[row, :] = 1. - iou_1toN(bbox, candidates)
+    return cost_matrix
+
+
+def _nn_euclidean_distance(s, q):
+    """
+    Compute pair-wise squared (Euclidean) distance between points in `s` and `q`.
+
+    Args:
+        s (ndarray): Sample points: an NxM matrix of N samples of dimensionality M.
+        q (ndarray): Query points: an LxM matrix of L samples of dimensionality M.
+
+    Returns:
+        distances (ndarray): A vector of length M that contains for each entry in `q` the
+            smallest Euclidean distance to a sample in `s`.
+    """
+    s, q = np.asarray(s), np.asarray(q)
+    if len(s) == 0 or len(q) == 0:
+        return np.zeros((len(s), len(q)))
+    s2, q2 = np.square(s).sum(axis=1), np.square(q).sum(axis=1)
+    distances = -2. * np.dot(s, q.T) + s2[:, None] + q2[None, :]
+    distances = np.clip(distances, 0., float(np.inf))
+
+    return np.maximum(0.0, distances.min(axis=0))
+
+
+def _nn_cosine_distance(s, q):
+    """
+    Compute pair-wise cosine distance between points in `s` and `q`.
+
+    Args:
+        s (ndarray): Sample points: an NxM matrix of N samples of dimensionality M.
+        q (ndarray): Query points: an LxM matrix of L samples of dimensionality M.
+
+    Returns:
+        distances (ndarray): A vector of length M that contains for each entry in `q` the
+            smallest Euclidean distance to a sample in `s`.
+    """
+    s = np.asarray(s) / np.linalg.norm(s, axis=1, keepdims=True)
+    q = np.asarray(q) / np.linalg.norm(q, axis=1, keepdims=True)
+    distances = 1. - np.dot(s, q.T)
+
+    return distances.min(axis=0)
+
+
+class NearestNeighborDistanceMetric(object):
+    """
+    A nearest neighbor distance metric that, for each target, returns
+    the closest distance to any sample that has been observed so far.
+
+    Args:
+        metric (str): Either "euclidean" or "cosine".
+        matching_threshold (float): The matching threshold. Samples with larger
+            distance are considered an invalid match.
+        budget (Optional[int]): If not None, fix samples per class to at most
+            this number. Removes the oldest samples when the budget is reached.
+
+    Attributes: 
+        samples (Dict[int -> List[ndarray]]): A dictionary that maps from target
+            identities to the list of samples that have been observed so far.
+    """
+
+    def __init__(self, metric, matching_threshold, budget=None):
+        if metric == "euclidean":
+            self._metric = _nn_euclidean_distance
+        elif metric == "cosine":
+            self._metric = _nn_cosine_distance
+        else:
+            raise ValueError(
+                "Invalid metric; must be either 'euclidean' or 'cosine'")
+        self.matching_threshold = matching_threshold
+        self.budget = budget
+        self.samples = {}
+
+    def partial_fit(self, features, targets, active_targets):
+        """
+        Update the distance metric with new data.
+
+        Args:
+            features (ndarray): An NxM matrix of N features of dimensionality M.
+            targets (ndarray): An integer array of associated target identities.
+            active_targets (List[int]): A list of targets that are currently
+                present in the scene.
+        """
+        for feature, target in zip(features, targets):
+            self.samples.setdefault(target, []).append(feature)
+            if self.budget is not None:
+                self.samples[target] = self.samples[target][-self.budget:]
+        self.samples = {k: self.samples[k] for k in active_targets}
+
+    def distance(self, features, targets):
+        """
+        Compute distance between features and targets.
+
+        Args:
+            features (ndarray): An NxM matrix of N features of dimensionality M.
+            targets (list[int]): A list of targets to match the given `features` against.
+
+        Returns:
+            cost_matrix (ndarray): a cost matrix of shape len(targets), len(features),
+                where element (i, j) contains the closest squared distance between
+                `targets[i]` and `features[j]`.
+        """
+        cost_matrix = np.zeros((len(targets), len(features)))
+        for i, target in enumerate(targets):
+            cost_matrix[i, :] = self._metric(self.samples[target], features)
+        return cost_matrix
+
+
+def min_cost_matching(distance_metric,
+                      max_distance,
+                      tracks,
+                      detections,
+                      track_indices=None,
+                      detection_indices=None):
+    """
+    Solve linear assignment problem.
+
+    Args:
+        distance_metric :
+            Callable[List[Track], List[Detection], List[int], List[int]) -> ndarray
+            The distance metric is given a list of tracks and detections as 
+            well as a list of N track indices and M detection indices. The 
+            metric should return the NxM dimensional cost matrix, where element
+            (i, j) is the association cost between the i-th track in the given
+            track indices and the j-th detection in the given detection_indices.
+        max_distance (float): Gating threshold. Associations with cost larger
+            than this value are disregarded.
+        tracks (list[Track]): A list of predicted tracks at the current time
+            step.
+        detections (list[Detection]): A list of detections at the current time
+            step.
+        track_indices (list[int]): List of track indices that maps rows in
+            `cost_matrix` to tracks in `tracks`.
+        detection_indices (List[int]): List of detection indices that maps
+            columns in `cost_matrix` to detections in `detections`.
+
+    Returns:
+        A tuple (List[(int, int)], List[int], List[int]) with the following
+        three entries:
+            * A list of matched track and detection indices.
+            * A list of unmatched track indices.
+            * A list of unmatched detection indices.
+    """
+    if track_indices is None:
+        track_indices = np.arange(len(tracks))
+    if detection_indices is None:
+        detection_indices = np.arange(len(detections))
+
+    if len(detection_indices) == 0 or len(track_indices) == 0:
+        return [], track_indices, detection_indices  # Nothing to match.
+
+    cost_matrix = distance_metric(tracks, detections, track_indices,
+                                  detection_indices)
+
+    cost_matrix[cost_matrix > max_distance] = max_distance + 1e-5
+    indices = linear_sum_assignment(cost_matrix)
+
+    matches, unmatched_tracks, unmatched_detections = [], [], []
+    for col, detection_idx in enumerate(detection_indices):
+        if col not in indices[1]:
+            unmatched_detections.append(detection_idx)
+    for row, track_idx in enumerate(track_indices):
+        if row not in indices[0]:
+            unmatched_tracks.append(track_idx)
+    for row, col in zip(indices[0], indices[1]):
+        track_idx = track_indices[row]
+        detection_idx = detection_indices[col]
+        if cost_matrix[row, col] > max_distance:
+            unmatched_tracks.append(track_idx)
+            unmatched_detections.append(detection_idx)
+        else:
+            matches.append((track_idx, detection_idx))
+    return matches, unmatched_tracks, unmatched_detections
+
+
+def matching_cascade(distance_metric,
+                     max_distance,
+                     cascade_depth,
+                     tracks,
+                     detections,
+                     track_indices=None,
+                     detection_indices=None):
+    """
+    Run matching cascade.
+
+    Args:
+        distance_metric :
+            Callable[List[Track], List[Detection], List[int], List[int]) -> ndarray
+            The distance metric is given a list of tracks and detections as 
+            well as a list of N track indices and M detection indices. The 
+            metric should return the NxM dimensional cost matrix, where element
+            (i, j) is the association cost between the i-th track in the given
+            track indices and the j-th detection in the given detection_indices.
+        max_distance (float): Gating threshold. Associations with cost larger
+            than this value are disregarded.
+        cascade_depth (int): The cascade depth, should be se to the maximum
+            track age.
+        tracks (list[Track]): A list of predicted tracks at the current time
+            step.
+        detections (list[Detection]): A list of detections at the current time
+            step.
+        track_indices (list[int]): List of track indices that maps rows in
+            `cost_matrix` to tracks in `tracks`.
+        detection_indices (List[int]): List of detection indices that maps
+            columns in `cost_matrix` to detections in `detections`.
+
+    Returns:
+        A tuple (List[(int, int)], List[int], List[int]) with the following
+        three entries:
+            * A list of matched track and detection indices.
+            * A list of unmatched track indices.
+            * A list of unmatched detection indices.
+    """
+    if track_indices is None:
+        track_indices = list(range(len(tracks)))
+    if detection_indices is None:
+        detection_indices = list(range(len(detections)))
+
+    unmatched_detections = detection_indices
+    matches = []
+    for level in range(cascade_depth):
+        if len(unmatched_detections) == 0:  # No detections left
+            break
+
+        track_indices_l = [
+            k for k in track_indices
+            if tracks[k].time_since_update == 1 + level
+        ]
+        if len(track_indices_l) == 0:  # Nothing to match at this level
+            continue
+
+        matches_l, _, unmatched_detections = \
+            min_cost_matching(
+                distance_metric, max_distance, tracks, detections,
+                track_indices_l, unmatched_detections)
+        matches += matches_l
+    unmatched_tracks = list(set(track_indices) - set(k for k, _ in matches))
+    return matches, unmatched_tracks, unmatched_detections
+
+
+def gate_cost_matrix(kf,
+                     cost_matrix,
+                     tracks,
+                     detections,
+                     track_indices,
+                     detection_indices,
+                     gated_cost=INFTY_COST,
+                     only_position=False):
+    """
+    Invalidate infeasible entries in cost matrix based on the state
+    distributions obtained by Kalman filtering.
+
+    Args:
+        kf (object): The Kalman filter.
+        cost_matrix (ndarray): The NxM dimensional cost matrix, where N is the
+            number of track indices and M is the number of detection indices,
+            such that entry (i, j) is the association cost between
+            `tracks[track_indices[i]]` and `detections[detection_indices[j]]`.
+        tracks (list[Track]): A list of predicted tracks at the current time
+            step.
+        detections (list[Detection]): A list of detections at the current time
+            step.
+        track_indices (List[int]): List of track indices that maps rows in
+            `cost_matrix` to tracks in `tracks`.
+        detection_indices (List[int]): List of detection indices that maps
+            columns in `cost_matrix` to detections in `detections`.
+        gated_cost (Optional[float]): Entries in the cost matrix corresponding
+            to infeasible associations are set this value. Defaults to a very
+            large value.
+        only_position (Optional[bool]): If True, only the x, y position of the
+            state distribution is considered during gating. Default False.
+    """
+    gating_dim = 2 if only_position else 4
+    gating_threshold = kalman_filter.chi2inv95[gating_dim]
+    measurements = np.asarray(
+        [detections[i].to_xyah() for i in detection_indices])
+    for row, track_idx in enumerate(track_indices):
+        track = tracks[track_idx]
+        gating_distance = kf.gating_distance(track.mean, track.covariance,
+                                             measurements, only_position)
+        cost_matrix[row, gating_distance > gating_threshold] = gated_cost
+    return cost_matrix

pptracking/python/mot/matching/jde_matching.py

@@ -0,0 +1,163 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+This code is based on https://github.com/Zhongdao/Towards-Realtime-MOT/blob/master/tracker/matching.py
+"""
+
+try:
+    import lap
+except:
+    print(
+        'Warning: Unable to use JDE/FairMOT/ByteTrack, please install lap, for example: `pip install lap`, see https://github.com/gatagat/lap'
+    )
+    pass
+
+import scipy
+import numpy as np
+from scipy.spatial.distance import cdist
+from ..motion import kalman_filter
+import warnings
+warnings.filterwarnings("ignore")
+
+__all__ = [
+    'merge_matches',
+    'linear_assignment',
+    'bbox_ious',
+    'iou_distance',
+    'embedding_distance',
+    'fuse_motion',
+]
+
+
+def merge_matches(m1, m2, shape):
+    O, P, Q = shape
+    m1 = np.asarray(m1)
+    m2 = np.asarray(m2)
+
+    M1 = scipy.sparse.coo_matrix(
+        (np.ones(len(m1)), (m1[:, 0], m1[:, 1])), shape=(O, P))
+    M2 = scipy.sparse.coo_matrix(
+        (np.ones(len(m2)), (m2[:, 0], m2[:, 1])), shape=(P, Q))
+
+    mask = M1 * M2
+    match = mask.nonzero()
+    match = list(zip(match[0], match[1]))
+    unmatched_O = tuple(set(range(O)) - set([i for i, j in match]))
+    unmatched_Q = tuple(set(range(Q)) - set([j for i, j in match]))
+
+    return match, unmatched_O, unmatched_Q
+
+
+def linear_assignment(cost_matrix, thresh):
+    try:
+        import lap
+    except Exception as e:
+        raise RuntimeError(
+            'Unable to use JDE/FairMOT/ByteTrack, please install lap, for example: `pip install lap`, see https://github.com/gatagat/lap'
+        )
+    if cost_matrix.size == 0:
+        return np.empty(
+            (0, 2), dtype=int), tuple(range(cost_matrix.shape[0])), tuple(
+                range(cost_matrix.shape[1]))
+    matches, unmatched_a, unmatched_b = [], [], []
+    cost, x, y = lap.lapjv(cost_matrix, extend_cost=True, cost_limit=thresh)
+    for ix, mx in enumerate(x):
+        if mx >= 0:
+            matches.append([ix, mx])
+    unmatched_a = np.where(x < 0)[0]
+    unmatched_b = np.where(y < 0)[0]
+    matches = np.asarray(matches)
+    return matches, unmatched_a, unmatched_b
+
+
+def bbox_ious(atlbrs, btlbrs):
+    boxes = np.ascontiguousarray(atlbrs, dtype=np.float)
+    query_boxes = np.ascontiguousarray(btlbrs, dtype=np.float)
+    N = boxes.shape[0]
+    K = query_boxes.shape[0]
+    ious = np.zeros((N, K), dtype=boxes.dtype)
+    if N * K == 0:
+        return ious
+
+    for k in range(K):
+        box_area = ((query_boxes[k, 2] - query_boxes[k, 0] + 1) *
+                    (query_boxes[k, 3] - query_boxes[k, 1] + 1))
+        for n in range(N):
+            iw = (min(boxes[n, 2], query_boxes[k, 2]) - max(
+                boxes[n, 0], query_boxes[k, 0]) + 1)
+            if iw > 0:
+                ih = (min(boxes[n, 3], query_boxes[k, 3]) - max(
+                    boxes[n, 1], query_boxes[k, 1]) + 1)
+                if ih > 0:
+                    ua = float((boxes[n, 2] - boxes[n, 0] + 1) * (boxes[
+                        n, 3] - boxes[n, 1] + 1) + box_area - iw * ih)
+                    ious[n, k] = iw * ih / ua
+    return ious
+
+
+def iou_distance(atracks, btracks):
+    """
+    Compute cost based on IoU between two list[STrack].
+    """
+    if (len(atracks) > 0 and isinstance(atracks[0], np.ndarray)) or (
+            len(btracks) > 0 and isinstance(btracks[0], np.ndarray)):
+        atlbrs = atracks
+        btlbrs = btracks
+    else:
+        atlbrs = [track.tlbr for track in atracks]
+        btlbrs = [track.tlbr for track in btracks]
+    _ious = bbox_ious(atlbrs, btlbrs)
+    cost_matrix = 1 - _ious
+
+    return cost_matrix
+
+
+def embedding_distance(tracks, detections, metric='euclidean'):
+    """
+    Compute cost based on features between two list[STrack].
+    """
+    cost_matrix = np.zeros((len(tracks), len(detections)), dtype=np.float)
+    if cost_matrix.size == 0:
+        return cost_matrix
+    det_features = np.asarray(
+        [track.curr_feat for track in detections], dtype=np.float)
+    track_features = np.asarray(
+        [track.smooth_feat for track in tracks], dtype=np.float)
+    cost_matrix = np.maximum(0.0, cdist(track_features, det_features,
+                                        metric))  # Nomalized features
+    return cost_matrix
+
+
+def fuse_motion(kf,
+                cost_matrix,
+                tracks,
+                detections,
+                only_position=False,
+                lambda_=0.98):
+    if cost_matrix.size == 0:
+        return cost_matrix
+    gating_dim = 2 if only_position else 4
+    gating_threshold = kalman_filter.chi2inv95[gating_dim]
+    measurements = np.asarray([det.to_xyah() for det in detections])
+    for row, track in enumerate(tracks):
+        gating_distance = kf.gating_distance(
+            track.mean,
+            track.covariance,
+            measurements,
+            only_position,
+            metric='maha')
+        cost_matrix[row, gating_distance > gating_threshold] = np.inf
+        cost_matrix[row] = lambda_ * cost_matrix[row] + (1 - lambda_
+                                                         ) * gating_distance
+    return cost_matrix

pptracking/python/mot/matching/ocsort_matching.py

@@ -0,0 +1,129 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+This code is based on https://github.com/noahcao/OC_SORT/blob/master/trackers/ocsort_tracker/association.py
+"""
+
+import os
+import numpy as np
+
+
+def iou_batch(bboxes1, bboxes2):
+    """
+    From SORT: Computes IOU between two bboxes in the form [x1,y1,x2,y2]
+    """
+    bboxes2 = np.expand_dims(bboxes2, 0)
+    bboxes1 = np.expand_dims(bboxes1, 1)
+
+    xx1 = np.maximum(bboxes1[..., 0], bboxes2[..., 0])
+    yy1 = np.maximum(bboxes1[..., 1], bboxes2[..., 1])
+    xx2 = np.minimum(bboxes1[..., 2], bboxes2[..., 2])
+    yy2 = np.minimum(bboxes1[..., 3], bboxes2[..., 3])
+    w = np.maximum(0., xx2 - xx1)
+    h = np.maximum(0., yy2 - yy1)
+    wh = w * h
+    o = wh / ((bboxes1[..., 2] - bboxes1[..., 0]) *
+              (bboxes1[..., 3] - bboxes1[..., 1]) +
+              (bboxes2[..., 2] - bboxes2[..., 0]) *
+              (bboxes2[..., 3] - bboxes2[..., 1]) - wh)
+    return (o)
+
+
+def speed_direction_batch(dets, tracks):
+    tracks = tracks[..., np.newaxis]
+    CX1, CY1 = (dets[:, 0] + dets[:, 2]) / 2.0, (dets[:, 1] + dets[:, 3]) / 2.0
+    CX2, CY2 = (tracks[:, 0] + tracks[:, 2]) / 2.0, (
+        tracks[:, 1] + tracks[:, 3]) / 2.0
+    dx = CX1 - CX2
+    dy = CY1 - CY2
+    norm = np.sqrt(dx**2 + dy**2) + 1e-6
+    dx = dx / norm
+    dy = dy / norm
+    return dy, dx  # size: num_track x num_det
+
+
+def linear_assignment(cost_matrix):
+    try:
+        import lap
+        _, x, y = lap.lapjv(cost_matrix, extend_cost=True)
+        return np.array([[y[i], i] for i in x if i >= 0])  #
+    except ImportError:
+        from scipy.optimize import linear_sum_assignment
+        x, y = linear_sum_assignment(cost_matrix)
+        return np.array(list(zip(x, y)))
+
+
+def associate(detections, trackers, iou_threshold, velocities, previous_obs,
+              vdc_weight):
+    if (len(trackers) == 0):
+        return np.empty(
+            (0, 2), dtype=int), np.arange(len(detections)), np.empty(
+                (0, 5), dtype=int)
+
+    Y, X = speed_direction_batch(detections, previous_obs)
+    inertia_Y, inertia_X = velocities[:, 0], velocities[:, 1]
+    inertia_Y = np.repeat(inertia_Y[:, np.newaxis], Y.shape[1], axis=1)
+    inertia_X = np.repeat(inertia_X[:, np.newaxis], X.shape[1], axis=1)
+    diff_angle_cos = inertia_X * X + inertia_Y * Y
+    diff_angle_cos = np.clip(diff_angle_cos, a_min=-1, a_max=1)
+    diff_angle = np.arccos(diff_angle_cos)
+    diff_angle = (np.pi / 2.0 - np.abs(diff_angle)) / np.pi
+
+    valid_mask = np.ones(previous_obs.shape[0])
+    valid_mask[np.where(previous_obs[:, 4] < 0)] = 0
+
+    iou_matrix = iou_batch(detections, trackers)
+    scores = np.repeat(
+        detections[:, -1][:, np.newaxis], trackers.shape[0], axis=1)
+    # iou_matrix = iou_matrix * scores # a trick sometiems works, we don't encourage this
+    valid_mask = np.repeat(valid_mask[:, np.newaxis], X.shape[1], axis=1)
+
+    angle_diff_cost = (valid_mask * diff_angle) * vdc_weight
+    angle_diff_cost = angle_diff_cost.T
+    angle_diff_cost = angle_diff_cost * scores
+
+    if min(iou_matrix.shape) > 0:
+        a = (iou_matrix > iou_threshold).astype(np.int32)
+        if a.sum(1).max() == 1 and a.sum(0).max() == 1:
+            matched_indices = np.stack(np.where(a), axis=1)
+        else:
+            matched_indices = linear_assignment(-(iou_matrix + angle_diff_cost
+                                                  ))
+    else:
+        matched_indices = np.empty(shape=(0, 2))
+
+    unmatched_detections = []
+    for d, det in enumerate(detections):
+        if (d not in matched_indices[:, 0]):
+            unmatched_detections.append(d)
+    unmatched_trackers = []
+    for t, trk in enumerate(trackers):
+        if (t not in matched_indices[:, 1]):
+            unmatched_trackers.append(t)
+
+    # filter out matched with low IOU
+    matches = []
+    for m in matched_indices:
+        if (iou_matrix[m[0], m[1]] < iou_threshold):
+            unmatched_detections.append(m[0])
+            unmatched_trackers.append(m[1])
+        else:
+            matches.append(m.reshape(1, 2))
+    if (len(matches) == 0):
+        matches = np.empty((0, 2), dtype=int)
+    else:
+        matches = np.concatenate(matches, axis=0)
+
+    return matches, np.array(unmatched_detections), np.array(
+        unmatched_trackers)

pptracking/python/mot/motion/__init__.py

@@ -0,0 +1,17 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from . import kalman_filter
+
+from .kalman_filter import *

pptracking/python/mot/motion/kalman_filter.py

@@ -0,0 +1,267 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+This code is based on https://github.com/nwojke/deep_sort/blob/master/deep_sort/kalman_filter.py
+"""
+
+import numpy as np
+import scipy.linalg
+
+__all__ = ['KalmanFilter']
+"""
+Table for the 0.95 quantile of the chi-square distribution with N degrees of
+freedom (contains values for N=1, ..., 9). Taken from MATLAB/Octave's chi2inv
+function and used as Mahalanobis gating threshold.
+"""
+
+chi2inv95 = {
+    1: 3.8415,
+    2: 5.9915,
+    3: 7.8147,
+    4: 9.4877,
+    5: 11.070,
+    6: 12.592,
+    7: 14.067,
+    8: 15.507,
+    9: 16.919
+}
+
+
+class KalmanFilter(object):
+    """
+    A simple Kalman filter for tracking bounding boxes in image space.
+
+    The 8-dimensional state space
+
+        x, y, a, h, vx, vy, va, vh
+
+    contains the bounding box center position (x, y), aspect ratio a, height h,
+    and their respective velocities.
+
+    Object motion follows a constant velocity model. The bounding box location
+    (x, y, a, h) is taken as direct observation of the state space (linear
+    observation model).
+
+    """
+
+    def __init__(self):
+        ndim, dt = 4, 1.
+
+        # Create Kalman filter model matrices.
+        self._motion_mat = np.eye(2 * ndim, 2 * ndim)
+        for i in range(ndim):
+            self._motion_mat[i, ndim + i] = dt
+        self._update_mat = np.eye(ndim, 2 * ndim)
+
+        # Motion and observation uncertainty are chosen relative to the current
+        # state estimate. These weights control the amount of uncertainty in
+        # the model. This is a bit hacky.
+        self._std_weight_position = 1. / 20
+        self._std_weight_velocity = 1. / 160
+
+    def initiate(self, measurement):
+        """
+        Create track from unassociated measurement.
+
+        Args:
+            measurement (ndarray): Bounding box coordinates (x, y, a, h) with
+                center position (x, y), aspect ratio a, and height h.
+
+        Returns:
+            The mean vector (8 dimensional) and covariance matrix (8x8
+            dimensional) of the new track. Unobserved velocities are 
+            initialized to 0 mean.
+        """
+        mean_pos = measurement
+        mean_vel = np.zeros_like(mean_pos)
+        mean = np.r_[mean_pos, mean_vel]
+
+        std = [
+            2 * self._std_weight_position * measurement[3],
+            2 * self._std_weight_position * measurement[3], 1e-2,
+            2 * self._std_weight_position * measurement[3],
+            10 * self._std_weight_velocity * measurement[3],
+            10 * self._std_weight_velocity * measurement[3], 1e-5,
+            10 * self._std_weight_velocity * measurement[3]
+        ]
+        covariance = np.diag(np.square(std))
+        return mean, covariance
+
+    def predict(self, mean, covariance):
+        """
+        Run Kalman filter prediction step.
+
+        Args:
+            mean (ndarray): The 8 dimensional mean vector of the object state
+                at the previous time step.
+            covariance (ndarray): The 8x8 dimensional covariance matrix of the
+                object state at the previous time step.
+
+        Returns:
+            The mean vector and covariance matrix of the predicted state. 
+            Unobserved velocities are initialized to 0 mean.
+        """
+        std_pos = [
+            self._std_weight_position * mean[3], self._std_weight_position *
+            mean[3], 1e-2, self._std_weight_position * mean[3]
+        ]
+        std_vel = [
+            self._std_weight_velocity * mean[3], self._std_weight_velocity *
+            mean[3], 1e-5, self._std_weight_velocity * mean[3]
+        ]
+        motion_cov = np.diag(np.square(np.r_[std_pos, std_vel]))
+
+        #mean = np.dot(self._motion_mat, mean)
+        mean = np.dot(mean, self._motion_mat.T)
+        covariance = np.linalg.multi_dot(
+            (self._motion_mat, covariance, self._motion_mat.T)) + motion_cov
+
+        return mean, covariance
+
+    def project(self, mean, covariance):
+        """
+        Project state distribution to measurement space.
+
+        Args
+            mean (ndarray): The state's mean vector (8 dimensional array).
+            covariance (ndarray): The state's covariance matrix (8x8 dimensional).
+
+        Returns:
+            The projected mean and covariance matrix of the given state estimate.
+        """
+        std = [
+            self._std_weight_position * mean[3], self._std_weight_position *
+            mean[3], 1e-1, self._std_weight_position * mean[3]
+        ]
+        innovation_cov = np.diag(np.square(std))
+
+        mean = np.dot(self._update_mat, mean)
+        covariance = np.linalg.multi_dot((self._update_mat, covariance,
+                                          self._update_mat.T))
+        return mean, covariance + innovation_cov
+
+    def multi_predict(self, mean, covariance):
+        """
+        Run Kalman filter prediction step (Vectorized version).
+        
+        Args:
+            mean (ndarray): The Nx8 dimensional mean matrix of the object states
+                at the previous time step.
+            covariance (ndarray): The Nx8x8 dimensional covariance matrics of the
+                object states at the previous time step.
+
+        Returns:
+            The mean vector and covariance matrix of the predicted state.
+            Unobserved velocities are initialized to 0 mean.
+        """
+        std_pos = [
+            self._std_weight_position * mean[:, 3], self._std_weight_position *
+            mean[:, 3], 1e-2 * np.ones_like(mean[:, 3]),
+            self._std_weight_position * mean[:, 3]
+        ]
+        std_vel = [
+            self._std_weight_velocity * mean[:, 3], self._std_weight_velocity *
+            mean[:, 3], 1e-5 * np.ones_like(mean[:, 3]),
+            self._std_weight_velocity * mean[:, 3]
+        ]
+        sqr = np.square(np.r_[std_pos, std_vel]).T
+
+        motion_cov = []
+        for i in range(len(mean)):
+            motion_cov.append(np.diag(sqr[i]))
+        motion_cov = np.asarray(motion_cov)
+
+        mean = np.dot(mean, self._motion_mat.T)
+        left = np.dot(self._motion_mat, covariance).transpose((1, 0, 2))
+        covariance = np.dot(left, self._motion_mat.T) + motion_cov
+
+        return mean, covariance
+
+    def update(self, mean, covariance, measurement):
+        """
+        Run Kalman filter correction step.
+
+        Args:
+            mean (ndarray): The predicted state's mean vector (8 dimensional).
+            covariance (ndarray): The state's covariance matrix (8x8 dimensional).
+            measurement (ndarray): The 4 dimensional measurement vector
+                (x, y, a, h), where (x, y) is the center position, a the aspect
+                ratio, and h the height of the bounding box.
+
+        Returns:
+            The measurement-corrected state distribution.
+        """
+        projected_mean, projected_cov = self.project(mean, covariance)
+
+        chol_factor, lower = scipy.linalg.cho_factor(
+            projected_cov, lower=True, check_finite=False)
+        kalman_gain = scipy.linalg.cho_solve(
+            (chol_factor, lower),
+            np.dot(covariance, self._update_mat.T).T,
+            check_finite=False).T
+        innovation = measurement - projected_mean
+
+        new_mean = mean + np.dot(innovation, kalman_gain.T)
+        new_covariance = covariance - np.linalg.multi_dot(
+            (kalman_gain, projected_cov, kalman_gain.T))
+        return new_mean, new_covariance
+
+    def gating_distance(self,
+                        mean,
+                        covariance,
+                        measurements,
+                        only_position=False,
+                        metric='maha'):
+        """
+        Compute gating distance between state distribution and measurements.
+        A suitable distance threshold can be obtained from `chi2inv95`. If
+        `only_position` is False, the chi-square distribution has 4 degrees of
+        freedom, otherwise 2.
+        
+        Args:
+            mean (ndarray): Mean vector over the state distribution (8
+                dimensional).
+            covariance (ndarray): Covariance of the state distribution (8x8
+                dimensional).
+            measurements (ndarray): An Nx4 dimensional matrix of N measurements,
+                each in format (x, y, a, h) where (x, y) is the bounding box center
+                position, a the aspect ratio, and h the height.
+            only_position (Optional[bool]): If True, distance computation is 
+                done with respect to the bounding box center position only.
+            metric (str): Metric type, 'gaussian' or 'maha'.
+
+        Returns
+            An array of length N, where the i-th element contains the squared
+            Mahalanobis distance between (mean, covariance) and `measurements[i]`.
+        """
+        mean, covariance = self.project(mean, covariance)
+        if only_position:
+            mean, covariance = mean[:2], covariance[:2, :2]
+            measurements = measurements[:, :2]
+
+        d = measurements - mean
+        if metric == 'gaussian':
+            return np.sum(d * d, axis=1)
+        elif metric == 'maha':
+            cholesky_factor = np.linalg.cholesky(covariance)
+            z = scipy.linalg.solve_triangular(
+                cholesky_factor,
+                d.T,
+                lower=True,
+                check_finite=False,
+                overwrite_b=True)
+            squared_maha = np.sum(z * z, axis=0)
+            return squared_maha
+        else:
+            raise ValueError('invalid distance metric')

pptracking/python/mot/mtmct/__init__.py

@@ -0,0 +1,24 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from . import utils
+from . import postprocess
+from .utils import *
+from .postprocess import *
+
+# The following codes are strongly related to zone and camera parameters
+from . import camera_utils
+from . import zone
+from .camera_utils import *
+from .zone import *

pptracking/python/mot/mtmct/camera_utils.py

@@ -0,0 +1,288 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+This code is based on https://github.com/LCFractal/AIC21-MTMC/tree/main/reid/reid-matching/tools
+
+Note: The following codes are strongly related to camera parameters of the AIC21 test-set S06,
+    so they can only be used in S06, and can not be used for other MTMCT datasets.
+"""
+
+import numpy as np
+try:
+    from sklearn.cluster import AgglomerativeClustering
+except:
+    print(
+        'Warning: Unable to use MTMCT in PP-Tracking, please install sklearn, for example: `pip install sklearn`'
+    )
+    pass
+from .utils import get_dire, get_match, get_cid_tid, combin_feature, combin_cluster
+from .utils import normalize, intracam_ignore, visual_rerank
+
+__all__ = [
+    'st_filter',
+    'get_labels_with_camera',
+]
+
+CAM_DIST = [[0, 40, 55, 100, 120, 145], [40, 0, 15, 60, 80, 105],
+            [55, 15, 0, 40, 65, 90], [100, 60, 40, 0, 20, 45],
+            [120, 80, 65, 20, 0, 25], [145, 105, 90, 45, 25, 0]]
+
+
+def st_filter(st_mask, cid_tids, cid_tid_dict):
+    count = len(cid_tids)
+    for i in range(count):
+        i_tracklet = cid_tid_dict[cid_tids[i]]
+        i_cid = i_tracklet['cam']
+        i_dire = get_dire(i_tracklet['zone_list'], i_cid)
+        i_iot = i_tracklet['io_time']
+        for j in range(count):
+            j_tracklet = cid_tid_dict[cid_tids[j]]
+            j_cid = j_tracklet['cam']
+            j_dire = get_dire(j_tracklet['zone_list'], j_cid)
+            j_iot = j_tracklet['io_time']
+
+            match_dire = True
+            cam_dist = CAM_DIST[i_cid - 41][j_cid - 41]
+            # if time overlopped
+            if i_iot[0] - cam_dist < j_iot[0] and j_iot[0] < i_iot[
+                    1] + cam_dist:
+                match_dire = False
+            if i_iot[0] - cam_dist < j_iot[1] and j_iot[1] < i_iot[
+                    1] + cam_dist:
+                match_dire = False
+
+            # not match after go out
+            if i_dire[1] in [1, 2]:  # i out
+                if i_iot[0] < j_iot[1] + cam_dist:
+                    match_dire = False
+
+            if i_dire[1] in [1, 2]:
+                if i_dire[0] in [3] and i_cid > j_cid:
+                    match_dire = False
+                if i_dire[0] in [4] and i_cid < j_cid:
+                    match_dire = False
+
+            if i_cid in [41] and i_dire[1] in [4]:
+                if i_iot[0] < j_iot[1] + cam_dist:
+                    match_dire = False
+                if i_iot[1] > 199:
+                    match_dire = False
+            if i_cid in [46] and i_dire[1] in [3]:
+                if i_iot[0] < j_iot[1] + cam_dist:
+                    match_dire = False
+
+            # match after come into
+            if i_dire[0] in [1, 2]:
+                if i_iot[1] > j_iot[0] - cam_dist:
+                    match_dire = False
+
+            if i_dire[0] in [1, 2]:
+                if i_dire[1] in [3] and i_cid > j_cid:
+                    match_dire = False
+                if i_dire[1] in [4] and i_cid < j_cid:
+                    match_dire = False
+
+            is_ignore = False
+            if ((i_dire[0] == i_dire[1] and i_dire[0] in [3, 4]) or
+                (j_dire[0] == j_dire[1] and j_dire[0] in [3, 4])):
+                is_ignore = True
+
+            if not is_ignore:
+                # direction conflict
+                if (i_dire[0] in [3] and j_dire[0] in [4]) or (
+                        i_dire[1] in [3] and j_dire[1] in [4]):
+                    match_dire = False
+                # filter before going next scene
+                if i_dire[1] in [3] and i_cid < j_cid:
+                    if i_iot[1] > j_iot[1] - cam_dist:
+                        match_dire = False
+                if i_dire[1] in [4] and i_cid > j_cid:
+                    if i_iot[1] > j_iot[1] - cam_dist:
+                        match_dire = False
+
+                if i_dire[0] in [3] and i_cid < j_cid:
+                    if i_iot[0] < j_iot[0] + cam_dist:
+                        match_dire = False
+                if i_dire[0] in [4] and i_cid > j_cid:
+                    if i_iot[0] < j_iot[0] + cam_dist:
+                        match_dire = False
+                ## 3-30
+                ## 4-1
+                if i_dire[0] in [3] and i_cid > j_cid:
+                    if i_iot[1] > j_iot[0] - cam_dist:
+                        match_dire = False
+                if i_dire[0] in [4] and i_cid < j_cid:
+                    if i_iot[1] > j_iot[0] - cam_dist:
+                        match_dire = False
+                # filter before going next scene
+                ## 4-7
+                if i_dire[1] in [3] and i_cid > j_cid:
+                    if i_iot[0] < j_iot[1] + cam_dist:
+                        match_dire = False
+                if i_dire[1] in [4] and i_cid < j_cid:
+                    if i_iot[0] < j_iot[1] + cam_dist:
+                        match_dire = False
+            else:
+                if i_iot[1] > 199:
+                    if i_dire[0] in [3] and i_cid < j_cid:
+                        if i_iot[0] < j_iot[0] + cam_dist:
+                            match_dire = False
+                    if i_dire[0] in [4] and i_cid > j_cid:
+                        if i_iot[0] < j_iot[0] + cam_dist:
+                            match_dire = False
+                    if i_dire[0] in [3] and i_cid > j_cid:
+                        match_dire = False
+                    if i_dire[0] in [4] and i_cid < j_cid:
+                        match_dire = False
+                if i_iot[0] < 1:
+                    if i_dire[1] in [3] and i_cid > j_cid:
+                        match_dire = False
+                    if i_dire[1] in [4] and i_cid < j_cid:
+                        match_dire = False
+
+            if not match_dire:
+                st_mask[i, j] = 0.0
+                st_mask[j, i] = 0.0
+    return st_mask
+
+
+def subcam_list(cid_tid_dict, cid_tids):
+    sub_3_4 = dict()
+    sub_4_3 = dict()
+    for cid_tid in cid_tids:
+        cid, tid = cid_tid
+        tracklet = cid_tid_dict[cid_tid]
+        zs, ze = get_dire(tracklet['zone_list'], cid)
+        if zs in [3] and cid not in [46]:  # 4 to 3
+            if not cid + 1 in sub_4_3:
+                sub_4_3[cid + 1] = []
+            sub_4_3[cid + 1].append(cid_tid)
+        if ze in [4] and cid not in [41]:  # 4 to 3
+            if not cid in sub_4_3:
+                sub_4_3[cid] = []
+            sub_4_3[cid].append(cid_tid)
+        if zs in [4] and cid not in [41]:  # 3 to 4
+            if not cid - 1 in sub_3_4:
+                sub_3_4[cid - 1] = []
+            sub_3_4[cid - 1].append(cid_tid)
+        if ze in [3] and cid not in [46]:  # 3 to 4
+            if not cid in sub_3_4:
+                sub_3_4[cid] = []
+            sub_3_4[cid].append(cid_tid)
+    sub_cid_tids = dict()
+    for i in sub_3_4:
+        sub_cid_tids[(i, i + 1)] = sub_3_4[i]
+    for i in sub_4_3:
+        sub_cid_tids[(i, i - 1)] = sub_4_3[i]
+    return sub_cid_tids
+
+
+def subcam_list2(cid_tid_dict, cid_tids):
+    sub_dict = dict()
+    for cid_tid in cid_tids:
+        cid, tid = cid_tid
+        if cid not in [41]:
+            if not cid in sub_dict:
+                sub_dict[cid] = []
+            sub_dict[cid].append(cid_tid)
+        if cid not in [46]:
+            if not cid + 1 in sub_dict:
+                sub_dict[cid + 1] = []
+            sub_dict[cid + 1].append(cid_tid)
+    return sub_dict
+
+
+def get_sim_matrix(cid_tid_dict,
+                   cid_tids,
+                   use_ff=True,
+                   use_rerank=True,
+                   use_st_filter=False):
+    # Note: camera releated get_sim_matrix function,
+    # which is different from the one in utils.py.
+    count = len(cid_tids)
+
+    q_arr = np.array(
+        [cid_tid_dict[cid_tids[i]]['mean_feat'] for i in range(count)])
+    g_arr = np.array(
+        [cid_tid_dict[cid_tids[i]]['mean_feat'] for i in range(count)])
+    q_arr = normalize(q_arr, axis=1)
+    g_arr = normalize(g_arr, axis=1)
+
+    st_mask = np.ones((count, count), dtype=np.float32)
+    st_mask = intracam_ignore(st_mask, cid_tids)
+
+    # different from utils.py
+    if use_st_filter:
+        st_mask = st_filter(st_mask, cid_tids, cid_tid_dict)
+
+    visual_sim_matrix = visual_rerank(
+        q_arr, g_arr, cid_tids, use_ff=use_ff, use_rerank=use_rerank)
+    visual_sim_matrix = visual_sim_matrix.astype('float32')
+
+    np.set_printoptions(precision=3)
+    sim_matrix = visual_sim_matrix * st_mask
+
+    np.fill_diagonal(sim_matrix, 0)
+    return sim_matrix
+
+
+def get_labels_with_camera(cid_tid_dict,
+                           cid_tids,
+                           use_ff=True,
+                           use_rerank=True,
+                           use_st_filter=False):
+    # 1st cluster
+    sub_cid_tids = subcam_list(cid_tid_dict, cid_tids)
+    sub_labels = dict()
+    dis_thrs = [0.7, 0.5, 0.5, 0.5, 0.5, 0.7, 0.5, 0.5, 0.5, 0.5]
+
+    for i, sub_c_to_c in enumerate(sub_cid_tids):
+        sim_matrix = get_sim_matrix(
+            cid_tid_dict,
+            sub_cid_tids[sub_c_to_c],
+            use_ff=use_ff,
+            use_rerank=use_rerank,
+            use_st_filter=use_st_filter)
+        cluster_labels = AgglomerativeClustering(
+            n_clusters=None,
+            distance_threshold=1 - dis_thrs[i],
+            affinity='precomputed',
+            linkage='complete').fit_predict(1 - sim_matrix)
+        labels = get_match(cluster_labels)
+        cluster_cid_tids = get_cid_tid(labels, sub_cid_tids[sub_c_to_c])
+        sub_labels[sub_c_to_c] = cluster_cid_tids
+    labels, sub_cluster = combin_cluster(sub_labels, cid_tids)
+
+    # 2nd cluster
+    cid_tid_dict_new = combin_feature(cid_tid_dict, sub_cluster)
+    sub_cid_tids = subcam_list2(cid_tid_dict_new, cid_tids)
+    sub_labels = dict()
+    for i, sub_c_to_c in enumerate(sub_cid_tids):
+        sim_matrix = get_sim_matrix(
+            cid_tid_dict_new,
+            sub_cid_tids[sub_c_to_c],
+            use_ff=use_ff,
+            use_rerank=use_rerank,
+            use_st_filter=use_st_filter)
+        cluster_labels = AgglomerativeClustering(
+            n_clusters=None,
+            distance_threshold=1 - 0.1,
+            affinity='precomputed',
+            linkage='complete').fit_predict(1 - sim_matrix)
+        labels = get_match(cluster_labels)
+        cluster_cid_tids = get_cid_tid(labels, sub_cid_tids[sub_c_to_c])
+        sub_labels[sub_c_to_c] = cluster_cid_tids
+    labels, sub_cluster = combin_cluster(sub_labels, cid_tids)
+
+    return labels

pptracking/python/mot/mtmct/postprocess.py

@@ -0,0 +1,386 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+This code is based on https://github.com/LCFractal/AIC21-MTMC/tree/main/reid/reid-matching/tools
+"""
+
+import os
+import re
+import cv2
+from tqdm import tqdm
+import numpy as np
+try:
+    import motmetrics as mm
+except:
+    print(
+        'Warning: Unable to use motmetrics in MTMCT in PP-Tracking, please install motmetrics, for example: `pip install motmetrics`, see https://github.com/longcw/py-motmetrics'
+    )
+    pass
+from functools import reduce
+
+from .utils import parse_pt_gt, parse_pt, compare_dataframes_mtmc
+from .utils import get_labels, getData, gen_new_mot
+from .camera_utils import get_labels_with_camera
+from .zone import Zone
+from ..visualize import plot_tracking
+
+__all__ = [
+    'trajectory_fusion',
+    'sub_cluster',
+    'gen_res',
+    'print_mtmct_result',
+    'get_mtmct_matching_results',
+    'save_mtmct_crops',
+    'save_mtmct_vis_results',
+]
+
+
+def trajectory_fusion(mot_feature, cid, cid_bias, use_zone=False,
+                      zone_path=''):
+    cur_bias = cid_bias[cid]
+    mot_list_break = {}
+    if use_zone:
+        zones = Zone(zone_path=zone_path)
+        zones.set_cam(cid)
+        mot_list = parse_pt(mot_feature, zones)
+    else:
+        mot_list = parse_pt(mot_feature)
+
+    if use_zone:
+        mot_list = zones.break_mot(mot_list, cid)
+        mot_list = zones.filter_mot(mot_list, cid)  # filter by zone
+        mot_list = zones.filter_bbox(mot_list, cid)  # filter bbox
+
+    mot_list_break = gen_new_mot(mot_list)  # save break feature for gen result
+
+    tid_data = dict()
+    for tid in mot_list:
+        tracklet = mot_list[tid]
+        if len(tracklet) <= 1:
+            continue
+        frame_list = list(tracklet.keys())
+        frame_list.sort()
+        # filter area too large
+        zone_list = [tracklet[f]['zone'] for f in frame_list]
+        feature_list = [
+            tracklet[f]['feat'] for f in frame_list
+            if (tracklet[f]['bbox'][3] - tracklet[f]['bbox'][1]
+                ) * (tracklet[f]['bbox'][2] - tracklet[f]['bbox'][0]) > 2000
+        ]
+        if len(feature_list) < 2:
+            feature_list = [tracklet[f]['feat'] for f in frame_list]
+        io_time = [
+            cur_bias + frame_list[0] / 10., cur_bias + frame_list[-1] / 10.
+        ]
+        all_feat = np.array([feat for feat in feature_list])
+        mean_feat = np.mean(all_feat, axis=0)
+        tid_data[tid] = {
+            'cam': cid,
+            'tid': tid,
+            'mean_feat': mean_feat,
+            'zone_list': zone_list,
+            'frame_list': frame_list,
+            'tracklet': tracklet,
+            'io_time': io_time
+        }
+    return tid_data, mot_list_break
+
+
+def sub_cluster(cid_tid_dict,
+                scene_cluster,
+                use_ff=True,
+                use_rerank=True,
+                use_camera=False,
+                use_st_filter=False):
+    '''
+    cid_tid_dict: all camera_id and track_id
+    scene_cluster: like [41, 42, 43, 44, 45, 46] in AIC21 MTMCT S06 test videos
+    '''
+    assert (len(scene_cluster) != 0), "Error: scene_cluster length equals 0"
+    cid_tids = sorted(
+        [key for key in cid_tid_dict.keys() if key[0] in scene_cluster])
+    if use_camera:
+        clu = get_labels_with_camera(
+            cid_tid_dict,
+            cid_tids,
+            use_ff=use_ff,
+            use_rerank=use_rerank,
+            use_st_filter=use_st_filter)
+    else:
+        clu = get_labels(
+            cid_tid_dict,
+            cid_tids,
+            use_ff=use_ff,
+            use_rerank=use_rerank,
+            use_st_filter=use_st_filter)
+    new_clu = list()
+    for c_list in clu:
+        if len(c_list) <= 1: continue
+        cam_list = [cid_tids[c][0] for c in c_list]
+        if len(cam_list) != len(set(cam_list)): continue
+        new_clu.append([cid_tids[c] for c in c_list])
+    all_clu = new_clu
+    cid_tid_label = dict()
+    for i, c_list in enumerate(all_clu):
+        for c in c_list:
+            cid_tid_label[c] = i + 1
+    return cid_tid_label
+
+
+def gen_res(output_dir_filename,
+            scene_cluster,
+            map_tid,
+            mot_list_breaks,
+            use_roi=False,
+            roi_dir=''):
+    f_w = open(output_dir_filename, 'w')
+    for idx, mot_feature in enumerate(mot_list_breaks):
+        cid = scene_cluster[idx]
+        img_rects = parse_pt_gt(mot_feature)
+        if use_roi:
+            assert (roi_dir != ''), "Error: roi_dir is not empty!"
+            roi = cv2.imread(os.path.join(roi_dir, f'c{cid:03d}/roi.jpg'), 0)
+            height, width = roi.shape
+
+        for fid in img_rects:
+            tid_rects = img_rects[fid]
+            fid = int(fid) + 1
+            for tid_rect in tid_rects:
+                tid = tid_rect[0]
+                rect = tid_rect[1:]
+                cx = 0.5 * rect[0] + 0.5 * rect[2]
+                cy = 0.5 * rect[1] + 0.5 * rect[3]
+                w = rect[2] - rect[0]
+                w = min(w * 1.2, w + 40)
+                h = rect[3] - rect[1]
+                h = min(h * 1.2, h + 40)
+                rect[2] -= rect[0]
+                rect[3] -= rect[1]
+                rect[0] = max(0, rect[0])
+                rect[1] = max(0, rect[1])
+                x1, y1 = max(0, cx - 0.5 * w), max(0, cy - 0.5 * h)
+                if use_roi:
+                    x2, y2 = min(width, cx + 0.5 * w), min(height,
+                                                           cy + 0.5 * h)
+                else:
+                    x2, y2 = cx + 0.5 * w, cy + 0.5 * h
+                w, h = x2 - x1, y2 - y1
+                new_rect = list(map(int, [x1, y1, w, h]))
+                rect = list(map(int, rect))
+                if (cid, tid) in map_tid:
+                    new_tid = map_tid[(cid, tid)]
+                    f_w.write(
+                        str(cid) + ' ' + str(new_tid) + ' ' + str(fid) + ' ' +
+                        ' '.join(map(str, new_rect)) + ' -1 -1'
+                        '\n')
+    print('gen_res: write file in {}'.format(output_dir_filename))
+    f_w.close()
+
+
+def print_mtmct_result(gt_file, pred_file):
+    names = [
+        'CameraId', 'Id', 'FrameId', 'X', 'Y', 'Width', 'Height', 'Xworld',
+        'Yworld'
+    ]
+    gt = getData(gt_file, names=names)
+    pred = getData(pred_file, names=names)
+    summary = compare_dataframes_mtmc(gt, pred)
+    print('MTMCT summary: ', summary.columns.tolist())
+
+    formatters = {
+        'idf1': '{:2.2f}'.format,
+        'idp': '{:2.2f}'.format,
+        'idr': '{:2.2f}'.format,
+        'mota': '{:2.2f}'.format
+    }
+    summary = summary[['idf1', 'idp', 'idr', 'mota']]
+    summary.loc[:, 'idp'] *= 100
+    summary.loc[:, 'idr'] *= 100
+    summary.loc[:, 'idf1'] *= 100
+    summary.loc[:, 'mota'] *= 100
+    try:
+        import motmetrics as mm
+    except Exception as e:
+        raise RuntimeError(
+            'Unable to use motmetrics in MTMCT in PP-Tracking, please install motmetrics, for example: `pip install motmetrics`, see https://github.com/longcw/py-motmetrics'
+        )
+    print(
+        mm.io.render_summary(
+            summary,
+            formatters=formatters,
+            namemap=mm.io.motchallenge_metric_names))
+
+
+def get_mtmct_matching_results(pred_mtmct_file,
+                               secs_interval=0.5,
+                               video_fps=20):
+    res = np.loadtxt(pred_mtmct_file)  # 'cid, tid, fid, x1, y1, w, h, -1, -1'
+    camera_ids = list(map(int, np.unique(res[:, 0])))
+
+    res = res[:, :7]
+    # each line in res: 'cid, tid, fid, x1, y1, w, h'
+
+    camera_tids = []
+    camera_results = dict()
+    for c_id in camera_ids:
+        camera_results[c_id] = res[res[:, 0] == c_id]
+        tids = np.unique(camera_results[c_id][:, 1])
+        tids = list(map(int, tids))
+        camera_tids.append(tids)
+
+    # select common tids throughout each video
+    common_tids = reduce(np.intersect1d, camera_tids)
+    if len(common_tids) == 0:
+        print(
+            'No common tracked ids in these videos, please check your MOT result or select new videos.'
+        )
+        return None, None
+
+    # get mtmct matching results by cid_tid_fid_results[c_id][t_id][f_id]
+    cid_tid_fid_results = dict()
+    cid_tid_to_fids = dict()
+    interval = int(secs_interval * video_fps)  # preferably less than 10
+    for c_id in camera_ids:
+        cid_tid_fid_results[c_id] = dict()
+        cid_tid_to_fids[c_id] = dict()
+        for t_id in common_tids:
+            tid_mask = camera_results[c_id][:, 1] == t_id
+            cid_tid_fid_results[c_id][t_id] = dict()
+
+            camera_trackid_results = camera_results[c_id][tid_mask]
+            fids = np.unique(camera_trackid_results[:, 2])
+            fids = fids[fids % interval == 0]
+            fids = list(map(int, fids))
+            cid_tid_to_fids[c_id][t_id] = fids
+
+            for f_id in fids:
+                st_frame = f_id
+                ed_frame = f_id + interval
+
+                st_mask = camera_trackid_results[:, 2] >= st_frame
+                ed_mask = camera_trackid_results[:, 2] < ed_frame
+                frame_mask = np.logical_and(st_mask, ed_mask)
+                cid_tid_fid_results[c_id][t_id][f_id] = camera_trackid_results[
+                    frame_mask]
+
+    return camera_results, cid_tid_fid_results
+
+
+def save_mtmct_crops(cid_tid_fid_res,
+                     images_dir,
+                     crops_dir,
+                     width=300,
+                     height=200):
+    camera_ids = cid_tid_fid_res.keys()
+    seqs_folder = os.listdir(images_dir)
+    seqs = []
+    for x in seqs_folder:
+        if os.path.isdir(os.path.join(images_dir, x)):
+            seqs.append(x)
+    assert len(seqs) == len(camera_ids)
+    seqs.sort()
+
+    if not os.path.exists(crops_dir):
+        os.makedirs(crops_dir)
+
+    common_tids = list(cid_tid_fid_res[list(camera_ids)[0]].keys())
+
+    # get crops by name 'tid_cid_fid.jpg
+    for t_id in common_tids:
+        for i, c_id in enumerate(camera_ids):
+            infer_dir = os.path.join(images_dir, seqs[i])
+            if os.path.exists(os.path.join(infer_dir, 'img1')):
+                infer_dir = os.path.join(infer_dir, 'img1')
+            all_images = os.listdir(infer_dir)
+            all_images.sort()
+
+            for f_id in cid_tid_fid_res[c_id][t_id].keys():
+                frame_idx = f_id - 1 if f_id > 0 else 0
+                im_path = os.path.join(infer_dir, all_images[frame_idx])
+
+                im = cv2.imread(im_path)  # (H, W, 3)
+
+                # only select one track
+                track = cid_tid_fid_res[c_id][t_id][f_id][0]
+
+                cid, tid, fid, x1, y1, w, h = [int(v) for v in track]
+                clip = im[y1:(y1 + h), x1:(x1 + w)]
+                clip = cv2.resize(clip, (width, height))
+
+                cv2.imwrite(
+                    os.path.join(crops_dir,
+                                 'tid{:06d}_cid{:06d}_fid{:06d}.jpg'.format(
+                                     tid, cid, fid)), clip)
+
+            print("Finish cropping image of tracked_id {} in camera: {}".
+                  format(t_id, c_id))
+
+
+def save_mtmct_vis_results(camera_results,
+                           images_dir,
+                           save_dir,
+                           save_videos=False):
+    # camera_results: 'cid, tid, fid, x1, y1, w, h'
+    camera_ids = camera_results.keys()
+    seqs_folder = os.listdir(images_dir)
+    seqs = []
+    for x in seqs_folder:
+        if os.path.isdir(os.path.join(images_dir, x)):
+            seqs.append(x)
+    assert len(seqs) == len(camera_ids)
+    seqs.sort()
+
+    if not os.path.exists(save_dir):
+        os.makedirs(save_dir)
+
+    for i, c_id in enumerate(camera_ids):
+        print("Start visualization for camera {} of sequence {}.".format(
+            c_id, seqs[i]))
+        cid_save_dir = os.path.join(save_dir, '{}'.format(seqs[i]))
+        if not os.path.exists(cid_save_dir):
+            os.makedirs(cid_save_dir)
+
+        infer_dir = os.path.join(images_dir, seqs[i])
+        if os.path.exists(os.path.join(infer_dir, 'img1')):
+            infer_dir = os.path.join(infer_dir, 'img1')
+        all_images = os.listdir(infer_dir)
+        all_images.sort()
+
+        for f_id, im_path in enumerate(all_images):
+            img = cv2.imread(os.path.join(infer_dir, im_path))
+            tracks = camera_results[c_id][camera_results[c_id][:, 2] == f_id]
+            if tracks.shape[0] > 0:
+                tracked_ids = tracks[:, 1]
+                xywhs = tracks[:, 3:]
+                online_im = plot_tracking(
+                    img, xywhs, tracked_ids, scores=None, frame_id=f_id)
+            else:
+                online_im = img
+                print('Frame {} of seq {} has no tracking results'.format(
+                    f_id, seqs[i]))
+
+            cv2.imwrite(
+                os.path.join(cid_save_dir, '{:05d}.jpg'.format(f_id)),
+                online_im)
+            if f_id % 40 == 0:
+                print('Processing frame {}'.format(f_id))
+
+        if save_videos:
+            output_video_path = os.path.join(
+                cid_save_dir, '..', '{}_mtmct_vis.mp4'.format(seqs[i]))
+            cmd_str = 'ffmpeg -f image2 -i {}/%05d.jpg {}'.format(
+                cid_save_dir, output_video_path)
+            os.system(cmd_str)
+            print('Save camera {} video in {}.'.format(seqs[i],
+                                                       output_video_path))

pptracking/python/mot/mtmct/utils.py

@@ -0,0 +1,604 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+This code is based on https://github.com/LCFractal/AIC21-MTMC/tree/main/reid/reid-matching/tools
+"""
+
+import os
+import re
+import cv2
+import gc
+import numpy as np
+import pandas as pd
+from tqdm import tqdm
+import warnings
+warnings.filterwarnings("ignore")
+
+__all__ = [
+    'parse_pt', 'parse_bias', 'get_dire', 'parse_pt_gt',
+    'compare_dataframes_mtmc', 'get_sim_matrix', 'get_labels', 'getData',
+    'gen_new_mot'
+]
+
+
+def parse_pt(mot_feature, zones=None):
+    mot_list = dict()
+    for line in mot_feature:
+        fid = int(re.sub('[a-z,A-Z]', "", mot_feature[line]['frame']))
+        tid = mot_feature[line]['id']
+        bbox = list(map(lambda x: int(float(x)), mot_feature[line]['bbox']))
+        if tid not in mot_list:
+            mot_list[tid] = dict()
+        out_dict = mot_feature[line]
+        if zones is not None:
+            out_dict['zone'] = zones.get_zone(bbox)
+        else:
+            out_dict['zone'] = None
+        mot_list[tid][fid] = out_dict
+    return mot_list
+
+
+def gen_new_mot(mot_list):
+    out_dict = dict()
+    for tracklet in mot_list:
+        tracklet = mot_list[tracklet]
+        for f in tracklet:
+            out_dict[tracklet[f]['imgname']] = tracklet[f]
+    return out_dict
+
+
+def mergesetfeat1_notrk(P, neg_vector, in_feats, in_labels):
+    out_feats = []
+    for i in range(in_feats.shape[0]):
+        camera_id = in_labels[i, 1]
+        feat = in_feats[i] - neg_vector[camera_id]
+        feat = P[camera_id].dot(feat)
+        feat = feat / np.linalg.norm(feat, ord=2)
+        out_feats.append(feat)
+    out_feats = np.vstack(out_feats)
+    return out_feats
+
+
+def compute_P2(prb_feats, gal_feats, gal_labels, la=3.0):
+    X = gal_feats
+    neg_vector = {}
+    u_labels = np.unique(gal_labels[:, 1])
+    P = {}
+    for label in u_labels:
+        curX = gal_feats[gal_labels[:, 1] == label, :]
+        neg_vector[label] = np.mean(curX, axis=0)
+        P[label] = np.linalg.inv(
+            curX.T.dot(curX) + curX.shape[0] * la * np.eye(X.shape[1]))
+    return P, neg_vector
+
+
+def parse_bias(cameras_bias):
+    cid_bias = dict()
+    for cameras in cameras_bias.keys():
+        cameras_id = re.sub('[a-z,A-Z]', "", cameras)
+        cameras_id = int(cameras_id)
+        bias = cameras_bias[cameras]
+        cid_bias[cameras_id] = float(bias)
+    return cid_bias
+
+
+def get_dire(zone_list, cid):
+    zs, ze = zone_list[0], zone_list[-1]
+    return (zs, ze)
+
+
+def intracam_ignore(st_mask, cid_tids):
+    count = len(cid_tids)
+    for i in range(count):
+        for j in range(count):
+            if cid_tids[i][0] == cid_tids[j][0]:
+                st_mask[i, j] = 0.
+    return st_mask
+
+
+def mergesetfeat(in_feats, in_labels, in_tracks):
+    trackset = list(set(list(in_tracks)))
+    out_feats = []
+    out_labels = []
+    for track in trackset:
+        feat = np.mean(in_feats[in_tracks == track], axis=0)
+        feat = feat / np.linalg.norm(feat, ord=2)
+        label = in_labels[in_tracks == track][0]
+        out_feats.append(feat)
+        out_labels.append(label)
+    out_feats = np.vstack(out_feats)
+    out_labels = np.vstack(out_labels)
+    return out_feats, out_labels
+
+
+def mergesetfeat3(X, labels, gX, glabels, beta=0.08, knn=20, lr=0.5):
+    for i in range(0, X.shape[0]):
+        if i % 1000 == 0:
+            print('feat3:%d/%d' % (i, X.shape[0]))
+        knnX = gX[glabels[:, 1] != labels[i, 1], :]
+        sim = knnX.dot(X[i, :])
+        knnX = knnX[sim > 0, :]
+        sim = sim[sim > 0]
+        if len(sim) > 0:
+            idx = np.argsort(-sim)
+            if len(sim) > 2 * knn:
+                sim = sim[idx[:2 * knn]]
+                knnX = knnX[idx[:2 * knn], :]
+            else:
+                sim = sim[idx]
+                knnX = knnX[idx, :]
+                knn = min(knn, len(sim))
+            knn_pos_weight = np.exp((sim[:knn] - 1) / beta)
+            knn_neg_weight = np.ones(len(sim) - knn)
+            knn_pos_prob = knn_pos_weight / np.sum(knn_pos_weight)
+            knn_neg_prob = knn_neg_weight / np.sum(knn_neg_weight)
+            X[i, :] += lr * (knn_pos_prob.dot(knnX[:knn, :]) -
+                             knn_neg_prob.dot(knnX[knn:, :]))
+            X[i, :] /= np.linalg.norm(X[i, :])
+    return X
+
+
+def run_fic(prb_feats, gal_feats, prb_labels, gal_labels, la=3.0):
+    P, neg_vector = compute_P2(prb_feats, gal_feats, gal_labels, la)
+    prb_feats_new = mergesetfeat1_notrk(P, neg_vector, prb_feats, prb_labels)
+    gal_feats_new = mergesetfeat1_notrk(P, neg_vector, gal_feats, gal_labels)
+    return prb_feats_new, gal_feats_new
+
+
+def run_fac(prb_feats,
+            gal_feats,
+            prb_labels,
+            gal_labels,
+            beta=0.08,
+            knn=20,
+            lr=0.5,
+            prb_epoch=2,
+            gal_epoch=3):
+    gal_feats_new = gal_feats.copy()
+    for i in range(prb_epoch):
+        gal_feats_new = mergesetfeat3(gal_feats_new, gal_labels, gal_feats,
+                                      gal_labels, beta, knn, lr)
+    prb_feats_new = prb_feats.copy()
+    for i in range(gal_epoch):
+        prb_feats_new = mergesetfeat3(prb_feats_new, prb_labels, gal_feats_new,
+                                      gal_labels, beta, knn, lr)
+    return prb_feats_new, gal_feats_new
+
+
+def euclidean_distance(qf, gf):
+    m = qf.shape[0]
+    n = gf.shape[0]
+    dist_mat = 2 - 2 * np.matmul(qf, gf.T)
+    return dist_mat
+
+
+def find_topk(a, k, axis=-1, largest=True, sorted=True):
+    if axis is None:
+        axis_size = a.size
+    else:
+        axis_size = a.shape[axis]
+    assert 1 <= k <= axis_size
+
+    a = np.asanyarray(a)
+    if largest:
+        index_array = np.argpartition(a, axis_size - k, axis=axis)
+        topk_indices = np.take(index_array, -np.arange(k) - 1, axis=axis)
+    else:
+        index_array = np.argpartition(a, k - 1, axis=axis)
+        topk_indices = np.take(index_array, np.arange(k), axis=axis)
+    topk_values = np.take_along_axis(a, topk_indices, axis=axis)
+    if sorted:
+        sorted_indices_in_topk = np.argsort(topk_values, axis=axis)
+        if largest:
+            sorted_indices_in_topk = np.flip(sorted_indices_in_topk, axis=axis)
+        sorted_topk_values = np.take_along_axis(
+            topk_values, sorted_indices_in_topk, axis=axis)
+        sorted_topk_indices = np.take_along_axis(
+            topk_indices, sorted_indices_in_topk, axis=axis)
+        return sorted_topk_values, sorted_topk_indices
+    return topk_values, topk_indices
+
+
+def batch_numpy_topk(qf, gf, k1, N=6000):
+    m = qf.shape[0]
+    n = gf.shape[0]
+    initial_rank = []
+    for j in range(n // N + 1):
+        temp_gf = gf[j * N:j * N + N]
+        temp_qd = []
+        for i in range(m // N + 1):
+            temp_qf = qf[i * N:i * N + N]
+            temp_d = euclidean_distance(temp_qf, temp_gf)
+            temp_qd.append(temp_d)
+        temp_qd = np.concatenate(temp_qd, axis=0)
+        temp_qd = temp_qd / (np.max(temp_qd, axis=0)[0])
+        temp_qd = temp_qd.T
+        initial_rank.append(
+            find_topk(
+                temp_qd, k=k1, axis=1, largest=False, sorted=True)[1])
+    del temp_qd
+    del temp_gf
+    del temp_qf
+    del temp_d
+    initial_rank = np.concatenate(initial_rank, axis=0)
+    return initial_rank
+
+
+def batch_euclidean_distance(qf, gf, N=6000):
+    m = qf.shape[0]
+    n = gf.shape[0]
+    dist_mat = []
+    for j in range(n // N + 1):
+        temp_gf = gf[j * N:j * N + N]
+        temp_qd = []
+        for i in range(m // N + 1):
+            temp_qf = qf[i * N:i * N + N]
+            temp_d = euclidean_distance(temp_qf, temp_gf)
+            temp_qd.append(temp_d)
+        temp_qd = np.concatenate(temp_qd, axis=0)
+        temp_qd = temp_qd / (np.max(temp_qd, axis=0)[0])
+        dist_mat.append(temp_qd.T)
+    del temp_qd
+    del temp_gf
+    del temp_qf
+    del temp_d
+    dist_mat = np.concatenate(dist_mat, axis=0)
+    return dist_mat
+
+
+def batch_v(feat, R, all_num):
+    V = np.zeros((all_num, all_num), dtype=np.float32)
+    m = feat.shape[0]
+    for i in tqdm(range(m)):
+        temp_gf = feat[i].reshape(1, -1)
+        temp_qd = euclidean_distance(temp_gf, feat)
+        temp_qd = temp_qd / (np.max(temp_qd))
+        temp_qd = temp_qd.reshape(-1)
+        temp_qd = temp_qd[R[i].tolist()]
+        weight = np.exp(-temp_qd)
+        weight = weight / np.sum(weight)
+        V[i, R[i]] = weight.astype(np.float32)
+    return V
+
+
+def k_reciprocal_neigh(initial_rank, i, k1):
+    forward_k_neigh_index = initial_rank[i, :k1 + 1]
+    backward_k_neigh_index = initial_rank[forward_k_neigh_index, :k1 + 1]
+    fi = np.where(backward_k_neigh_index == i)[0]
+    return forward_k_neigh_index[fi]
+
+
+def ReRank2(probFea, galFea, k1=20, k2=6, lambda_value=0.3):
+    query_num = probFea.shape[0]
+    all_num = query_num + galFea.shape[0]
+    feat = np.concatenate((probFea, galFea), axis=0)
+
+    initial_rank = batch_numpy_topk(feat, feat, k1 + 1, N=6000)
+    del probFea
+    del galFea
+    gc.collect()  # empty memory
+    R = []
+    for i in tqdm(range(all_num)):
+        # k-reciprocal neighbors
+        k_reciprocal_index = k_reciprocal_neigh(initial_rank, i, k1)
+        k_reciprocal_expansion_index = k_reciprocal_index
+        for j in range(len(k_reciprocal_index)):
+            candidate = k_reciprocal_index[j]
+            candidate_k_reciprocal_index = k_reciprocal_neigh(
+                initial_rank, candidate, int(np.around(k1 / 2)))
+            if len(
+                    np.intersect1d(candidate_k_reciprocal_index,
+                                   k_reciprocal_index)) > 2. / 3 * len(
+                                       candidate_k_reciprocal_index):
+                k_reciprocal_expansion_index = np.append(
+                    k_reciprocal_expansion_index, candidate_k_reciprocal_index)
+        k_reciprocal_expansion_index = np.unique(k_reciprocal_expansion_index)
+        R.append(k_reciprocal_expansion_index)
+
+    gc.collect()  # empty memory
+    V = batch_v(feat, R, all_num)
+    del R
+    gc.collect()  # empty memory
+    initial_rank = initial_rank[:, :k2]
+
+    # Faster version
+    if k2 != 1:
+        V_qe = np.zeros_like(V, dtype=np.float16)
+        for i in range(all_num):
+            V_qe[i, :] = np.mean(V[initial_rank[i], :], axis=0)
+        V = V_qe
+        del V_qe
+    del initial_rank
+    gc.collect()  # empty memory
+    invIndex = []
+    for i in range(all_num):
+        invIndex.append(np.where(V[:, i] != 0)[0])
+    jaccard_dist = np.zeros((query_num, all_num), dtype=np.float32)
+    for i in tqdm(range(query_num)):
+        temp_min = np.zeros(shape=[1, all_num], dtype=np.float32)
+        indNonZero = np.where(V[i, :] != 0)[0]
+        indImages = [invIndex[ind] for ind in indNonZero]
+        for j in range(len(indNonZero)):
+            temp_min[0, indImages[j]] = temp_min[0, indImages[j]] + np.minimum(
+                V[i, indNonZero[j]], V[indImages[j], indNonZero[j]])
+        jaccard_dist[i] = 1 - temp_min / (2. - temp_min)
+    del V
+    gc.collect()  # empty memory
+    original_dist = batch_euclidean_distance(feat, feat[:query_num, :])
+    final_dist = jaccard_dist * (1 - lambda_value
+                                 ) + original_dist * lambda_value
+    del original_dist
+    del jaccard_dist
+    final_dist = final_dist[:query_num, query_num:]
+    return final_dist
+
+
+def visual_rerank(prb_feats,
+                  gal_feats,
+                  cid_tids,
+                  use_ff=False,
+                  use_rerank=False):
+    """Rerank by visual cures."""
+    gal_labels = np.array([[0, item[0]] for item in cid_tids])
+    prb_labels = gal_labels.copy()
+    if use_ff:
+        print('current use ff finetuned parameters....')
+        # Step1-1: fic. finetuned parameters: [la]
+        prb_feats, gal_feats = run_fic(prb_feats, gal_feats, prb_labels,
+                                       gal_labels, 3.0)
+        # Step1=2: fac. finetuned parameters: [beta,knn,lr,prb_epoch,gal_epoch]
+        prb_feats, gal_feats = run_fac(prb_feats, gal_feats, prb_labels,
+                                       gal_labels, 0.08, 20, 0.5, 1, 1)
+    if use_rerank:
+        print('current use rerank finetuned parameters....')
+        # Step2: k-reciprocal. finetuned parameters: [k1,k2,lambda_value]
+        sims = ReRank2(prb_feats, gal_feats, 20, 3, 0.3)
+    else:
+        sims = 1.0 - np.dot(prb_feats, gal_feats.T)
+
+    # NOTE: sims here is actually dist, the smaller the more similar
+    return 1.0 - sims
+
+
+def normalize(nparray, axis=0):
+    try:
+        from sklearn import preprocessing
+    except Exception as e:
+        raise RuntimeError(
+            'Unable to use sklearn in MTMCT in PP-Tracking, please install sklearn, for example: `pip install sklearn`'
+        )
+    nparray = preprocessing.normalize(nparray, norm='l2', axis=axis)
+    return nparray
+
+
+def get_match(cluster_labels):
+    cluster_dict = dict()
+    cluster = list()
+    for i, l in enumerate(cluster_labels):
+        if l in list(cluster_dict.keys()):
+            cluster_dict[l].append(i)
+        else:
+            cluster_dict[l] = [i]
+    for idx in cluster_dict:
+        cluster.append(cluster_dict[idx])
+    return cluster
+
+
+def get_cid_tid(cluster_labels, cid_tids):
+    cluster = list()
+    for labels in cluster_labels:
+        cid_tid_list = list()
+        for label in labels:
+            cid_tid_list.append(cid_tids[label])
+        cluster.append(cid_tid_list)
+    return cluster
+
+
+def combin_feature(cid_tid_dict, sub_cluster):
+    for sub_ct in sub_cluster:
+        if len(sub_ct) < 2: continue
+        mean_feat = np.array([cid_tid_dict[i]['mean_feat'] for i in sub_ct])
+        for i in sub_ct:
+            cid_tid_dict[i]['mean_feat'] = mean_feat.mean(axis=0)
+    return cid_tid_dict
+
+
+def combin_cluster(sub_labels, cid_tids):
+    cluster = list()
+    for sub_c_to_c in sub_labels:
+        if len(cluster) < 1:
+            cluster = sub_labels[sub_c_to_c]
+            continue
+        for c_ts in sub_labels[sub_c_to_c]:
+            is_add = False
+            for i_c, c_set in enumerate(cluster):
+                if len(set(c_ts) & set(c_set)) > 0:
+                    new_list = list(set(c_ts) | set(c_set))
+                    cluster[i_c] = new_list
+                    is_add = True
+                    break
+            if not is_add:
+                cluster.append(c_ts)
+    labels = list()
+    num_tr = 0
+    for c_ts in cluster:
+        label_list = list()
+        for c_t in c_ts:
+            label_list.append(cid_tids.index(c_t))
+            num_tr += 1
+        label_list.sort()
+        labels.append(label_list)
+    return labels, cluster
+
+
+def parse_pt_gt(mot_feature):
+    img_rects = dict()
+    for line in mot_feature:
+        fid = int(re.sub('[a-z,A-Z]', "", mot_feature[line]['frame']))
+        tid = mot_feature[line]['id']
+        rect = list(map(lambda x: int(float(x)), mot_feature[line]['bbox']))
+        if fid not in img_rects:
+            img_rects[fid] = list()
+        rect.insert(0, tid)
+        img_rects[fid].append(rect)
+    return img_rects
+
+
+# eval result
+def compare_dataframes_mtmc(gts, ts):
+    try:
+        import motmetrics as mm
+    except Exception as e:
+        raise RuntimeError(
+            'Unable to use motmetrics in MTMCT in PP-Tracking, please install motmetrics, for example: `pip install motmetrics`, see https://github.com/longcw/py-motmetrics'
+        )
+    """Compute ID-based evaluation metrics for MTMCT
+    Return:
+        df (pandas.DataFrame): Results of the evaluations in a df with only the 'idf1', 'idp', and 'idr' columns.
+    """
+    gtds = []
+    tsds = []
+    gtcams = gts['CameraId'].drop_duplicates().tolist()
+    tscams = ts['CameraId'].drop_duplicates().tolist()
+    maxFrameId = 0
+
+    for k in sorted(gtcams):
+        gtd = gts.query('CameraId == %d' % k)
+        gtd = gtd[['FrameId', 'Id', 'X', 'Y', 'Width', 'Height']]
+        # max FrameId in gtd only
+        mfid = gtd['FrameId'].max()
+        gtd['FrameId'] += maxFrameId
+        gtd = gtd.set_index(['FrameId', 'Id'])
+        gtds.append(gtd)
+
+        if k in tscams:
+            tsd = ts.query('CameraId == %d' % k)
+            tsd = tsd[['FrameId', 'Id', 'X', 'Y', 'Width', 'Height']]
+            # max FrameId among both gtd and tsd
+            mfid = max(mfid, tsd['FrameId'].max())
+            tsd['FrameId'] += maxFrameId
+            tsd = tsd.set_index(['FrameId', 'Id'])
+            tsds.append(tsd)
+
+        maxFrameId += mfid
+
+    # compute multi-camera tracking evaluation stats
+    multiCamAcc = mm.utils.compare_to_groundtruth(
+        pd.concat(gtds), pd.concat(tsds), 'iou')
+    metrics = list(mm.metrics.motchallenge_metrics)
+    metrics.extend(['num_frames', 'idfp', 'idfn', 'idtp'])
+    mh = mm.metrics.create()
+    summary = mh.compute(multiCamAcc, metrics=metrics, name='MultiCam')
+    return summary
+
+
+def get_sim_matrix(cid_tid_dict,
+                   cid_tids,
+                   use_ff=True,
+                   use_rerank=True,
+                   use_st_filter=False):
+    # Note: camera independent get_sim_matrix function,
+    # which is different from the one in camera_utils.py.
+    count = len(cid_tids)
+
+    q_arr = np.array(
+        [cid_tid_dict[cid_tids[i]]['mean_feat'] for i in range(count)])
+    g_arr = np.array(
+        [cid_tid_dict[cid_tids[i]]['mean_feat'] for i in range(count)])
+    q_arr = normalize(q_arr, axis=1)
+    g_arr = normalize(g_arr, axis=1)
+
+    st_mask = np.ones((count, count), dtype=np.float32)
+    st_mask = intracam_ignore(st_mask, cid_tids)
+
+    visual_sim_matrix = visual_rerank(
+        q_arr, g_arr, cid_tids, use_ff=use_ff, use_rerank=use_rerank)
+    visual_sim_matrix = visual_sim_matrix.astype('float32')
+
+    np.set_printoptions(precision=3)
+    sim_matrix = visual_sim_matrix * st_mask
+
+    np.fill_diagonal(sim_matrix, 0)
+    return sim_matrix
+
+
+def get_labels(cid_tid_dict,
+               cid_tids,
+               use_ff=True,
+               use_rerank=True,
+               use_st_filter=False):
+    try:
+        from sklearn.cluster import AgglomerativeClustering
+    except Exception as e:
+        raise RuntimeError(
+            'Unable to use sklearn in MTMCT in PP-Tracking, please install sklearn, for example: `pip install sklearn`'
+        )
+    # 1st cluster
+    sim_matrix = get_sim_matrix(
+        cid_tid_dict,
+        cid_tids,
+        use_ff=use_ff,
+        use_rerank=use_rerank,
+        use_st_filter=use_st_filter)
+    cluster_labels = AgglomerativeClustering(
+        n_clusters=None,
+        distance_threshold=0.5,
+        affinity='precomputed',
+        linkage='complete').fit_predict(1 - sim_matrix)
+    labels = get_match(cluster_labels)
+    sub_cluster = get_cid_tid(labels, cid_tids)
+
+    # 2nd cluster
+    cid_tid_dict_new = combin_feature(cid_tid_dict, sub_cluster)
+    sim_matrix = get_sim_matrix(
+        cid_tid_dict_new,
+        cid_tids,
+        use_ff=use_ff,
+        use_rerank=use_rerank,
+        use_st_filter=use_st_filter)
+    cluster_labels = AgglomerativeClustering(
+        n_clusters=None,
+        distance_threshold=0.9,
+        affinity='precomputed',
+        linkage='complete').fit_predict(1 - sim_matrix)
+    labels = get_match(cluster_labels)
+    sub_cluster = get_cid_tid(labels, cid_tids)
+
+    return labels
+
+
+def getData(fpath, names=None, sep='\s+|\t+|,'):
+    """ Get the necessary track data from a file handle.
+    Args:
+        fpath (str) : Original path of file reading from.
+        names (list[str]): List of column names for the data.
+        sep (str): Allowed separators regular expression string.
+    Return:
+        df (pandas.DataFrame): Data frame containing the data loaded from the
+            stream with optionally assigned column names. No index is set on the data.
+    """
+    try:
+        df = pd.read_csv(
+            fpath,
+            sep=sep,
+            index_col=None,
+            skipinitialspace=True,
+            header=None,
+            names=names,
+            engine='python')
+        return df
+
+    except Exception as e:
+        raise ValueError("Could not read input from %s. Error: %s" %
+                         (fpath, repr(e)))

pptracking/python/mot/mtmct/zone.py

@@ -0,0 +1,412 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+This code is based on https://github.com/LCFractal/AIC21-MTMC/tree/main/reid/reid-matching/tools
+
+Note: The following codes are strongly related to zone of the AIC21 test-set S06,
+    so they can only be used in S06, and can not be used for other MTMCT datasets.
+"""
+
+import os
+import cv2
+import numpy as np
+try:
+    from sklearn.cluster import AgglomerativeClustering
+except:
+    print(
+        'Warning: Unable to use MTMCT in PP-Tracking, please install sklearn, for example: `pip install sklearn`'
+    )
+    pass
+
+BBOX_B = 10 / 15
+
+
+class Zone(object):
+    def __init__(self, zone_path='datasets/zone'):
+        # 0: b 1: g 3: r 123:w
+        # w r not high speed
+        # b g high speed
+        assert zone_path != '', "Error: zone_path is not empty!"
+        zones = {}
+        for img_name in os.listdir(zone_path):
+            camnum = int(img_name.split('.')[0][-3:])
+            zone_img = cv2.imread(os.path.join(zone_path, img_name))
+            zones[camnum] = zone_img
+        self.zones = zones
+        self.current_cam = 0
+
+    def set_cam(self, cam):
+        self.current_cam = cam
+
+    def get_zone(self, bbox):
+        cx = int((bbox[0] + bbox[2]) / 2)
+        cy = int((bbox[1] + bbox[3]) / 2)
+        pix = self.zones[self.current_cam][max(cy - 1, 0), max(cx - 1, 0), :]
+        zone_num = 0
+        if pix[0] > 50 and pix[1] > 50 and pix[2] > 50:  # w
+            zone_num = 1
+        if pix[0] < 50 and pix[1] < 50 and pix[2] > 50:  # r
+            zone_num = 2
+        if pix[0] < 50 and pix[1] > 50 and pix[2] < 50:  # g
+            zone_num = 3
+        if pix[0] > 50 and pix[1] < 50 and pix[2] < 50:  # b
+            zone_num = 4
+        return zone_num
+
+    def is_ignore(self, zone_list, frame_list, cid):
+        # 0 not in any corssroad, 1 white 2 red 3 green 4 bule
+        zs, ze = zone_list[0], zone_list[-1]
+        fs, fe = frame_list[0], frame_list[-1]
+        if zs == ze:
+            # if always on one section, excluding
+            if ze in [1, 2]:
+                return 2
+            if zs != 0 and 0 in zone_list:
+                return 0
+            if fe - fs > 1500:
+                return 2
+            if fs < 2:
+                if cid in [45]:
+                    if ze in [3, 4]:
+                        return 1
+                    else:
+                        return 2
+            if fe > 1999:
+                if cid in [41]:
+                    if ze not in [3]:
+                        return 2
+                    else:
+                        return 0
+            if fs < 2 or fe > 1999:
+                if ze in [3, 4]:
+                    return 0
+            if ze in [3, 4]:
+                return 1
+            return 2
+        else:
+            # if camera section change
+            if cid in [41, 42, 43, 44, 45, 46]:
+                # come from road extension, exclusing
+                if zs == 1 and ze == 2:
+                    return 2
+                if zs == 2 and ze == 1:
+                    return 2
+            if cid in [41]:
+                # On 41 camera, no vehicle come into 42 camera
+                if (zs in [1, 2]) and ze == 4:
+                    return 2
+                if zs == 4 and (ze in [1, 2]):
+                    return 2
+            if cid in [46]:
+                # On 46 camera，no vehicle come into 45
+                if (zs in [1, 2]) and ze == 3:
+                    return 2
+                if zs == 3 and (ze in [1, 2]):
+                    return 2
+            return 0
+
+    def filter_mot(self, mot_list, cid):
+        new_mot_list = dict()
+        sub_mot_list = dict()
+        for tracklet in mot_list:
+            tracklet_dict = mot_list[tracklet]
+            frame_list = list(tracklet_dict.keys())
+            frame_list.sort()
+            zone_list = []
+            for f in frame_list:
+                zone_list.append(tracklet_dict[f]['zone'])
+            if self.is_ignore(zone_list, frame_list, cid) == 0:
+                new_mot_list[tracklet] = tracklet_dict
+            if self.is_ignore(zone_list, frame_list, cid) == 1:
+                sub_mot_list[tracklet] = tracklet_dict
+        return new_mot_list
+
+    def filter_bbox(self, mot_list, cid):
+        new_mot_list = dict()
+        yh = self.zones[cid].shape[0]
+        for tracklet in mot_list:
+            tracklet_dict = mot_list[tracklet]
+            frame_list = list(tracklet_dict.keys())
+            frame_list.sort()
+            bbox_list = []
+            for f in frame_list:
+                bbox_list.append(tracklet_dict[f]['bbox'])
+            bbox_x = [b[0] for b in bbox_list]
+            bbox_y = [b[1] for b in bbox_list]
+            bbox_w = [b[2] - b[0] for b in bbox_list]
+            bbox_h = [b[3] - b[1] for b in bbox_list]
+            new_frame_list = list()
+            if 0 in bbox_x or 0 in bbox_y:
+                b0 = [
+                    i for i, f in enumerate(frame_list)
+                    if bbox_x[i] < 5 or bbox_y[i] + bbox_h[i] > yh - 5
+                ]
+                if len(b0) == len(frame_list):
+                    if cid in [41, 42, 44, 45, 46]:
+                        continue
+                    max_w = max(bbox_w)
+                    max_h = max(bbox_h)
+                    for i, f in enumerate(frame_list):
+                        if bbox_w[i] > max_w * BBOX_B and bbox_h[
+                                i] > max_h * BBOX_B:
+                            new_frame_list.append(f)
+                else:
+                    l_i, r_i = 0, len(frame_list) - 1
+                    if len(b0) == 0:
+                        continue
+                    if b0[0] == 0:
+                        for i in range(len(b0) - 1):
+                            if b0[i] + 1 == b0[i + 1]:
+                                l_i = b0[i + 1]
+                            else:
+                                break
+                    if b0[-1] == len(frame_list) - 1:
+                        for i in range(len(b0) - 1):
+                            i = len(b0) - 1 - i
+                            if b0[i] - 1 == b0[i - 1]:
+                                r_i = b0[i - 1]
+                            else:
+                                break
+
+                    max_lw, max_lh = bbox_w[l_i], bbox_h[l_i]
+                    max_rw, max_rh = bbox_w[r_i], bbox_h[r_i]
+                    for i, f in enumerate(frame_list):
+                        if i < l_i:
+                            if bbox_w[i] > max_lw * BBOX_B and bbox_h[
+                                    i] > max_lh * BBOX_B:
+                                new_frame_list.append(f)
+                        elif i > r_i:
+                            if bbox_w[i] > max_rw * BBOX_B and bbox_h[
+                                    i] > max_rh * BBOX_B:
+                                new_frame_list.append(f)
+                        else:
+                            new_frame_list.append(f)
+                new_tracklet_dict = dict()
+                for f in new_frame_list:
+                    new_tracklet_dict[f] = tracklet_dict[f]
+                new_mot_list[tracklet] = new_tracklet_dict
+            else:
+                new_mot_list[tracklet] = tracklet_dict
+        return new_mot_list
+
+    def break_mot(self, mot_list, cid):
+        new_mot_list = dict()
+        new_num_tracklets = max(mot_list) + 1
+        for tracklet in mot_list:
+            tracklet_dict = mot_list[tracklet]
+            frame_list = list(tracklet_dict.keys())
+            frame_list.sort()
+            zone_list = []
+            back_tracklet = False
+            new_zone_f = 0
+            pre_frame = frame_list[0]
+            time_break = False
+            for f in frame_list:
+                if f - pre_frame > 100:
+                    if cid in [44, 45]:
+                        time_break = True
+                        break
+                if not cid in [41, 44, 45, 46]:
+                    break
+                pre_frame = f
+                new_zone = tracklet_dict[f]['zone']
+                if len(zone_list) > 0 and zone_list[-1] == new_zone:
+                    continue
+                if new_zone_f > 1:
+                    if len(zone_list) > 1 and new_zone in zone_list:
+                        back_tracklet = True
+                    zone_list.append(new_zone)
+                    new_zone_f = 0
+                else:
+                    new_zone_f += 1
+            if back_tracklet:
+                new_tracklet_dict = dict()
+                pre_bbox = -1
+                pre_arrow = 0
+                have_break = False
+                for f in frame_list:
+                    now_bbox = tracklet_dict[f]['bbox']
+                    if type(pre_bbox) == int:
+                        if pre_bbox == -1:
+                            pre_bbox = now_bbox
+                    now_arrow = now_bbox[0] - pre_bbox[0]
+                    if pre_arrow * now_arrow < 0 and len(
+                            new_tracklet_dict) > 15 and not have_break:
+                        new_mot_list[tracklet] = new_tracklet_dict
+                        new_tracklet_dict = dict()
+                        have_break = True
+                    if have_break:
+                        tracklet_dict[f]['id'] = new_num_tracklets
+                    new_tracklet_dict[f] = tracklet_dict[f]
+                    pre_bbox, pre_arrow = now_bbox, now_arrow
+
+                if have_break:
+                    new_mot_list[new_num_tracklets] = new_tracklet_dict
+                    new_num_tracklets += 1
+                else:
+                    new_mot_list[tracklet] = new_tracklet_dict
+            elif time_break:
+                new_tracklet_dict = dict()
+                have_break = False
+                pre_frame = frame_list[0]
+                for f in frame_list:
+                    if f - pre_frame > 100:
+                        new_mot_list[tracklet] = new_tracklet_dict
+                        new_tracklet_dict = dict()
+                        have_break = True
+                    new_tracklet_dict[f] = tracklet_dict[f]
+                    pre_frame = f
+                if have_break:
+                    new_mot_list[new_num_tracklets] = new_tracklet_dict
+                    new_num_tracklets += 1
+                else:
+                    new_mot_list[tracklet] = new_tracklet_dict
+            else:
+                new_mot_list[tracklet] = tracklet_dict
+        return new_mot_list
+
+    def intra_matching(self, mot_list, sub_mot_list):
+        sub_zone_dict = dict()
+        new_mot_list = dict()
+        new_mot_list, new_sub_mot_list = self.do_intra_matching2(mot_list,
+                                                                 sub_mot_list)
+        return new_mot_list
+
+    def do_intra_matching2(self, mot_list, sub_list):
+        new_zone_dict = dict()
+
+        def get_trac_info(tracklet1):
+            t1_f = list(tracklet1)
+            t1_f.sort()
+            t1_fs = t1_f[0]
+            t1_fe = t1_f[-1]
+            t1_zs = tracklet1[t1_fs]['zone']
+            t1_ze = tracklet1[t1_fe]['zone']
+            t1_boxs = tracklet1[t1_fs]['bbox']
+            t1_boxe = tracklet1[t1_fe]['bbox']
+            t1_boxs = [(t1_boxs[2] + t1_boxs[0]) / 2,
+                       (t1_boxs[3] + t1_boxs[1]) / 2]
+            t1_boxe = [(t1_boxe[2] + t1_boxe[0]) / 2,
+                       (t1_boxe[3] + t1_boxe[1]) / 2]
+            return t1_fs, t1_fe, t1_zs, t1_ze, t1_boxs, t1_boxe
+
+        for t1id in sub_list:
+            tracklet1 = sub_list[t1id]
+            if tracklet1 == -1:
+                continue
+            t1_fs, t1_fe, t1_zs, t1_ze, t1_boxs, t1_boxe = get_trac_info(
+                tracklet1)
+            sim_dict = dict()
+            for t2id in mot_list:
+                tracklet2 = mot_list[t2id]
+                t2_fs, t2_fe, t2_zs, t2_ze, t2_boxs, t2_boxe = get_trac_info(
+                    tracklet2)
+                if t1_ze == t2_zs:
+                    if abs(t2_fs - t1_fe) < 5 and abs(t2_boxe[0] - t1_boxs[
+                            0]) < 50 and abs(t2_boxe[1] - t1_boxs[1]) < 50:
+                        t1_feat = tracklet1[t1_fe]['feat']
+                        t2_feat = tracklet2[t2_fs]['feat']
+                        sim_dict[t2id] = np.matmul(t1_feat, t2_feat)
+                if t1_zs == t2_ze:
+                    if abs(t2_fe - t1_fs) < 5 and abs(t2_boxs[0] - t1_boxe[
+                            0]) < 50 and abs(t2_boxs[1] - t1_boxe[1]) < 50:
+                        t1_feat = tracklet1[t1_fs]['feat']
+                        t2_feat = tracklet2[t2_fe]['feat']
+                        sim_dict[t2id] = np.matmul(t1_feat, t2_feat)
+            if len(sim_dict) > 0:
+                max_sim = 0
+                max_id = 0
+                for t2id in sim_dict:
+                    if sim_dict[t2id] > max_sim:
+                        sim_dict[t2id] = max_sim
+                        max_id = t2id
+                if max_sim > 0.5:
+                    t2 = mot_list[max_id]
+                    for t1f in tracklet1:
+                        if t1f not in t2:
+                            tracklet1[t1f]['id'] = max_id
+                            t2[t1f] = tracklet1[t1f]
+                    mot_list[max_id] = t2
+                    sub_list[t1id] = -1
+        return mot_list, sub_list
+
+    def do_intra_matching(self, sub_zone_dict, sub_zone):
+        new_zone_dict = dict()
+        id_list = list(sub_zone_dict)
+        id2index = dict()
+        for index, id in enumerate(id_list):
+            id2index[id] = index
+
+        def get_trac_info(tracklet1):
+            t1_f = list(tracklet1)
+            t1_f.sort()
+            t1_fs = t1_f[0]
+            t1_fe = t1_f[-1]
+            t1_zs = tracklet1[t1_fs]['zone']
+            t1_ze = tracklet1[t1_fe]['zone']
+            t1_boxs = tracklet1[t1_fs]['bbox']
+            t1_boxe = tracklet1[t1_fe]['bbox']
+            t1_boxs = [(t1_boxs[2] + t1_boxs[0]) / 2,
+                       (t1_boxs[3] + t1_boxs[1]) / 2]
+            t1_boxe = [(t1_boxe[2] + t1_boxe[0]) / 2,
+                       (t1_boxe[3] + t1_boxe[1]) / 2]
+            return t1_fs, t1_fe, t1_zs, t1_ze, t1_boxs, t1_boxe
+
+        sim_matrix = np.zeros([len(id_list), len(id_list)])
+
+        for t1id in sub_zone_dict:
+            tracklet1 = sub_zone_dict[t1id]
+            t1_fs, t1_fe, t1_zs, t1_ze, t1_boxs, t1_boxe = get_trac_info(
+                tracklet1)
+            t1_feat = tracklet1[t1_fe]['feat']
+            for t2id in sub_zone_dict:
+                if t1id == t2id:
+                    continue
+                tracklet2 = sub_zone_dict[t2id]
+                t2_fs, t2_fe, t2_zs, t2_ze, t2_boxs, t2_boxe = get_trac_info(
+                    tracklet2)
+                if t1_zs != t1_ze and t2_ze != t2_zs or t1_fe > t2_fs:
+                    continue
+                if abs(t1_boxe[0] - t2_boxs[0]) > 50 or abs(t1_boxe[1] -
+                                                            t2_boxs[1]) > 50:
+                    continue
+                if t2_fs - t1_fe > 5:
+                    continue
+                t2_feat = tracklet2[t2_fs]['feat']
+                sim_matrix[id2index[t1id], id2index[t2id]] = np.matmul(t1_feat,
+                                                                       t2_feat)
+                sim_matrix[id2index[t2id], id2index[t1id]] = np.matmul(t1_feat,
+                                                                       t2_feat)
+        sim_matrix = 1 - sim_matrix
+        cluster_labels = AgglomerativeClustering(
+            n_clusters=None,
+            distance_threshold=0.7,
+            affinity='precomputed',
+            linkage='complete').fit_predict(sim_matrix)
+        new_zone_dict = dict()
+        label2id = dict()
+        for index, label in enumerate(cluster_labels):
+            tracklet = sub_zone_dict[id_list[index]]
+            if label not in label2id:
+                new_id = tracklet[list(tracklet)[0]]
+                new_tracklet = dict()
+            else:
+                new_id = label2id[label]
+                new_tracklet = new_zone_dict[label2id[label]]
+            for tf in tracklet:
+                tracklet[tf]['id'] = new_id
+                new_tracklet[tf] = tracklet[tf]
+            new_zone_dict[label] = new_tracklet
+
+        return new_zone_dict

pptracking/python/mot/tracker/__init__.py

@@ -0,0 +1,25 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from . import base_jde_tracker
+from . import base_sde_tracker
+from . import jde_tracker
+from . import deepsort_tracker
+from . import ocsort_tracker
+
+from .base_jde_tracker import *
+from .base_sde_tracker import *
+from .jde_tracker import *
+from .deepsort_tracker import *
+from .ocsort_tracker import *

pptracking/python/mot/tracker/base_jde_tracker.py

@@ -0,0 +1,286 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+This code is based on https://github.com/Zhongdao/Towards-Realtime-MOT/blob/master/tracker/multitracker.py
+"""
+
+import numpy as np
+from collections import defaultdict
+from collections import deque, OrderedDict
+from ..matching import jde_matching as matching
+
+__all__ = [
+    'TrackState',
+    'BaseTrack',
+    'STrack',
+    'joint_stracks',
+    'sub_stracks',
+    'remove_duplicate_stracks',
+]
+
+
+class TrackState(object):
+    New = 0
+    Tracked = 1
+    Lost = 2
+    Removed = 3
+
+
+class BaseTrack(object):
+    _count_dict = defaultdict(int)  # support single class and multi classes
+
+    track_id = 0
+    is_activated = False
+    state = TrackState.New
+
+    history = OrderedDict()
+    features = []
+    curr_feat = None
+    score = 0
+    start_frame = 0
+    frame_id = 0
+    time_since_update = 0
+
+    # multi-camera
+    location = (np.inf, np.inf)
+
+    @property
+    def end_frame(self):
+        return self.frame_id
+
+    @staticmethod
+    def next_id(cls_id):
+        BaseTrack._count_dict[cls_id] += 1
+        return BaseTrack._count_dict[cls_id]
+
+    # @even: reset track id
+    @staticmethod
+    def init_count(num_classes):
+        """
+        Initiate _count for all object classes
+        :param num_classes:
+        """
+        for cls_id in range(num_classes):
+            BaseTrack._count_dict[cls_id] = 0
+
+    @staticmethod
+    def reset_track_count(cls_id):
+        BaseTrack._count_dict[cls_id] = 0
+
+    def activate(self, *args):
+        raise NotImplementedError
+
+    def predict(self):
+        raise NotImplementedError
+
+    def update(self, *args, **kwargs):
+        raise NotImplementedError
+
+    def mark_lost(self):
+        self.state = TrackState.Lost
+
+    def mark_removed(self):
+        self.state = TrackState.Removed
+
+
+class STrack(BaseTrack):
+    def __init__(self, tlwh, score, cls_id, buff_size=30, temp_feat=None):
+        # wait activate
+        self._tlwh = np.asarray(tlwh, dtype=np.float)
+        self.score = score
+        self.cls_id = cls_id
+        self.track_len = 0
+
+        self.kalman_filter = None
+        self.mean, self.covariance = None, None
+        self.is_activated = False
+
+        self.use_reid = True if temp_feat is not None else False
+        if self.use_reid:
+            self.smooth_feat = None
+            self.update_features(temp_feat)
+            self.features = deque([], maxlen=buff_size)
+            self.alpha = 0.9
+
+    def update_features(self, feat):
+        # L2 normalizing, this function has no use for BYTETracker
+        feat /= np.linalg.norm(feat)
+        self.curr_feat = feat
+        if self.smooth_feat is None:
+            self.smooth_feat = feat
+        else:
+            self.smooth_feat = self.alpha * self.smooth_feat + (
+                1.0 - self.alpha) * feat
+        self.features.append(feat)
+        self.smooth_feat /= np.linalg.norm(self.smooth_feat)
+
+    def predict(self):
+        mean_state = self.mean.copy()
+        if self.state != TrackState.Tracked:
+            mean_state[7] = 0
+        self.mean, self.covariance = self.kalman_filter.predict(
+            mean_state, self.covariance)
+
+    @staticmethod
+    def multi_predict(tracks, kalman_filter):
+        if len(tracks) > 0:
+            multi_mean = np.asarray([track.mean.copy() for track in tracks])
+            multi_covariance = np.asarray(
+                [track.covariance for track in tracks])
+            for i, st in enumerate(tracks):
+                if st.state != TrackState.Tracked:
+                    multi_mean[i][7] = 0
+            multi_mean, multi_covariance = kalman_filter.multi_predict(
+                multi_mean, multi_covariance)
+            for i, (mean, cov) in enumerate(zip(multi_mean, multi_covariance)):
+                tracks[i].mean = mean
+                tracks[i].covariance = cov
+
+    def reset_track_id(self):
+        self.reset_track_count(self.cls_id)
+
+    def activate(self, kalman_filter, frame_id):
+        """Start a new track"""
+        self.kalman_filter = kalman_filter
+        # update track id for the object class
+        self.track_id = self.next_id(self.cls_id)
+        self.mean, self.covariance = self.kalman_filter.initiate(
+            self.tlwh_to_xyah(self._tlwh))
+
+        self.track_len = 0
+        self.state = TrackState.Tracked  # set flag 'tracked'
+
+        if frame_id == 1:  # to record the first frame's detection result
+            self.is_activated = True
+
+        self.frame_id = frame_id
+        self.start_frame = frame_id
+
+    def re_activate(self, new_track, frame_id, new_id=False):
+        self.mean, self.covariance = self.kalman_filter.update(
+            self.mean, self.covariance, self.tlwh_to_xyah(new_track.tlwh))
+        if self.use_reid:
+            self.update_features(new_track.curr_feat)
+        self.track_len = 0
+        self.state = TrackState.Tracked
+        self.is_activated = True
+        self.frame_id = frame_id
+        if new_id:  # update track id for the object class
+            self.track_id = self.next_id(self.cls_id)
+
+    def update(self, new_track, frame_id, update_feature=True):
+        self.frame_id = frame_id
+        self.track_len += 1
+
+        new_tlwh = new_track.tlwh
+        self.mean, self.covariance = self.kalman_filter.update(
+            self.mean, self.covariance, self.tlwh_to_xyah(new_tlwh))
+        self.state = TrackState.Tracked  # set flag 'tracked'
+        self.is_activated = True  # set flag 'activated'
+
+        self.score = new_track.score
+        if update_feature and self.use_reid:
+            self.update_features(new_track.curr_feat)
+
+    @property
+    def tlwh(self):
+        """Get current position in bounding box format `(top left x, top left y,
+                width, height)`.
+        """
+        if self.mean is None:
+            return self._tlwh.copy()
+
+        ret = self.mean[:4].copy()
+        ret[2] *= ret[3]
+        ret[:2] -= ret[2:] / 2
+        return ret
+
+    @property
+    def tlbr(self):
+        """Convert bounding box to format `(min x, min y, max x, max y)`, i.e.,
+        `(top left, bottom right)`.
+        """
+        ret = self.tlwh.copy()
+        ret[2:] += ret[:2]
+        return ret
+
+    @staticmethod
+    def tlwh_to_xyah(tlwh):
+        """Convert bounding box to format `(center x, center y, aspect ratio,
+        height)`, where the aspect ratio is `width / height`.
+        """
+        ret = np.asarray(tlwh).copy()
+        ret[:2] += ret[2:] / 2
+        ret[2] /= ret[3]
+        return ret
+
+    def to_xyah(self):
+        return self.tlwh_to_xyah(self.tlwh)
+
+    @staticmethod
+    def tlbr_to_tlwh(tlbr):
+        ret = np.asarray(tlbr).copy()
+        ret[2:] -= ret[:2]
+        return ret
+
+    @staticmethod
+    def tlwh_to_tlbr(tlwh):
+        ret = np.asarray(tlwh).copy()
+        ret[2:] += ret[:2]
+        return ret
+
+    def __repr__(self):
+        return 'OT_({}-{})_({}-{})'.format(self.cls_id, self.track_id,
+                                           self.start_frame, self.end_frame)
+
+
+def joint_stracks(tlista, tlistb):
+    exists = {}
+    res = []
+    for t in tlista:
+        exists[t.track_id] = 1
+        res.append(t)
+    for t in tlistb:
+        tid = t.track_id
+        if not exists.get(tid, 0):
+            exists[tid] = 1
+            res.append(t)
+    return res
+
+
+def sub_stracks(tlista, tlistb):
+    stracks = {}
+    for t in tlista:
+        stracks[t.track_id] = t
+    for t in tlistb:
+        tid = t.track_id
+        if stracks.get(tid, 0):
+            del stracks[tid]
+    return list(stracks.values())
+
+
+def remove_duplicate_stracks(stracksa, stracksb):
+    pdist = matching.iou_distance(stracksa, stracksb)
+    pairs = np.where(pdist < 0.15)
+    dupa, dupb = list(), list()
+    for p, q in zip(*pairs):
+        timep = stracksa[p].frame_id - stracksa[p].start_frame
+        timeq = stracksb[q].frame_id - stracksb[q].start_frame
+        if timep > timeq:
+            dupb.append(q)
+        else:
+            dupa.append(p)
+    resa = [t for i, t in enumerate(stracksa) if not i in dupa]
+    resb = [t for i, t in enumerate(stracksb) if not i in dupb]
+    return resa, resb

pptracking/python/mot/tracker/base_sde_tracker.py

@@ -0,0 +1,153 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+This code is based on https://github.com/nwojke/deep_sort/blob/master/deep_sort/track.py
+"""
+
+import datetime
+
+__all__ = ['TrackState', 'Track']
+
+
+class TrackState(object):
+    """
+    Enumeration type for the single target track state. Newly created tracks are
+    classified as `tentative` until enough evidence has been collected. Then,
+    the track state is changed to `confirmed`. Tracks that are no longer alive
+    are classified as `deleted` to mark them for removal from the set of active
+    tracks.
+    """
+    Tentative = 1
+    Confirmed = 2
+    Deleted = 3
+
+
+class Track(object):
+    """
+    A single target track with state space `(x, y, a, h)` and associated
+    velocities, where `(x, y)` is the center of the bounding box, `a` is the
+    aspect ratio and `h` is the height.
+
+    Args:
+        mean (ndarray): Mean vector of the initial state distribution.
+        covariance (ndarray): Covariance matrix of the initial state distribution.
+        track_id (int): A unique track identifier.
+        n_init (int): Number of consecutive detections before the track is confirmed.
+            The track state is set to `Deleted` if a miss occurs within the first
+            `n_init` frames.
+        max_age (int): The maximum number of consecutive misses before the track
+            state is set to `Deleted`.
+        cls_id (int): The category id of the tracked box.
+        score (float): The confidence score of the tracked box.
+        feature (Optional[ndarray]): Feature vector of the detection this track
+            originates from. If not None, this feature is added to the `features` cache.
+
+    Attributes:
+        hits (int): Total number of measurement updates.
+        age (int): Total number of frames since first occurance.
+        time_since_update (int): Total number of frames since last measurement
+            update.
+        state (TrackState): The current track state.
+        features (List[ndarray]): A cache of features. On each measurement update,
+            the associated feature vector is added to this list.
+    """
+
+    def __init__(self,
+                 mean,
+                 covariance,
+                 track_id,
+                 n_init,
+                 max_age,
+                 cls_id,
+                 score,
+                 feature=None):
+        self.mean = mean
+        self.covariance = covariance
+        self.track_id = track_id
+        self.hits = 1
+        self.age = 1
+        self.time_since_update = 0
+        self.cls_id = cls_id
+        self.score = score
+        self.start_time = datetime.datetime.now()
+
+        self.state = TrackState.Tentative
+        self.features = []
+        self.feat = feature
+        if feature is not None:
+            self.features.append(feature)
+
+        self._n_init = n_init
+        self._max_age = max_age
+
+    def to_tlwh(self):
+        """Get position in format `(top left x, top left y, width, height)`."""
+        ret = self.mean[:4].copy()
+        ret[2] *= ret[3]
+        ret[:2] -= ret[2:] / 2
+        return ret
+
+    def to_tlbr(self):
+        """Get position in bounding box format `(min x, miny, max x, max y)`."""
+        ret = self.to_tlwh()
+        ret[2:] = ret[:2] + ret[2:]
+        return ret
+
+    def predict(self, kalman_filter):
+        """
+        Propagate the state distribution to the current time step using a Kalman
+        filter prediction step.
+        """
+        self.mean, self.covariance = kalman_filter.predict(self.mean,
+                                                           self.covariance)
+        self.age += 1
+        self.time_since_update += 1
+
+    def update(self, kalman_filter, detection):
+        """
+        Perform Kalman filter measurement update step and update the associated
+        detection feature cache.
+        """
+        self.mean, self.covariance = kalman_filter.update(self.mean,
+                                                          self.covariance,
+                                                          detection.to_xyah())
+        self.features.append(detection.feature)
+        self.feat = detection.feature
+        self.cls_id = detection.cls_id
+        self.score = detection.score
+
+        self.hits += 1
+        self.time_since_update = 0
+        if self.state == TrackState.Tentative and self.hits >= self._n_init:
+            self.state = TrackState.Confirmed
+
+    def mark_missed(self):
+        """Mark this track as missed (no association at the current time step).
+        """
+        if self.state == TrackState.Tentative:
+            self.state = TrackState.Deleted
+        elif self.time_since_update > self._max_age:
+            self.state = TrackState.Deleted
+
+    def is_tentative(self):
+        """Returns True if this track is tentative (unconfirmed)."""
+        return self.state == TrackState.Tentative
+
+    def is_confirmed(self):
+        """Returns True if this track is confirmed."""
+        return self.state == TrackState.Confirmed
+
+    def is_deleted(self):
+        """Returns True if this track is dead and should be deleted."""
+        return self.state == TrackState.Deleted

pptracking/python/mot/tracker/deepsort_tracker.py

@@ -0,0 +1,185 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+This code is based on https://github.com/nwojke/deep_sort/blob/master/deep_sort/tracker.py
+"""
+
+import numpy as np
+
+from ..motion import KalmanFilter
+from ..matching.deepsort_matching import NearestNeighborDistanceMetric
+from ..matching.deepsort_matching import iou_cost, min_cost_matching, matching_cascade, gate_cost_matrix
+from .base_sde_tracker import Track
+from ..utils import Detection
+
+__all__ = ['DeepSORTTracker']
+
+
+class DeepSORTTracker(object):
+    """
+    DeepSORT tracker
+
+    Args:
+        input_size (list): input feature map size to reid model, [h, w] format,
+            [64, 192] as default.
+        min_box_area (int): min box area to filter out low quality boxes
+        vertical_ratio (float): w/h, the vertical ratio of the bbox to filter
+            bad results, set 1.6 default for pedestrian tracking. If set <=0
+            means no need to filter bboxes.
+        budget (int): If not None, fix samples per class to at most this number.
+            Removes the oldest samples when the budget is reached.
+        max_age (int): maximum number of missed misses before a track is deleted
+        n_init (float): Number of frames that a track remains in initialization
+            phase. Number of consecutive detections before the track is confirmed. 
+            The track state is set to `Deleted` if a miss occurs within the first 
+            `n_init` frames.
+        metric_type (str): either "euclidean" or "cosine", the distance metric 
+            used for measurement to track association.
+        matching_threshold (float): samples with larger distance are 
+            considered an invalid match.
+        max_iou_distance (float): max iou distance threshold
+        motion (object): KalmanFilter instance
+    """
+
+    def __init__(self,
+                 input_size=[64, 192],
+                 min_box_area=0,
+                 vertical_ratio=-1,
+                 budget=100,
+                 max_age=70,
+                 n_init=3,
+                 metric_type='cosine',
+                 matching_threshold=0.2,
+                 max_iou_distance=0.9,
+                 motion='KalmanFilter'):
+        self.input_size = input_size
+        self.min_box_area = min_box_area
+        self.vertical_ratio = vertical_ratio
+        self.max_age = max_age
+        self.n_init = n_init
+        self.metric = NearestNeighborDistanceMetric(metric_type,
+                                                    matching_threshold, budget)
+        self.max_iou_distance = max_iou_distance
+        if motion == 'KalmanFilter':
+            self.motion = KalmanFilter()
+
+        self.tracks = []
+        self._next_id = 1
+
+    def predict(self):
+        """
+        Propagate track state distributions one time step forward.
+        This function should be called once every time step, before `update`.
+        """
+        for track in self.tracks:
+            track.predict(self.motion)
+
+    def update(self, pred_dets, pred_embs):
+        """
+        Perform measurement update and track management.
+        Args:
+            pred_dets (np.array): Detection results of the image, the shape is
+                [N, 6], means 'cls_id, score, x0, y0, x1, y1'.
+            pred_embs (np.array): Embedding results of the image, the shape is
+                [N, 128], usually pred_embs.shape[1] is a multiple of 128.
+        """
+        pred_cls_ids = pred_dets[:, 0:1]
+        pred_scores = pred_dets[:, 1:2]
+        pred_xyxys = pred_dets[:, 2:6]
+        pred_tlwhs = np.concatenate(
+            (pred_xyxys[:, 0:2], pred_xyxys[:, 2:4] - pred_xyxys[:, 0:2] + 1),
+            axis=1)
+
+        detections = [
+            Detection(tlwh, score, feat, cls_id)
+            for tlwh, score, feat, cls_id in zip(pred_tlwhs, pred_scores,
+                                                 pred_embs, pred_cls_ids)
+        ]
+
+        # Run matching cascade.
+        matches, unmatched_tracks, unmatched_detections = \
+            self._match(detections)
+
+        # Update track set.
+        for track_idx, detection_idx in matches:
+            self.tracks[track_idx].update(self.motion,
+                                          detections[detection_idx])
+        for track_idx in unmatched_tracks:
+            self.tracks[track_idx].mark_missed()
+        for detection_idx in unmatched_detections:
+            self._initiate_track(detections[detection_idx])
+        self.tracks = [t for t in self.tracks if not t.is_deleted()]
+
+        # Update distance metric.
+        active_targets = [t.track_id for t in self.tracks if t.is_confirmed()]
+        features, targets = [], []
+        for track in self.tracks:
+            if not track.is_confirmed():
+                continue
+            features += track.features
+            targets += [track.track_id for _ in track.features]
+            track.features = []
+        self.metric.partial_fit(
+            np.asarray(features), np.asarray(targets), active_targets)
+        output_stracks = self.tracks
+        return output_stracks
+
+    def _match(self, detections):
+        def gated_metric(tracks, dets, track_indices, detection_indices):
+            features = np.array([dets[i].feature for i in detection_indices])
+            targets = np.array([tracks[i].track_id for i in track_indices])
+            cost_matrix = self.metric.distance(features, targets)
+            cost_matrix = gate_cost_matrix(self.motion, cost_matrix, tracks,
+                                           dets, track_indices,
+                                           detection_indices)
+            return cost_matrix
+
+        # Split track set into confirmed and unconfirmed tracks.
+        confirmed_tracks = [
+            i for i, t in enumerate(self.tracks) if t.is_confirmed()
+        ]
+        unconfirmed_tracks = [
+            i for i, t in enumerate(self.tracks) if not t.is_confirmed()
+        ]
+
+        # Associate confirmed tracks using appearance features.
+        matches_a, unmatched_tracks_a, unmatched_detections = \
+            matching_cascade(
+                gated_metric, self.metric.matching_threshold, self.max_age,
+                self.tracks, detections, confirmed_tracks)
+
+        # Associate remaining tracks together with unconfirmed tracks using IOU.
+        iou_track_candidates = unconfirmed_tracks + [
+            k for k in unmatched_tracks_a
+            if self.tracks[k].time_since_update == 1
+        ]
+        unmatched_tracks_a = [
+            k for k in unmatched_tracks_a
+            if self.tracks[k].time_since_update != 1
+        ]
+        matches_b, unmatched_tracks_b, unmatched_detections = \
+            min_cost_matching(
+                iou_cost, self.max_iou_distance, self.tracks,
+                detections, iou_track_candidates, unmatched_detections)
+
+        matches = matches_a + matches_b
+        unmatched_tracks = list(set(unmatched_tracks_a + unmatched_tracks_b))
+        return matches, unmatched_tracks, unmatched_detections
+
+    def _initiate_track(self, detection):
+        mean, covariance = self.motion.initiate(detection.to_xyah())
+        self.tracks.append(
+            Track(mean, covariance, self._next_id, self.n_init, self.max_age,
+                  detection.cls_id, detection.score, detection.feature))
+        self._next_id += 1

pptracking/python/mot/tracker/jde_tracker.py

@@ -0,0 +1,343 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+This code is based on https://github.com/Zhongdao/Towards-Realtime-MOT/blob/master/tracker/multitracker.py
+"""
+
+import numpy as np
+from collections import defaultdict
+
+from ..matching import jde_matching as matching
+from ..motion import KalmanFilter
+from .base_jde_tracker import TrackState, STrack
+from .base_jde_tracker import joint_stracks, sub_stracks, remove_duplicate_stracks
+
+__all__ = ['JDETracker']
+
+
+class JDETracker(object):
+    __shared__ = ['num_classes']
+    """
+    JDE tracker, support single class and multi classes
+
+    Args:
+        use_byte (bool): Whether use ByteTracker, default False
+        num_classes (int): the number of classes
+        det_thresh (float): threshold of detection score
+        track_buffer (int): buffer for tracker
+        min_box_area (int): min box area to filter out low quality boxes
+        vertical_ratio (float): w/h, the vertical ratio of the bbox to filter
+            bad results. If set <= 0 means no need to filter bboxes，usually set
+            1.6 for pedestrian tracking.
+        tracked_thresh (float): linear assignment threshold of tracked 
+            stracks and detections
+        r_tracked_thresh (float): linear assignment threshold of 
+            tracked stracks and unmatched detections
+        unconfirmed_thresh (float): linear assignment threshold of 
+            unconfirmed stracks and unmatched detections
+        conf_thres (float): confidence threshold for tracking, also used in
+            ByteTracker as higher confidence threshold
+        match_thres (float): linear assignment threshold of tracked 
+            stracks and detections in ByteTracker
+        low_conf_thres (float): lower confidence threshold for tracking in
+            ByteTracker
+        input_size (list): input feature map size to reid model, [h, w] format,
+            [64, 192] as default.
+        motion (str): motion model, KalmanFilter as default
+        metric_type (str): either "euclidean" or "cosine", the distance metric 
+            used for measurement to track association.
+    """
+
+    def __init__(self,
+                 use_byte=False,
+                 num_classes=1,
+                 det_thresh=0.3,
+                 track_buffer=30,
+                 min_box_area=0,
+                 vertical_ratio=0,
+                 tracked_thresh=0.7,
+                 r_tracked_thresh=0.5,
+                 unconfirmed_thresh=0.7,
+                 conf_thres=0,
+                 match_thres=0.8,
+                 low_conf_thres=0.2,
+                 input_size=[64, 192],
+                 motion='KalmanFilter',
+                 metric_type='euclidean'):
+        self.use_byte = use_byte
+        self.num_classes = num_classes
+        self.det_thresh = det_thresh if not use_byte else conf_thres + 0.1
+        self.track_buffer = track_buffer
+        self.min_box_area = min_box_area
+        self.vertical_ratio = vertical_ratio
+
+        self.tracked_thresh = tracked_thresh
+        self.r_tracked_thresh = r_tracked_thresh
+        self.unconfirmed_thresh = unconfirmed_thresh
+        self.conf_thres = conf_thres
+        self.match_thres = match_thres
+        self.low_conf_thres = low_conf_thres
+
+        self.input_size = input_size
+        if motion == 'KalmanFilter':
+            self.motion = KalmanFilter()
+        self.metric_type = metric_type
+
+        self.frame_id = 0
+        self.tracked_tracks_dict = defaultdict(list)  # dict(list[STrack])
+        self.lost_tracks_dict = defaultdict(list)  # dict(list[STrack])
+        self.removed_tracks_dict = defaultdict(list)  # dict(list[STrack])
+
+        self.max_time_lost = 0
+        # max_time_lost will be calculated: int(frame_rate / 30.0 * track_buffer)
+
+    def update(self, pred_dets, pred_embs=None):
+        """
+        Processes the image frame and finds bounding box(detections).
+        Associates the detection with corresponding tracklets and also handles
+            lost, removed, refound and active tracklets.
+
+        Args:
+            pred_dets (np.array): Detection results of the image, the shape is
+                [N, 6], means 'cls_id, score, x0, y0, x1, y1'.
+            pred_embs (np.array): Embedding results of the image, the shape is
+                [N, 128] or [N, 512].
+
+        Return:
+            output_stracks_dict (dict(list)): The list contains information
+                regarding the online_tracklets for the received image tensor.
+        """
+        self.frame_id += 1
+        if self.frame_id == 1:
+            STrack.init_count(self.num_classes)
+        activated_tracks_dict = defaultdict(list)
+        refined_tracks_dict = defaultdict(list)
+        lost_tracks_dict = defaultdict(list)
+        removed_tracks_dict = defaultdict(list)
+        output_tracks_dict = defaultdict(list)
+
+        pred_dets_dict = defaultdict(list)
+        pred_embs_dict = defaultdict(list)
+
+        # unify single and multi classes detection and embedding results
+        for cls_id in range(self.num_classes):
+            cls_idx = (pred_dets[:, 0:1] == cls_id).squeeze(-1)
+            pred_dets_dict[cls_id] = pred_dets[cls_idx]
+            if pred_embs is not None:
+                pred_embs_dict[cls_id] = pred_embs[cls_idx]
+            else:
+                pred_embs_dict[cls_id] = None
+
+        for cls_id in range(self.num_classes):
+            """ Step 1: Get detections by class"""
+            pred_dets_cls = pred_dets_dict[cls_id]
+            pred_embs_cls = pred_embs_dict[cls_id]
+            remain_inds = (pred_dets_cls[:, 1:2] > self.conf_thres).squeeze(-1)
+            if remain_inds.sum() > 0:
+                pred_dets_cls = pred_dets_cls[remain_inds]
+                if pred_embs_cls is None:
+                    # in original ByteTrack
+                    detections = [
+                        STrack(
+                            STrack.tlbr_to_tlwh(tlbrs[2:6]),
+                            tlbrs[1],
+                            cls_id,
+                            30,
+                            temp_feat=None) for tlbrs in pred_dets_cls
+                    ]
+                else:
+                    pred_embs_cls = pred_embs_cls[remain_inds]
+                    detections = [
+                        STrack(
+                            STrack.tlbr_to_tlwh(tlbrs[2:6]), tlbrs[1], cls_id,
+                            30, temp_feat)
+                        for (tlbrs, temp_feat
+                             ) in zip(pred_dets_cls, pred_embs_cls)
+                    ]
+            else:
+                detections = []
+            ''' Add newly detected tracklets to tracked_stracks'''
+            unconfirmed_dict = defaultdict(list)
+            tracked_tracks_dict = defaultdict(list)
+            for track in self.tracked_tracks_dict[cls_id]:
+                if not track.is_activated:
+                    # previous tracks which are not active in the current frame are added in unconfirmed list
+                    unconfirmed_dict[cls_id].append(track)
+                else:
+                    # Active tracks are added to the local list 'tracked_stracks'
+                    tracked_tracks_dict[cls_id].append(track)
+            """ Step 2: First association, with embedding"""
+            # building tracking pool for the current frame
+            track_pool_dict = defaultdict(list)
+            track_pool_dict[cls_id] = joint_stracks(
+                tracked_tracks_dict[cls_id], self.lost_tracks_dict[cls_id])
+
+            # Predict the current location with KalmanFilter
+            STrack.multi_predict(track_pool_dict[cls_id], self.motion)
+
+            if pred_embs_cls is None:
+                # in original ByteTrack
+                dists = matching.iou_distance(track_pool_dict[cls_id],
+                                              detections)
+                matches, u_track, u_detection = matching.linear_assignment(
+                    dists, thresh=self.match_thres)  # not self.tracked_thresh
+            else:
+                dists = matching.embedding_distance(
+                    track_pool_dict[cls_id],
+                    detections,
+                    metric=self.metric_type)
+                dists = matching.fuse_motion(
+                    self.motion, dists, track_pool_dict[cls_id], detections)
+                matches, u_track, u_detection = matching.linear_assignment(
+                    dists, thresh=self.tracked_thresh)
+
+            for i_tracked, idet in matches:
+                # i_tracked is the id of the track and idet is the detection
+                track = track_pool_dict[cls_id][i_tracked]
+                det = detections[idet]
+                if track.state == TrackState.Tracked:
+                    # If the track is active, add the detection to the track
+                    track.update(detections[idet], self.frame_id)
+                    activated_tracks_dict[cls_id].append(track)
+                else:
+                    # We have obtained a detection from a track which is not active,
+                    # hence put the track in refind_stracks list
+                    track.re_activate(det, self.frame_id, new_id=False)
+                    refined_tracks_dict[cls_id].append(track)
+
+            # None of the steps below happen if there are no undetected tracks.
+            """ Step 3: Second association, with IOU"""
+            if self.use_byte:
+                inds_low = pred_dets_dict[cls_id][:, 1:2] > self.low_conf_thres
+                inds_high = pred_dets_dict[cls_id][:, 1:2] < self.conf_thres
+                inds_second = np.logical_and(inds_low, inds_high).squeeze(-1)
+                pred_dets_cls_second = pred_dets_dict[cls_id][inds_second]
+
+                # association the untrack to the low score detections
+                if len(pred_dets_cls_second) > 0:
+                    if pred_embs_dict[cls_id] is None:
+                        # in original ByteTrack
+                        detections_second = [
+                            STrack(
+                                STrack.tlbr_to_tlwh(tlbrs[2:6]),
+                                tlbrs[1],
+                                cls_id,
+                                30,
+                                temp_feat=None)
+                            for tlbrs in pred_dets_cls_second
+                        ]
+                    else:
+                        pred_embs_cls_second = pred_embs_dict[cls_id][
+                            inds_second]
+                        detections_second = [
+                            STrack(
+                                STrack.tlbr_to_tlwh(tlbrs[2:6]), tlbrs[1],
+                                cls_id, 30, temp_feat)
+                            for (tlbrs, temp_feat) in zip(pred_dets_cls_second,
+                                                          pred_embs_cls_second)
+                        ]
+                else:
+                    detections_second = []
+                r_tracked_stracks = [
+                    track_pool_dict[cls_id][i] for i in u_track
+                    if track_pool_dict[cls_id][i].state == TrackState.Tracked
+                ]
+                dists = matching.iou_distance(r_tracked_stracks,
+                                              detections_second)
+                matches, u_track, u_detection_second = matching.linear_assignment(
+                    dists, thresh=0.4)  # not r_tracked_thresh
+            else:
+                detections = [detections[i] for i in u_detection]
+                r_tracked_stracks = []
+                for i in u_track:
+                    if track_pool_dict[cls_id][i].state == TrackState.Tracked:
+                        r_tracked_stracks.append(track_pool_dict[cls_id][i])
+                dists = matching.iou_distance(r_tracked_stracks, detections)
+
+                matches, u_track, u_detection = matching.linear_assignment(
+                    dists, thresh=self.r_tracked_thresh)
+
+            for i_tracked, idet in matches:
+                track = r_tracked_stracks[i_tracked]
+                det = detections[
+                    idet] if not self.use_byte else detections_second[idet]
+                if track.state == TrackState.Tracked:
+                    track.update(det, self.frame_id)
+                    activated_tracks_dict[cls_id].append(track)
+                else:
+                    track.re_activate(det, self.frame_id, new_id=False)
+                    refined_tracks_dict[cls_id].append(track)
+
+            for it in u_track:
+                track = r_tracked_stracks[it]
+                if not track.state == TrackState.Lost:
+                    track.mark_lost()
+                    lost_tracks_dict[cls_id].append(track)
+            '''Deal with unconfirmed tracks, usually tracks with only one beginning frame'''
+            detections = [detections[i] for i in u_detection]
+            dists = matching.iou_distance(unconfirmed_dict[cls_id], detections)
+            matches, u_unconfirmed, u_detection = matching.linear_assignment(
+                dists, thresh=self.unconfirmed_thresh)
+            for i_tracked, idet in matches:
+                unconfirmed_dict[cls_id][i_tracked].update(detections[idet],
+                                                           self.frame_id)
+                activated_tracks_dict[cls_id].append(unconfirmed_dict[cls_id][
+                    i_tracked])
+            for it in u_unconfirmed:
+                track = unconfirmed_dict[cls_id][it]
+                track.mark_removed()
+                removed_tracks_dict[cls_id].append(track)
+            """ Step 4: Init new stracks"""
+            for inew in u_detection:
+                track = detections[inew]
+                if track.score < self.det_thresh:
+                    continue
+                track.activate(self.motion, self.frame_id)
+                activated_tracks_dict[cls_id].append(track)
+            """ Step 5: Update state"""
+            for track in self.lost_tracks_dict[cls_id]:
+                if self.frame_id - track.end_frame > self.max_time_lost:
+                    track.mark_removed()
+                    removed_tracks_dict[cls_id].append(track)
+
+            self.tracked_tracks_dict[cls_id] = [
+                t for t in self.tracked_tracks_dict[cls_id]
+                if t.state == TrackState.Tracked
+            ]
+            self.tracked_tracks_dict[cls_id] = joint_stracks(
+                self.tracked_tracks_dict[cls_id],
+                activated_tracks_dict[cls_id])
+            self.tracked_tracks_dict[cls_id] = joint_stracks(
+                self.tracked_tracks_dict[cls_id], refined_tracks_dict[cls_id])
+            self.lost_tracks_dict[cls_id] = sub_stracks(
+                self.lost_tracks_dict[cls_id],
+                self.tracked_tracks_dict[cls_id])
+            self.lost_tracks_dict[cls_id].extend(lost_tracks_dict[cls_id])
+            self.lost_tracks_dict[cls_id] = sub_stracks(
+                self.lost_tracks_dict[cls_id],
+                self.removed_tracks_dict[cls_id])
+            self.removed_tracks_dict[cls_id].extend(removed_tracks_dict[
+                cls_id])
+            self.tracked_tracks_dict[cls_id], self.lost_tracks_dict[
+                cls_id] = remove_duplicate_stracks(
+                    self.tracked_tracks_dict[cls_id],
+                    self.lost_tracks_dict[cls_id])
+
+            # get scores of lost tracks
+            output_tracks_dict[cls_id] = [
+                track for track in self.tracked_tracks_dict[cls_id]
+                if track.is_activated
+            ]
+
+        return output_tracks_dict

pptracking/python/mot/tracker/ocsort_tracker.py

@@ -0,0 +1,366 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+This code is based on https://github.com/noahcao/OC_SORT/blob/master/trackers/ocsort_tracker/ocsort.py
+"""
+
+import numpy as np
+try:
+    from filterpy.kalman import KalmanFilter
+except:
+    print(
+        'Warning: Unable to use OC-SORT, please install filterpy, for example: `pip install filterpy`, see https://github.com/rlabbe/filterpy'
+    )
+    pass
+
+from ..matching.ocsort_matching import associate, linear_assignment, iou_batch
+
+
+def k_previous_obs(observations, cur_age, k):
+    if len(observations) == 0:
+        return [-1, -1, -1, -1, -1]
+    for i in range(k):
+        dt = k - i
+        if cur_age - dt in observations:
+            return observations[cur_age - dt]
+    max_age = max(observations.keys())
+    return observations[max_age]
+
+
+def convert_bbox_to_z(bbox):
+    """
+    Takes a bounding box in the form [x1,y1,x2,y2] and returns z in the form
+      [x,y,s,r] where x,y is the centre of the box and s is the scale/area and r is
+      the aspect ratio
+    """
+    w = bbox[2] - bbox[0]
+    h = bbox[3] - bbox[1]
+    x = bbox[0] + w / 2.
+    y = bbox[1] + h / 2.
+    s = w * h  # scale is just area
+    r = w / float(h + 1e-6)
+    return np.array([x, y, s, r]).reshape((4, 1))
+
+
+def convert_x_to_bbox(x, score=None):
+    """
+    Takes a bounding box in the centre form [x,y,s,r] and returns it in the form
+      [x1,y1,x2,y2] where x1,y1 is the top left and x2,y2 is the bottom right
+    """
+    w = np.sqrt(x[2] * x[3])
+    h = x[2] / w
+    if (score == None):
+        return np.array(
+            [x[0] - w / 2., x[1] - h / 2., x[0] + w / 2.,
+             x[1] + h / 2.]).reshape((1, 4))
+    else:
+        score = np.array([score])
+        return np.array([
+            x[0] - w / 2., x[1] - h / 2., x[0] + w / 2., x[1] + h / 2., score
+        ]).reshape((1, 5))
+
+
+def speed_direction(bbox1, bbox2):
+    cx1, cy1 = (bbox1[0] + bbox1[2]) / 2.0, (bbox1[1] + bbox1[3]) / 2.0
+    cx2, cy2 = (bbox2[0] + bbox2[2]) / 2.0, (bbox2[1] + bbox2[3]) / 2.0
+    speed = np.array([cy2 - cy1, cx2 - cx1])
+    norm = np.sqrt((cy2 - cy1)**2 + (cx2 - cx1)**2) + 1e-6
+    return speed / norm
+
+
+class KalmanBoxTracker(object):
+    """
+    This class represents the internal state of individual tracked objects observed as bbox.
+
+    Args:
+        bbox (np.array): bbox in [x1,y1,x2,y2,score] format.
+        delta_t (int): delta_t of previous observation
+    """
+    count = 0
+
+    def __init__(self, bbox, delta_t=3):
+        try:
+            from filterpy.kalman import KalmanFilter
+        except Exception as e:
+            raise RuntimeError(
+                'Unable to use OC-SORT, please install filterpy, for example: `pip install filterpy`, see https://github.com/rlabbe/filterpy'
+            )
+        self.kf = KalmanFilter(dim_x=7, dim_z=4)
+        self.kf.F = np.array([[1, 0, 0, 0, 1, 0, 0], [0, 1, 0, 0, 0, 1, 0],
+                              [0, 0, 1, 0, 0, 0, 1], [0, 0, 0, 1, 0, 0, 0],
+                              [0, 0, 0, 0, 1, 0, 0], [0, 0, 0, 0, 0, 1, 0],
+                              [0, 0, 0, 0, 0, 0, 1]])
+        self.kf.H = np.array([[1, 0, 0, 0, 0, 0, 0], [0, 1, 0, 0, 0, 0, 0],
+                              [0, 0, 1, 0, 0, 0, 0], [0, 0, 0, 1, 0, 0, 0]])
+        self.kf.R[2:, 2:] *= 10.
+        self.kf.P[4:, 4:] *= 1000.
+        # give high uncertainty to the unobservable initial velocities
+        self.kf.P *= 10.
+        self.kf.Q[-1, -1] *= 0.01
+        self.kf.Q[4:, 4:] *= 0.01
+
+        self.score = bbox[4]
+        self.kf.x[:4] = convert_bbox_to_z(bbox)
+        self.time_since_update = 0
+        self.id = KalmanBoxTracker.count
+        KalmanBoxTracker.count += 1
+        self.history = []
+        self.hits = 0
+        self.hit_streak = 0
+        self.age = 0
+        """
+        NOTE: [-1,-1,-1,-1,-1] is a compromising placeholder for non-observation status, the same for the return of 
+        function k_previous_obs. It is ugly and I do not like it. But to support generate observation array in a 
+        fast and unified way, which you would see below k_observations = np.array([k_previous_obs(...]]), let's bear it for now.
+        """
+        self.last_observation = np.array([-1, -1, -1, -1, -1])  # placeholder
+        self.observations = dict()
+        self.history_observations = []
+        self.velocity = None
+        self.delta_t = delta_t
+
+    def update(self, bbox):
+        """
+        Updates the state vector with observed bbox.
+        """
+        if bbox is not None:
+            if self.last_observation.sum() >= 0:  # no previous observation
+                previous_box = None
+                for i in range(self.delta_t):
+                    dt = self.delta_t - i
+                    if self.age - dt in self.observations:
+                        previous_box = self.observations[self.age - dt]
+                        break
+                if previous_box is None:
+                    previous_box = self.last_observation
+                """
+                  Estimate the track speed direction with observations \Delta t steps away
+                """
+                self.velocity = speed_direction(previous_box, bbox)
+            """
+              Insert new observations. This is a ugly way to maintain both self.observations
+              and self.history_observations. Bear it for the moment.
+            """
+            self.last_observation = bbox
+            self.observations[self.age] = bbox
+            self.history_observations.append(bbox)
+
+            self.time_since_update = 0
+            self.history = []
+            self.hits += 1
+            self.hit_streak += 1
+            self.kf.update(convert_bbox_to_z(bbox))
+        else:
+            self.kf.update(bbox)
+
+    def predict(self):
+        """
+        Advances the state vector and returns the predicted bounding box estimate.
+        """
+        if ((self.kf.x[6] + self.kf.x[2]) <= 0):
+            self.kf.x[6] *= 0.0
+
+        self.kf.predict()
+        self.age += 1
+        if (self.time_since_update > 0):
+            self.hit_streak = 0
+        self.time_since_update += 1
+        self.history.append(convert_x_to_bbox(self.kf.x, score=self.score))
+        return self.history[-1]
+
+    def get_state(self):
+        return convert_x_to_bbox(self.kf.x, score=self.score)
+
+
+class OCSORTTracker(object):
+    """
+    OCSORT tracker, support single class
+
+    Args:
+        det_thresh (float): threshold of detection score
+        max_age (int): maximum number of missed misses before a track is deleted
+        min_hits (int): minimum hits for associate
+        iou_threshold (float): iou threshold for associate
+        delta_t (int): delta_t of previous observation
+        inertia (float): vdc_weight of angle_diff_cost for associate
+        vertical_ratio (float): w/h, the vertical ratio of the bbox to filter
+            bad results. If set <= 0 means no need to filter bboxes，usually set
+            1.6 for pedestrian tracking.
+        min_box_area (int): min box area to filter out low quality boxes
+        use_byte (bool): Whether use ByteTracker, default False
+    """
+
+    def __init__(self,
+                 det_thresh=0.6,
+                 max_age=30,
+                 min_hits=3,
+                 iou_threshold=0.3,
+                 delta_t=3,
+                 inertia=0.2,
+                 vertical_ratio=-1,
+                 min_box_area=0,
+                 use_byte=False):
+        self.det_thresh = det_thresh
+        self.max_age = max_age
+        self.min_hits = min_hits
+        self.iou_threshold = iou_threshold
+        self.delta_t = delta_t
+        self.inertia = inertia
+        self.vertical_ratio = vertical_ratio
+        self.min_box_area = min_box_area
+        self.use_byte = use_byte
+
+        self.trackers = []
+        self.frame_count = 0
+        KalmanBoxTracker.count = 0
+
+    def update(self, pred_dets, pred_embs=None):
+        """
+        Args:
+            pred_dets (np.array): Detection results of the image, the shape is
+                [N, 6], means 'cls_id, score, x0, y0, x1, y1'.
+            pred_embs (np.array): Embedding results of the image, the shape is
+                [N, 128] or [N, 512], default as None.
+
+        Return:
+            tracking boxes (np.array): [M, 6], means 'x0, y0, x1, y1, score, id'.
+        """
+        if pred_dets is None:
+            return np.empty((0, 6))
+
+        self.frame_count += 1
+
+        bboxes = pred_dets[:, 2:]
+        scores = pred_dets[:, 1:2]
+        dets = np.concatenate((bboxes, scores), axis=1)
+        scores = scores.squeeze(-1)
+
+        inds_low = scores > 0.1
+        inds_high = scores < self.det_thresh
+        inds_second = np.logical_and(inds_low, inds_high)
+        # self.det_thresh > score > 0.1, for second matching
+        dets_second = dets[inds_second]  # detections for second matching
+        remain_inds = scores > self.det_thresh
+        dets = dets[remain_inds]
+
+        # get predicted locations from existing trackers.
+        trks = np.zeros((len(self.trackers), 5))
+        to_del = []
+        ret = []
+        for t, trk in enumerate(trks):
+            pos = self.trackers[t].predict()[0]
+            trk[:] = [pos[0], pos[1], pos[2], pos[3], 0]
+            if np.any(np.isnan(pos)):
+                to_del.append(t)
+        trks = np.ma.compress_rows(np.ma.masked_invalid(trks))
+        for t in reversed(to_del):
+            self.trackers.pop(t)
+
+        velocities = np.array([
+            trk.velocity if trk.velocity is not None else np.array((0, 0))
+            for trk in self.trackers
+        ])
+        last_boxes = np.array([trk.last_observation for trk in self.trackers])
+        k_observations = np.array([
+            k_previous_obs(trk.observations, trk.age, self.delta_t)
+            for trk in self.trackers
+        ])
+        """
+            First round of association
+        """
+        matched, unmatched_dets, unmatched_trks = associate(
+            dets, trks, self.iou_threshold, velocities, k_observations,
+            self.inertia)
+        for m in matched:
+            self.trackers[m[1]].update(dets[m[0], :])
+        """
+            Second round of associaton by OCR
+        """
+        # BYTE association
+        if self.use_byte and len(dets_second) > 0 and unmatched_trks.shape[
+                0] > 0:
+            u_trks = trks[unmatched_trks]
+            iou_left = iou_batch(
+                dets_second,
+                u_trks)  # iou between low score detections and unmatched tracks
+            iou_left = np.array(iou_left)
+            if iou_left.max() > self.iou_threshold:
+                """
+                    NOTE: by using a lower threshold, e.g., self.iou_threshold - 0.1, you may
+                    get a higher performance especially on MOT17/MOT20 datasets. But we keep it
+                    uniform here for simplicity
+                """
+                matched_indices = linear_assignment(-iou_left)
+                to_remove_trk_indices = []
+                for m in matched_indices:
+                    det_ind, trk_ind = m[0], unmatched_trks[m[1]]
+                    if iou_left[m[0], m[1]] < self.iou_threshold:
+                        continue
+                    self.trackers[trk_ind].update(dets_second[det_ind, :])
+                    to_remove_trk_indices.append(trk_ind)
+                unmatched_trks = np.setdiff1d(unmatched_trks,
+                                              np.array(to_remove_trk_indices))
+
+        if unmatched_dets.shape[0] > 0 and unmatched_trks.shape[0] > 0:
+            left_dets = dets[unmatched_dets]
+            left_trks = last_boxes[unmatched_trks]
+            iou_left = iou_batch(left_dets, left_trks)
+            iou_left = np.array(iou_left)
+            if iou_left.max() > self.iou_threshold:
+                """
+                    NOTE: by using a lower threshold, e.g., self.iou_threshold - 0.1, you may
+                    get a higher performance especially on MOT17/MOT20 datasets. But we keep it
+                    uniform here for simplicity
+                """
+                rematched_indices = linear_assignment(-iou_left)
+                to_remove_det_indices = []
+                to_remove_trk_indices = []
+                for m in rematched_indices:
+                    det_ind, trk_ind = unmatched_dets[m[0]], unmatched_trks[m[
+                        1]]
+                    if iou_left[m[0], m[1]] < self.iou_threshold:
+                        continue
+                    self.trackers[trk_ind].update(dets[det_ind, :])
+                    to_remove_det_indices.append(det_ind)
+                    to_remove_trk_indices.append(trk_ind)
+                unmatched_dets = np.setdiff1d(unmatched_dets,
+                                              np.array(to_remove_det_indices))
+                unmatched_trks = np.setdiff1d(unmatched_trks,
+                                              np.array(to_remove_trk_indices))
+
+        for m in unmatched_trks:
+            self.trackers[m].update(None)
+
+        # create and initialise new trackers for unmatched detections
+        for i in unmatched_dets:
+            trk = KalmanBoxTracker(dets[i, :], delta_t=self.delta_t)
+            self.trackers.append(trk)
+        i = len(self.trackers)
+        for trk in reversed(self.trackers):
+            if trk.last_observation.sum() < 0:
+                d = trk.get_state()[0]
+            else:
+                d = trk.last_observation  # tlbr + score
+            if (trk.time_since_update < 1) and (
+                    trk.hit_streak >= self.min_hits or
+                    self.frame_count <= self.min_hits):
+                # +1 as MOT benchmark requires positive
+                ret.append(np.concatenate((d, [trk.id + 1])).reshape(1, -1))
+            i -= 1
+            # remove dead tracklet
+            if (trk.time_since_update > self.max_age):
+                self.trackers.pop(i)
+        if (len(ret) > 0):
+            return np.concatenate(ret)
+        return np.empty((0, 6))

pptracking/python/mot/utils.py

@@ -0,0 +1,437 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import cv2
+import time
+import numpy as np
+import collections
+import math
+
+__all__ = [
+    'MOTTimer', 'Detection', 'write_mot_results', 'load_det_results',
+    'preprocess_reid', 'get_crops', 'clip_box', 'scale_coords',
+    'flow_statistic', 'update_object_info'
+]
+
+
+class MOTTimer(object):
+    """
+    This class used to compute and print the current FPS while evaling.
+    """
+
+    def __init__(self, window_size=20):
+        self.start_time = 0.
+        self.diff = 0.
+        self.duration = 0.
+        self.deque = collections.deque(maxlen=window_size)
+
+    def tic(self):
+        # using time.time instead of time.clock because time time.clock
+        # does not normalize for multithreading
+        self.start_time = time.time()
+
+    def toc(self, average=True):
+        self.diff = time.time() - self.start_time
+        self.deque.append(self.diff)
+        if average:
+            self.duration = np.mean(self.deque)
+        else:
+            self.duration = np.sum(self.deque)
+        return self.duration
+
+    def clear(self):
+        self.start_time = 0.
+        self.diff = 0.
+        self.duration = 0.
+
+
+class Detection(object):
+    """
+    This class represents a bounding box detection in a single image.
+
+    Args:
+        tlwh (Tensor): Bounding box in format `(top left x, top left y,
+            width, height)`.
+        score (Tensor): Bounding box confidence score.
+        feature (Tensor): A feature vector that describes the object 
+            contained in this image.
+        cls_id (Tensor): Bounding box category id.
+    """
+
+    def __init__(self, tlwh, score, feature, cls_id):
+        self.tlwh = np.asarray(tlwh, dtype=np.float32)
+        self.score = float(score)
+        self.feature = np.asarray(feature, dtype=np.float32)
+        self.cls_id = int(cls_id)
+
+    def to_tlbr(self):
+        """
+        Convert bounding box to format `(min x, min y, max x, max y)`, i.e.,
+        `(top left, bottom right)`.
+        """
+        ret = self.tlwh.copy()
+        ret[2:] += ret[:2]
+        return ret
+
+    def to_xyah(self):
+        """
+        Convert bounding box to format `(center x, center y, aspect ratio,
+        height)`, where the aspect ratio is `width / height`.
+        """
+        ret = self.tlwh.copy()
+        ret[:2] += ret[2:] / 2
+        ret[2] /= ret[3]
+        return ret
+
+
+def write_mot_results(filename, results, data_type='mot', num_classes=1):
+    # support single and multi classes
+    if data_type in ['mot', 'mcmot']:
+        save_format = '{frame},{id},{x1},{y1},{w},{h},{score},{cls_id},-1,-1\n'
+    elif data_type == 'kitti':
+        save_format = '{frame} {id} car 0 0 -10 {x1} {y1} {x2} {y2} -10 -10 -10 -1000 -1000 -1000 -10\n'
+    else:
+        raise ValueError(data_type)
+
+    f = open(filename, 'w')
+    for cls_id in range(num_classes):
+        for frame_id, tlwhs, tscores, track_ids in results[cls_id]:
+            if data_type == 'kitti':
+                frame_id -= 1
+            for tlwh, score, track_id in zip(tlwhs, tscores, track_ids):
+                if track_id < 0: continue
+                if data_type == 'mot':
+                    cls_id = -1
+
+                x1, y1, w, h = tlwh
+                x2, y2 = x1 + w, y1 + h
+                line = save_format.format(
+                    frame=frame_id,
+                    id=track_id,
+                    x1=x1,
+                    y1=y1,
+                    x2=x2,
+                    y2=y2,
+                    w=w,
+                    h=h,
+                    score=score,
+                    cls_id=cls_id)
+                f.write(line)
+    print('MOT results save in {}'.format(filename))
+
+
+def load_det_results(det_file, num_frames):
+    assert os.path.exists(det_file) and os.path.isfile(det_file), \
+        '{} is not exist or not a file.'.format(det_file)
+    labels = np.loadtxt(det_file, dtype='float32', delimiter=',')
+    assert labels.shape[1] == 7, \
+        "Each line of {} should have 7 items: '[frame_id],[x0],[y0],[w],[h],[score],[class_id]'.".format(det_file)
+    results_list = []
+    for frame_i in range(num_frames):
+        results = {'bbox': [], 'score': [], 'cls_id': []}
+        lables_with_frame = labels[labels[:, 0] == frame_i + 1]
+        # each line of lables_with_frame:
+        # [frame_id],[x0],[y0],[w],[h],[score],[class_id]
+        for l in lables_with_frame:
+            results['bbox'].append(l[1:5])
+            results['score'].append(l[5:6])
+            results['cls_id'].append(l[6:7])
+        results_list.append(results)
+    return results_list
+
+
+def scale_coords(coords, input_shape, im_shape, scale_factor):
+    # Note: ratio has only one value, scale_factor[0] == scale_factor[1]
+    # 
+    # This function only used for JDE YOLOv3 or other detectors with 
+    # LetterBoxResize and JDEBBoxPostProcess, coords output from detector had
+    # not scaled back to the origin image.
+
+    ratio = scale_factor[0]
+    pad_w = (input_shape[1] - int(im_shape[1])) / 2
+    pad_h = (input_shape[0] - int(im_shape[0])) / 2
+    coords[:, 0::2] -= pad_w
+    coords[:, 1::2] -= pad_h
+    coords[:, 0:4] /= ratio
+    coords[:, :4] = np.clip(coords[:, :4], a_min=0, a_max=coords[:, :4].max())
+    return coords.round()
+
+
+def clip_box(xyxy, ori_image_shape):
+    H, W = ori_image_shape
+    xyxy[:, 0::2] = np.clip(xyxy[:, 0::2], a_min=0, a_max=W)
+    xyxy[:, 1::2] = np.clip(xyxy[:, 1::2], a_min=0, a_max=H)
+    w = xyxy[:, 2:3] - xyxy[:, 0:1]
+    h = xyxy[:, 3:4] - xyxy[:, 1:2]
+    mask = np.logical_and(h > 0, w > 0)
+    keep_idx = np.nonzero(mask)
+    return xyxy[keep_idx[0]], keep_idx
+
+
+def get_crops(xyxy, ori_img, w, h):
+    crops = []
+    xyxy = xyxy.astype(np.int64)
+    ori_img = ori_img.transpose(1, 0, 2)  # [h,w,3]->[w,h,3]
+    for i, bbox in enumerate(xyxy):
+        crop = ori_img[bbox[0]:bbox[2], bbox[1]:bbox[3], :]
+        crops.append(crop)
+    crops = preprocess_reid(crops, w, h)
+    return crops
+
+
+def preprocess_reid(imgs,
+                    w=64,
+                    h=192,
+                    mean=[0.485, 0.456, 0.406],
+                    std=[0.229, 0.224, 0.225]):
+    im_batch = []
+    for img in imgs:
+        img = cv2.resize(img, (w, h))
+        img = img[:, :, ::-1].astype('float32').transpose((2, 0, 1)) / 255
+        img_mean = np.array(mean).reshape((3, 1, 1))
+        img_std = np.array(std).reshape((3, 1, 1))
+        img -= img_mean
+        img /= img_std
+        img = np.expand_dims(img, axis=0)
+        im_batch.append(img)
+    im_batch = np.concatenate(im_batch, 0)
+    return im_batch
+
+
+def flow_statistic(result,
+                   secs_interval,
+                   do_entrance_counting,
+                   do_break_in_counting,
+                   region_type,
+                   video_fps,
+                   entrance,
+                   id_set,
+                   interval_id_set,
+                   in_id_list,
+                   out_id_list,
+                   prev_center,
+                   records,
+                   data_type='mot',
+                   ids2names=['pedestrian']):
+    # Count in/out number: 
+    # Note that 'region_type' should be one of ['horizontal', 'vertical', 'custom'],
+    # 'horizontal' and 'vertical' means entrance is the center line as the entrance when do_entrance_counting, 
+    # 'custom' means entrance is a region defined by users when do_break_in_counting.
+
+    if do_entrance_counting:
+        assert region_type in [
+            'horizontal', 'vertical'
+        ], "region_type should be 'horizontal' or 'vertical' when do entrance counting."
+        entrance_x, entrance_y = entrance[0], entrance[1]
+        frame_id, tlwhs, tscores, track_ids = result
+        for tlwh, score, track_id in zip(tlwhs, tscores, track_ids):
+            if track_id < 0: continue
+            if data_type == 'kitti':
+                frame_id -= 1
+            x1, y1, w, h = tlwh
+            center_x = x1 + w / 2.
+            center_y = y1 + h / 2.
+            if track_id in prev_center:
+                if region_type == 'horizontal':
+                    # horizontal center line
+                    if prev_center[track_id][1] <= entrance_y and \
+                    center_y > entrance_y:
+                        in_id_list.append(track_id)
+                    if prev_center[track_id][1] >= entrance_y and \
+                    center_y < entrance_y:
+                        out_id_list.append(track_id)
+                else:
+                    # vertical center line
+                    if prev_center[track_id][0] <= entrance_x and \
+                    center_x > entrance_x:
+                        in_id_list.append(track_id)
+                    if prev_center[track_id][0] >= entrance_x and \
+                    center_x < entrance_x:
+                        out_id_list.append(track_id)
+                prev_center[track_id][0] = center_x
+                prev_center[track_id][1] = center_y
+            else:
+                prev_center[track_id] = [center_x, center_y]
+
+    if do_break_in_counting:
+        assert region_type in [
+            'custom'
+        ], "region_type should be 'custom' when do break_in counting."
+        assert len(
+            entrance
+        ) >= 4, "entrance should be at least 3 points and (w,h) of image when do break_in counting."
+        im_w, im_h = entrance[-1][:]
+        entrance = np.array(entrance[:-1])
+
+        frame_id, tlwhs, tscores, track_ids = result
+        for tlwh, score, track_id in zip(tlwhs, tscores, track_ids):
+            if track_id < 0: continue
+            if data_type == 'kitti':
+                frame_id -= 1
+            x1, y1, w, h = tlwh
+            center_x = min(x1 + w / 2., im_w - 1)
+            if ids2names[0] == 'pedestrian':
+                center_y = min(y1 + h, im_h - 1)
+            else:
+                center_y = min(y1 + h / 2, im_h - 1)
+
+            # counting objects in region of the first frame
+            if frame_id == 1:
+                if in_quadrangle([center_x, center_y], entrance, im_h, im_w):
+                    in_id_list.append(-1)
+                else:
+                    prev_center[track_id] = [center_x, center_y]
+            else:
+                if track_id in prev_center:
+                    if not in_quadrangle(prev_center[track_id], entrance, im_h,
+                                         im_w) and in_quadrangle(
+                                             [center_x, center_y], entrance,
+                                             im_h, im_w):
+                        in_id_list.append(track_id)
+                    prev_center[track_id] = [center_x, center_y]
+                else:
+                    prev_center[track_id] = [center_x, center_y]
+
+# Count totol number, number at a manual-setting interval
+    frame_id, tlwhs, tscores, track_ids = result
+    for tlwh, score, track_id in zip(tlwhs, tscores, track_ids):
+        if track_id < 0: continue
+        id_set.add(track_id)
+        interval_id_set.add(track_id)
+
+    # Reset counting at the interval beginning
+    if frame_id % video_fps == 0 and frame_id / video_fps % secs_interval == 0:
+        curr_interval_count = len(interval_id_set)
+        interval_id_set.clear()
+    info = "Frame id: {}, Total count: {}".format(frame_id, len(id_set))
+    if do_entrance_counting:
+        info += ", In count: {}, Out count: {}".format(
+            len(in_id_list), len(out_id_list))
+    if do_break_in_counting:
+        info += ", Break_in count: {}".format(len(in_id_list))
+    if frame_id % video_fps == 0 and frame_id / video_fps % secs_interval == 0:
+        info += ", Count during {} secs: {}".format(secs_interval,
+                                                    curr_interval_count)
+        interval_id_set.clear()
+    # print(info)
+    info += "\n"
+    records.append(info)
+
+    return {
+        "id_set": id_set,
+        "interval_id_set": interval_id_set,
+        "in_id_list": in_id_list,
+        "out_id_list": out_id_list,
+        "prev_center": prev_center,
+        "records": records,
+    }
+
+
+def distance(center_1, center_2):
+    return math.sqrt(
+        math.pow(center_1[0] - center_2[0], 2) + math.pow(center_1[1] -
+                                                          center_2[1], 2))
+
+
+# update vehicle parking info
+def update_object_info(object_in_region_info,
+                       result,
+                       region_type,
+                       entrance,
+                       fps,
+                       illegal_parking_time,
+                       distance_threshold_frame=3,
+                       distance_threshold_interval=50):
+    '''
+    For consecutive frames, the distance between two frame is smaller than distance_threshold_frame, regard as parking
+    For parking in general, the move distance should smaller than distance_threshold_interval
+    The moving distance of the vehicle is scaled according to the y, which is inversely proportional to y.
+    '''
+
+    assert region_type in [
+        'custom'
+    ], "region_type should be 'custom' when do break_in counting."
+    assert len(
+        entrance
+    ) >= 4, "entrance should be at least 3 points and (w,h) of image when do break_in counting."
+
+    frame_id, tlwhs, tscores, track_ids = result  # result from mot
+
+    im_w, im_h = entrance[-1][:]
+    entrance = np.array(entrance[:-1])
+
+    illegal_parking_dict = {}
+    for tlwh, score, track_id in zip(tlwhs, tscores, track_ids):
+        if track_id < 0: continue
+
+        x1, y1, w, h = tlwh
+        center_x = min(x1 + w / 2., im_w - 1)
+        center_y = min(y1 + h / 2, im_h - 1)
+
+        if not in_quadrangle([center_x, center_y], entrance, im_h, im_w):
+            continue
+
+        current_center = (center_x, center_y)
+        if track_id not in object_in_region_info.keys(
+        ):  # first time appear in region
+            object_in_region_info[track_id] = {}
+            object_in_region_info[track_id]["start_frame"] = frame_id
+            object_in_region_info[track_id]["end_frame"] = frame_id
+            object_in_region_info[track_id]["prev_center"] = current_center
+            object_in_region_info[track_id]["start_center"] = current_center
+        else:
+            prev_center = object_in_region_info[track_id]["prev_center"]
+
+            dis = distance(current_center, prev_center)
+            scaled_dis = 200 * dis / (
+                current_center[1] + 1)  # scale distance according to y
+            dis = scaled_dis
+
+            if dis < distance_threshold_frame:  # not move
+                object_in_region_info[track_id]["end_frame"] = frame_id
+                object_in_region_info[track_id]["prev_center"] = current_center
+            else:  # move
+                object_in_region_info[track_id]["start_frame"] = frame_id
+                object_in_region_info[track_id]["end_frame"] = frame_id
+                object_in_region_info[track_id]["prev_center"] = current_center
+                object_in_region_info[track_id][
+                    "start_center"] = current_center
+
+        # whether current object parking
+        distance_from_start = distance(
+            object_in_region_info[track_id]["start_center"], current_center)
+        if distance_from_start > distance_threshold_interval:
+            # moved
+            object_in_region_info[track_id]["start_frame"] = frame_id
+            object_in_region_info[track_id]["end_frame"] = frame_id
+            object_in_region_info[track_id]["prev_center"] = current_center
+            object_in_region_info[track_id]["start_center"] = current_center
+            continue
+
+        if (object_in_region_info[track_id]["end_frame"]-object_in_region_info[track_id]["start_frame"]) /fps >= illegal_parking_time \
+            and distance_from_start<distance_threshold_interval:
+            illegal_parking_dict[track_id] = {"bbox": [x1, y1, w, h]}
+
+    return object_in_region_info, illegal_parking_dict
+
+
+def in_quadrangle(point, entrance, im_h, im_w):
+    mask = np.zeros((im_h, im_w, 1), np.uint8)
+    cv2.fillPoly(mask, [entrance], 255)
+    p = tuple(map(int, point))
+    if mask[p[1], p[0], :] > 0:
+        return True
+    else:
+        return False

pptracking/python/mot/visualize.py

@@ -0,0 +1,379 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import division
+
+import os
+import cv2
+import numpy as np
+from PIL import Image, ImageDraw, ImageFile
+ImageFile.LOAD_TRUNCATED_IMAGES = True
+from collections import deque
+
+
+def visualize_box_mask(im, results, labels, threshold=0.5):
+    """
+    Args:
+        im (str/np.ndarray): path of image/np.ndarray read by cv2
+        results (dict): include 'boxes': np.ndarray: shape:[N,6], N: number of box,
+                        matix element:[class, score, x_min, y_min, x_max, y_max]
+        labels (list): labels:['class1', ..., 'classn']
+        threshold (float): Threshold of score.
+    Returns:
+        im (PIL.Image.Image): visualized image
+    """
+    if isinstance(im, str):
+        im = Image.open(im).convert('RGB')
+    else:
+        im = Image.fromarray(im)
+    if 'boxes' in results and len(results['boxes']) > 0:
+        im = draw_box(im, results['boxes'], labels, threshold=threshold)
+    return im
+
+
+def get_color_map_list(num_classes):
+    """
+    Args:
+        num_classes (int): number of class
+    Returns:
+        color_map (list): RGB color list
+    """
+    color_map = num_classes * [0, 0, 0]
+    for i in range(0, num_classes):
+        j = 0
+        lab = i
+        while lab:
+            color_map[i * 3] |= (((lab >> 0) & 1) << (7 - j))
+            color_map[i * 3 + 1] |= (((lab >> 1) & 1) << (7 - j))
+            color_map[i * 3 + 2] |= (((lab >> 2) & 1) << (7 - j))
+            j += 1
+            lab >>= 3
+    color_map = [color_map[i:i + 3] for i in range(0, len(color_map), 3)]
+    return color_map
+
+
+def draw_box(im, np_boxes, labels, threshold=0.5):
+    """
+    Args:
+        im (PIL.Image.Image): PIL image
+        np_boxes (np.ndarray): shape:[N,6], N: number of box,
+                               matix element:[class, score, x_min, y_min, x_max, y_max]
+        labels (list): labels:['class1', ..., 'classn']
+        threshold (float): threshold of box
+    Returns:
+        im (PIL.Image.Image): visualized image
+    """
+    draw_thickness = min(im.size) // 320
+    draw = ImageDraw.Draw(im)
+    clsid2color = {}
+    color_list = get_color_map_list(len(labels))
+    expect_boxes = (np_boxes[:, 1] > threshold) & (np_boxes[:, 0] > -1)
+    np_boxes = np_boxes[expect_boxes, :]
+
+    for dt in np_boxes:
+        clsid, bbox, score = int(dt[0]), dt[2:], dt[1]
+        if clsid not in clsid2color:
+            clsid2color[clsid] = color_list[clsid]
+        color = tuple(clsid2color[clsid])
+
+        if len(bbox) == 4:
+            xmin, ymin, xmax, ymax = bbox
+            print('class_id:{:d}, confidence:{:.4f}, left_top:[{:.2f},{:.2f}],'
+                  'right_bottom:[{:.2f},{:.2f}]'.format(
+                      int(clsid), score, xmin, ymin, xmax, ymax))
+            # draw bbox
+            draw.line(
+                [(xmin, ymin), (xmin, ymax), (xmax, ymax), (xmax, ymin),
+                 (xmin, ymin)],
+                width=draw_thickness,
+                fill=color)
+        elif len(bbox) == 8:
+            x1, y1, x2, y2, x3, y3, x4, y4 = bbox
+            draw.line(
+                [(x1, y1), (x2, y2), (x3, y3), (x4, y4), (x1, y1)],
+                width=2,
+                fill=color)
+            xmin = min(x1, x2, x3, x4)
+            ymin = min(y1, y2, y3, y4)
+
+        # draw label
+        text = "{} {:.4f}".format(labels[clsid], score)
+        tw, th = draw.textsize(text)
+        draw.rectangle(
+            [(xmin + 1, ymin - th), (xmin + tw + 1, ymin)], fill=color)
+        draw.text((xmin + 1, ymin - th), text, fill=(255, 255, 255))
+    return im
+
+
+def get_color(idx):
+    idx = idx * 3
+    color = ((37 * idx) % 255, (17 * idx) % 255, (29 * idx) % 255)
+    return color
+
+
+def plot_tracking(image,
+                  tlwhs,
+                  obj_ids,
+                  scores=None,
+                  frame_id=0,
+                  fps=0.,
+                  ids2names=[],
+                  do_entrance_counting=False,
+                  entrance=None):
+    im = np.ascontiguousarray(np.copy(image))
+    im_h, im_w = im.shape[:2]
+
+    text_scale = max(0.5, image.shape[1] / 3000.)
+    text_thickness = 2
+    line_thickness = max(1, int(image.shape[1] / 500.))
+
+    cv2.putText(
+        im,
+        'frame: %d fps: %.2f num: %d' % (frame_id, fps, len(tlwhs)),
+        (0, int(15 * text_scale) + 5),
+        cv2.FONT_ITALIC,
+        text_scale, (0, 0, 255),
+        thickness=text_thickness)
+    for i, tlwh in enumerate(tlwhs):
+        x1, y1, w, h = tlwh
+        intbox = tuple(map(int, (x1, y1, x1 + w, y1 + h)))
+        obj_id = int(obj_ids[i])
+        id_text = 'ID: {}'.format(int(obj_id))
+        if ids2names != []:
+            assert len(
+                ids2names) == 1, "plot_tracking only supports single classes."
+            id_text = 'ID: {}_'.format(ids2names[0]) + id_text
+        _line_thickness = 1 if obj_id <= 0 else line_thickness
+        color = get_color(abs(obj_id))
+        cv2.rectangle(
+            im,
+            intbox[0:2],
+            intbox[2:4],
+            color=color,
+            thickness=line_thickness)
+        cv2.putText(
+            im,
+            id_text, (intbox[0], intbox[1] - 25),
+            cv2.FONT_ITALIC,
+            text_scale, (0, 255, 255),
+            thickness=text_thickness)
+
+        if scores is not None:
+            text = 'score: {:.2f}'.format(float(scores[i]))
+            cv2.putText(
+                im,
+                text, (intbox[0], intbox[1] - 6),
+                cv2.FONT_ITALIC,
+                text_scale, (0, 255, 0),
+                thickness=text_thickness)
+    if do_entrance_counting:
+        entrance_line = tuple(map(int, entrance))
+        cv2.rectangle(
+            im,
+            entrance_line[0:2],
+            entrance_line[2:4],
+            color=(0, 255, 255),
+            thickness=line_thickness)
+    return im
+
+
+def plot_tracking_dict(image,
+                       num_classes,
+                       tlwhs_dict,
+                       obj_ids_dict,
+                       scores_dict,
+                       frame_id=0,
+                       fps=0.,
+                       ids2names=[],
+                       do_entrance_counting=False,
+                       do_break_in_counting=False,
+                       do_illegal_parking_recognition=False,
+                       illegal_parking_dict=None,
+                       entrance=None,
+                       records=None,
+                       center_traj=None):
+    im = np.ascontiguousarray(np.copy(image))
+    im_h, im_w = im.shape[:2]
+    if do_break_in_counting or do_illegal_parking_recognition:
+        entrance = np.array(entrance[:-1])  # last pair is [im_w, im_h] 
+
+    text_scale = max(0.5, image.shape[1] / 3000.)
+    text_thickness = 2
+    line_thickness = max(1, int(image.shape[1] / 500.))
+
+    if num_classes == 1:
+        if records is not None:
+            start = records[-1].find('Total')
+            end = records[-1].find('In')
+            cv2.putText(
+                im,
+                records[-1][start:end], (0, int(40 * text_scale) + 10),
+                cv2.FONT_ITALIC,
+                text_scale, (0, 0, 255),
+                thickness=text_thickness)
+
+    if num_classes == 1 and do_entrance_counting:
+        entrance_line = tuple(map(int, entrance))
+        cv2.rectangle(
+            im,
+            entrance_line[0:2],
+            entrance_line[2:4],
+            color=(0, 255, 255),
+            thickness=line_thickness)
+        # find start location for entrance counting data
+        start = records[-1].find('In')
+        cv2.putText(
+            im,
+            records[-1][start:-1], (0, int(60 * text_scale) + 10),
+            cv2.FONT_ITALIC,
+            text_scale, (0, 0, 255),
+            thickness=text_thickness)
+
+    if num_classes == 1 and (do_break_in_counting or
+                             do_illegal_parking_recognition):
+        np_masks = np.zeros((im_h, im_w, 1), np.uint8)
+        cv2.fillPoly(np_masks, [entrance], 255)
+
+        # Draw region mask
+        alpha = 0.3
+        im = np.array(im).astype('float32')
+        mask = np_masks[:, :, 0]
+        color_mask = [0, 0, 255]
+        idx = np.nonzero(mask)
+        color_mask = np.array(color_mask)
+        im[idx[0], idx[1], :] *= 1.0 - alpha
+        im[idx[0], idx[1], :] += alpha * color_mask
+        im = np.array(im).astype('uint8')
+
+        if do_break_in_counting:
+            # find start location for break in counting data
+            start = records[-1].find('Break_in')
+            cv2.putText(
+                im,
+                records[-1][start:-1],
+                (entrance[0][0] - 10, entrance[0][1] - 10),
+                cv2.FONT_ITALIC,
+                text_scale, (0, 0, 255),
+                thickness=text_thickness)
+
+        if illegal_parking_dict is not None and len(illegal_parking_dict) != 0:
+            for key, value in illegal_parking_dict.items():
+                x1, y1, w, h = value['bbox']
+                plate = value['plate']
+                if plate is None:
+                    plate = ""
+
+                # red box
+                cv2.rectangle(im, (int(x1), int(y1)),
+                              (int(x1 + w), int(y1 + h)), (0, 0, 255), 2)
+
+                cv2.putText(
+                    im,
+                    "illegal_parking:" + plate,
+                    (int(x1) + 5, int(16 * text_scale + y1 + 15)),
+                    cv2.FONT_ITALIC,
+                    text_scale * 1.5, (0, 0, 255),
+                    thickness=text_thickness)
+
+    for cls_id in range(num_classes):
+        tlwhs = tlwhs_dict[cls_id]
+        obj_ids = obj_ids_dict[cls_id]
+        scores = scores_dict[cls_id]
+        cv2.putText(
+            im,
+            'frame: %d fps: %.2f num: %d' % (frame_id, fps, len(tlwhs)),
+            (0, int(15 * text_scale) + 5),
+            cv2.FONT_ITALIC,
+            text_scale, (0, 0, 255),
+            thickness=text_thickness)
+
+        record_id = set()
+        for i, tlwh in enumerate(tlwhs):
+            x1, y1, w, h = tlwh
+            intbox = tuple(map(int, (x1, y1, x1 + w, y1 + h)))
+            center = tuple(map(int, (x1 + w / 2., y1 + h / 2.)))
+            obj_id = int(obj_ids[i])
+            if center_traj is not None:
+                record_id.add(obj_id)
+                if obj_id not in center_traj[cls_id]:
+                    center_traj[cls_id][obj_id] = deque(maxlen=30)
+                center_traj[cls_id][obj_id].append(center)
+
+            id_text = '{}'.format(int(obj_id))
+            if ids2names != []:
+                id_text = '{}_{}'.format(ids2names[cls_id], id_text)
+            else:
+                id_text = 'class{}_{}'.format(cls_id, id_text)
+
+            _line_thickness = 1 if obj_id <= 0 else line_thickness
+
+            in_region = False
+            if do_break_in_counting:
+                center_x = min(x1 + w / 2., im_w - 1)
+                center_down_y = min(y1 + h, im_h - 1)
+                if in_quadrangle([center_x, center_down_y], entrance, im_h,
+                                 im_w):
+                    in_region = True
+
+            color = get_color(abs(obj_id)) if in_region == False else (0, 0,
+                                                                       255)
+            cv2.rectangle(
+                im,
+                intbox[0:2],
+                intbox[2:4],
+                color=color,
+                thickness=line_thickness)
+            cv2.putText(
+                im,
+                id_text, (intbox[0], intbox[1] - 25),
+                cv2.FONT_ITALIC,
+                text_scale,
+                color,
+                thickness=text_thickness)
+
+            if do_break_in_counting and in_region:
+                cv2.putText(
+                    im,
+                    'Break in now.', (intbox[0], intbox[1] - 50),
+                    cv2.FONT_ITALIC,
+                    text_scale, (0, 0, 255),
+                    thickness=text_thickness)
+
+            if scores is not None:
+                text = 'score: {:.2f}'.format(float(scores[i]))
+                cv2.putText(
+                    im,
+                    text, (intbox[0], intbox[1] - 6),
+                    cv2.FONT_ITALIC,
+                    text_scale,
+                    color,
+                    thickness=text_thickness)
+        if center_traj is not None:
+            for traj in center_traj:
+                for i in traj.keys():
+                    if i not in record_id:
+                        continue
+                    for point in traj[i]:
+                        cv2.circle(im, point, 3, (0, 0, 255), -1)
+    return im
+
+
+def in_quadrangle(point, entrance, im_h, im_w):
+    mask = np.zeros((im_h, im_w, 1), np.uint8)
+    cv2.fillPoly(mask, [entrance], 255)
+    p = tuple(map(int, point))
+    if mask[p[1], p[0], :] > 0:
+        return True
+    else:
+        return False

pptracking/python/mot_jde_infer.py

@@ -0,0 +1,508 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import time
+import yaml
+import cv2
+import numpy as np
+from collections import defaultdict
+import paddle
+
+from benchmark_utils import PaddleInferBenchmark
+from preprocess import decode_image
+from mot_utils import argsparser, Timer, get_current_memory_mb
+from det_infer import Detector, get_test_images, print_arguments, bench_log, PredictConfig
+
+# add python path
+import sys
+parent_path = os.path.abspath(os.path.join(__file__, *(['..'] * 2)))
+sys.path.insert(0, parent_path)
+
+from mot import JDETracker
+from mot.utils import MOTTimer, write_mot_results, flow_statistic
+from mot.visualize import plot_tracking, plot_tracking_dict
+
+# Global dictionary
+MOT_JDE_SUPPORT_MODELS = {
+    'JDE',
+    'FairMOT',
+}
+
+
+class JDE_Detector(Detector):
+    """
+    Args:
+        model_dir (str): root path of model.pdiparams, model.pdmodel and infer_cfg.yml
+        device (str): Choose the device you want to run, it can be: CPU/GPU/XPU, default is CPU
+        run_mode (str): mode of running(paddle/trt_fp32/trt_fp16)
+        batch_size (int): size of pre batch in inference
+        trt_min_shape (int): min shape for dynamic shape in trt
+        trt_max_shape (int): max shape for dynamic shape in trt
+        trt_opt_shape (int): opt shape for dynamic shape in trt
+        trt_calib_mode (bool): If the model is produced by TRT offline quantitative
+            calibration, trt_calib_mode need to set True
+        cpu_threads (int): cpu threads
+        enable_mkldnn (bool): whether to open MKLDNN
+        output_dir (string): The path of output, default as 'output'
+        threshold (float): Score threshold of the detected bbox, default as 0.5
+        save_images (bool): Whether to save visualization image results, default as False
+        save_mot_txts (bool): Whether to save tracking results (txt), default as False
+        draw_center_traj (bool): Whether drawing the trajectory of center, default as False
+        secs_interval (int): The seconds interval to count after tracking, default as 10
+        skip_frame_num (int): Skip frame num to get faster MOT results, default as -1
+        do_entrance_counting(bool): Whether counting the numbers of identifiers entering 
+            or getting out from the entrance, default as False，only support single class
+            counting in MOT.
+        do_break_in_counting(bool): Whether counting the numbers of identifiers break in
+            the area, default as False，only support single class counting in MOT,
+            and the video should be taken by a static camera.
+        region_type (str): Area type for entrance counting or break in counting, 'horizontal'
+            and 'vertical' used when do entrance counting. 'custom' used when do break in counting. 
+            Note that only support single-class MOT, and the video should be taken by a static camera.
+        region_polygon (list): Clockwise point coords (x0,y0,x1,y1...) of polygon of area when
+            do_break_in_counting. Note that only support single-class MOT and
+            the video should be taken by a static camera.
+    """
+
+    def __init__(self,
+                 model_dir,
+                 tracker_config=None,
+                 device='CPU',
+                 run_mode='paddle',
+                 batch_size=1,
+                 trt_min_shape=1,
+                 trt_max_shape=1088,
+                 trt_opt_shape=608,
+                 trt_calib_mode=False,
+                 cpu_threads=1,
+                 enable_mkldnn=False,
+                 output_dir='output',
+                 threshold=0.5,
+                 save_images=False,
+                 save_mot_txts=False,
+                 draw_center_traj=False,
+                 secs_interval=10,
+                 skip_frame_num=-1,
+                 do_entrance_counting=False,
+                 do_break_in_counting=False,
+                 region_type='horizontal',
+                 region_polygon=[]):
+        super(JDE_Detector, self).__init__(
+            model_dir=model_dir,
+            device=device,
+            run_mode=run_mode,
+            batch_size=batch_size,
+            trt_min_shape=trt_min_shape,
+            trt_max_shape=trt_max_shape,
+            trt_opt_shape=trt_opt_shape,
+            trt_calib_mode=trt_calib_mode,
+            cpu_threads=cpu_threads,
+            enable_mkldnn=enable_mkldnn,
+            output_dir=output_dir,
+            threshold=threshold, )
+        self.save_images = save_images
+        self.save_mot_txts = save_mot_txts
+        self.draw_center_traj = draw_center_traj
+        self.secs_interval = secs_interval
+        self.skip_frame_num = skip_frame_num
+        self.do_entrance_counting = do_entrance_counting
+        self.do_break_in_counting = do_break_in_counting
+        self.region_type = region_type
+        self.region_polygon = region_polygon
+        if self.region_type == 'custom':
+            assert len(
+                self.region_polygon
+            ) > 6, 'region_type is custom, region_polygon should be at least 3 pairs of point coords.'
+
+        assert batch_size == 1, "MOT model only supports batch_size=1."
+        self.det_times = Timer(with_tracker=True)
+        self.num_classes = len(self.pred_config.labels)
+        if self.skip_frame_num > 1:
+            self.previous_det_result = None
+
+        # tracker config
+        assert self.pred_config.tracker, "The exported JDE Detector model should have tracker."
+        cfg = self.pred_config.tracker
+        min_box_area = cfg.get('min_box_area', 0.0)
+        vertical_ratio = cfg.get('vertical_ratio', 0.0)
+        conf_thres = cfg.get('conf_thres', 0.0)
+        tracked_thresh = cfg.get('tracked_thresh', 0.7)
+        metric_type = cfg.get('metric_type', 'euclidean')
+
+        self.tracker = JDETracker(
+            num_classes=self.num_classes,
+            min_box_area=min_box_area,
+            vertical_ratio=vertical_ratio,
+            conf_thres=conf_thres,
+            tracked_thresh=tracked_thresh,
+            metric_type=metric_type)
+
+    def postprocess(self, inputs, result):
+        # postprocess output of predictor
+        np_boxes = result['pred_dets']
+        if np_boxes.shape[0] <= 0:
+            print('[WARNNING] No object detected.')
+            result = {'pred_dets': np.zeros([0, 6]), 'pred_embs': None}
+        result = {k: v for k, v in result.items() if v is not None}
+        return result
+
+    def tracking(self, det_results):
+        pred_dets = det_results['pred_dets']  # cls_id, score, x0, y0, x1, y1
+        pred_embs = det_results['pred_embs']
+        online_targets_dict = self.tracker.update(pred_dets, pred_embs)
+
+        online_tlwhs = defaultdict(list)
+        online_scores = defaultdict(list)
+        online_ids = defaultdict(list)
+        for cls_id in range(self.num_classes):
+            online_targets = online_targets_dict[cls_id]
+            for t in online_targets:
+                tlwh = t.tlwh
+                tid = t.track_id
+                tscore = t.score
+                if tlwh[2] * tlwh[3] <= self.tracker.min_box_area: continue
+                if self.tracker.vertical_ratio > 0 and tlwh[2] / tlwh[
+                        3] > self.tracker.vertical_ratio:
+                    continue
+                online_tlwhs[cls_id].append(tlwh)
+                online_ids[cls_id].append(tid)
+                online_scores[cls_id].append(tscore)
+        return online_tlwhs, online_scores, online_ids
+
+    def predict(self, repeats=1):
+        '''
+        Args:
+            repeats (int): repeats number for prediction
+        Returns:
+            result (dict): include 'pred_dets': np.ndarray: shape:[N,6], N: number of box,
+                            matix element:[class, score, x_min, y_min, x_max, y_max]
+                            FairMOT(JDE)'s result include 'pred_embs': np.ndarray:
+                            shape: [N, 128]
+        '''
+        # model prediction
+        np_pred_dets, np_pred_embs = None, None
+        for i in range(repeats):
+            self.predictor.run()
+            output_names = self.predictor.get_output_names()
+            boxes_tensor = self.predictor.get_output_handle(output_names[0])
+            np_pred_dets = boxes_tensor.copy_to_cpu()
+            embs_tensor = self.predictor.get_output_handle(output_names[1])
+            np_pred_embs = embs_tensor.copy_to_cpu()
+
+        result = dict(pred_dets=np_pred_dets, pred_embs=np_pred_embs)
+        return result
+
+    def predict_image(self,
+                      image_list,
+                      run_benchmark=False,
+                      repeats=1,
+                      visual=True,
+                      seq_name=None,
+                      reuse_det_result=False):
+        mot_results = []
+        num_classes = self.num_classes
+        image_list.sort()
+        ids2names = self.pred_config.labels
+        data_type = 'mcmot' if num_classes > 1 else 'mot'
+        for frame_id, img_file in enumerate(image_list):
+            batch_image_list = [img_file]  # bs=1 in MOT model
+            if run_benchmark:
+                # preprocess
+                inputs = self.preprocess(batch_image_list)  # warmup
+                self.det_times.preprocess_time_s.start()
+                inputs = self.preprocess(batch_image_list)
+                self.det_times.preprocess_time_s.end()
+
+                # model prediction
+                result_warmup = self.predict(repeats=repeats)  # warmup
+                self.det_times.inference_time_s.start()
+                result = self.predict(repeats=repeats)
+                self.det_times.inference_time_s.end(repeats=repeats)
+
+                # postprocess
+                result_warmup = self.postprocess(inputs, result)  # warmup
+                self.det_times.postprocess_time_s.start()
+                det_result = self.postprocess(inputs, result)
+                self.det_times.postprocess_time_s.end()
+
+                # tracking
+                result_warmup = self.tracking(det_result)
+                self.det_times.tracking_time_s.start()
+                online_tlwhs, online_scores, online_ids = self.tracking(
+                    det_result)
+                self.det_times.tracking_time_s.end()
+                self.det_times.img_num += 1
+
+                cm, gm, gu = get_current_memory_mb()
+                self.cpu_mem += cm
+                self.gpu_mem += gm
+                self.gpu_util += gu
+
+            else:
+                self.det_times.preprocess_time_s.start()
+                if not reuse_det_result:
+                    inputs = self.preprocess(batch_image_list)
+                self.det_times.preprocess_time_s.end()
+
+                self.det_times.inference_time_s.start()
+                if not reuse_det_result:
+                    result = self.predict()
+                self.det_times.inference_time_s.end()
+
+                self.det_times.postprocess_time_s.start()
+                if not reuse_det_result:
+                    det_result = self.postprocess(inputs, result)
+                    self.previous_det_result = det_result
+                else:
+                    assert self.previous_det_result is not None
+                    det_result = self.previous_det_result
+                self.det_times.postprocess_time_s.end()
+
+                # tracking process
+                self.det_times.tracking_time_s.start()
+                online_tlwhs, online_scores, online_ids = self.tracking(
+                    det_result)
+                self.det_times.tracking_time_s.end()
+                self.det_times.img_num += 1
+
+            if visual:
+                if len(image_list) > 1 and frame_id % 10 == 0:
+                    print('Tracking frame {}'.format(frame_id))
+                frame, _ = decode_image(img_file, {})
+
+                im = plot_tracking_dict(
+                    frame,
+                    num_classes,
+                    online_tlwhs,
+                    online_ids,
+                    online_scores,
+                    frame_id=frame_id,
+                    ids2names=ids2names)
+                if seq_name is None:
+                    seq_name = image_list[0].split('/')[-2]
+                save_dir = os.path.join(self.output_dir, seq_name)
+                if not os.path.exists(save_dir):
+                    os.makedirs(save_dir)
+                cv2.imwrite(
+                    os.path.join(save_dir, '{:05d}.jpg'.format(frame_id)), im)
+
+            mot_results.append([online_tlwhs, online_scores, online_ids])
+        return mot_results
+
+    def predict_video(self, video_file, camera_id):
+        video_out_name = 'mot_output.mp4'
+        if camera_id != -1:
+            capture = cv2.VideoCapture(camera_id)
+        else:
+            capture = cv2.VideoCapture(video_file)
+            video_out_name = os.path.split(video_file)[-1]
+        # Get Video info : resolution, fps, frame count
+        width = int(capture.get(cv2.CAP_PROP_FRAME_WIDTH))
+        height = int(capture.get(cv2.CAP_PROP_FRAME_HEIGHT))
+        fps = int(capture.get(cv2.CAP_PROP_FPS))
+        frame_count = int(capture.get(cv2.CAP_PROP_FRAME_COUNT))
+        print("fps: %d, frame_count: %d" % (fps, frame_count))
+
+        if not os.path.exists(self.output_dir):
+            os.makedirs(self.output_dir)
+        out_path = os.path.join(self.output_dir, video_out_name)
+        video_format = 'mp4v'
+        fourcc = cv2.VideoWriter_fourcc(*video_format)
+        writer = cv2.VideoWriter(out_path, fourcc, fps, (width, height))
+
+        frame_id = 0
+        timer = MOTTimer()
+        results = defaultdict(list)  # support single class and multi classes
+        num_classes = self.num_classes
+        data_type = 'mcmot' if num_classes > 1 else 'mot'
+        ids2names = self.pred_config.labels
+
+        center_traj = None
+        entrance = None
+        records = None
+        if self.draw_center_traj:
+            center_traj = [{} for i in range(num_classes)]
+        if num_classes == 1:
+            id_set = set()
+            interval_id_set = set()
+            in_id_list = list()
+            out_id_list = list()
+            prev_center = dict()
+            records = list()
+            if self.do_entrance_counting or self.do_break_in_counting:
+                if self.region_type == 'horizontal':
+                    entrance = [0, height / 2., width, height / 2.]
+                elif self.region_type == 'vertical':
+                    entrance = [width / 2, 0., width / 2, height]
+                elif self.region_type == 'custom':
+                    entrance = []
+                    assert len(
+                        self.region_polygon
+                    ) % 2 == 0, "region_polygon should be pairs of coords points when do break_in counting."
+                    for i in range(0, len(self.region_polygon), 2):
+                        entrance.append([
+                            self.region_polygon[i], self.region_polygon[i + 1]
+                        ])
+                    entrance.append([width, height])
+                else:
+                    raise ValueError("region_type:{} is not supported.".format(
+                        self.region_type))
+
+        video_fps = fps
+
+        while (1):
+            ret, frame = capture.read()
+            if not ret:
+                break
+            if frame_id % 10 == 0:
+                print('Tracking frame: %d' % (frame_id))
+
+            timer.tic()
+            mot_skip_frame_num = self.skip_frame_num
+            reuse_det_result = False
+            if mot_skip_frame_num > 1 and frame_id > 0 and frame_id % mot_skip_frame_num > 0:
+                reuse_det_result = True
+            seq_name = video_out_name.split('.')[0]
+            mot_results = self.predict_image(
+                [frame],
+                visual=False,
+                seq_name=seq_name,
+                reuse_det_result=reuse_det_result)
+            timer.toc()
+
+            online_tlwhs, online_scores, online_ids = mot_results[0]
+            for cls_id in range(num_classes):
+                results[cls_id].append(
+                    (frame_id + 1, online_tlwhs[cls_id], online_scores[cls_id],
+                     online_ids[cls_id]))
+
+            # NOTE: just implement flow statistic for single class
+            if num_classes == 1:
+                result = (frame_id + 1, online_tlwhs[0], online_scores[0],
+                          online_ids[0])
+                statistic = flow_statistic(
+                    result,
+                    self.secs_interval,
+                    self.do_entrance_counting,
+                    self.do_break_in_counting,
+                    self.region_type,
+                    video_fps,
+                    entrance,
+                    id_set,
+                    interval_id_set,
+                    in_id_list,
+                    out_id_list,
+                    prev_center,
+                    records,
+                    data_type,
+                    ids2names=self.pred_config.labels)
+                records = statistic['records']
+
+            fps = 1. / timer.duration
+            im = plot_tracking_dict(
+                frame,
+                num_classes,
+                online_tlwhs,
+                online_ids,
+                online_scores,
+                frame_id=frame_id,
+                fps=fps,
+                ids2names=ids2names,
+                do_entrance_counting=self.do_entrance_counting,
+                entrance=entrance,
+                records=records,
+                center_traj=center_traj)
+
+            writer.write(im)
+            if camera_id != -1:
+                cv2.imshow('Mask Detection', im)
+                if cv2.waitKey(1) & 0xFF == ord('q'):
+                    break
+            frame_id += 1
+
+        if self.save_mot_txts:
+            result_filename = os.path.join(
+                self.output_dir, video_out_name.split('.')[-2] + '.txt')
+
+            write_mot_results(result_filename, results, data_type, num_classes)
+
+            if num_classes == 1:
+                result_filename = os.path.join(
+                    self.output_dir,
+                    video_out_name.split('.')[-2] + '_flow_statistic.txt')
+                f = open(result_filename, 'w')
+                for line in records:
+                    f.write(line)
+                print('Flow statistic save in {}'.format(result_filename))
+                f.close()
+
+        writer.release()
+
+
+def main():
+    detector = JDE_Detector(
+        FLAGS.model_dir,
+        tracker_config=None,
+        device=FLAGS.device,
+        run_mode=FLAGS.run_mode,
+        batch_size=1,
+        trt_min_shape=FLAGS.trt_min_shape,
+        trt_max_shape=FLAGS.trt_max_shape,
+        trt_opt_shape=FLAGS.trt_opt_shape,
+        trt_calib_mode=FLAGS.trt_calib_mode,
+        cpu_threads=FLAGS.cpu_threads,
+        enable_mkldnn=FLAGS.enable_mkldnn,
+        output_dir=FLAGS.output_dir,
+        threshold=FLAGS.threshold,
+        save_images=FLAGS.save_images,
+        save_mot_txts=FLAGS.save_mot_txts,
+        draw_center_traj=FLAGS.draw_center_traj,
+        secs_interval=FLAGS.secs_interval,
+        skip_frame_num=FLAGS.skip_frame_num,
+        do_entrance_counting=FLAGS.do_entrance_counting,
+        do_break_in_counting=FLAGS.do_break_in_counting,
+        region_type=FLAGS.region_type,
+        region_polygon=FLAGS.region_polygon)
+
+    # predict from video file or camera video stream
+    if FLAGS.video_file is not None or FLAGS.camera_id != -1:
+        detector.predict_video(FLAGS.video_file, FLAGS.camera_id)
+    else:
+        # predict from image
+        img_list = get_test_images(FLAGS.image_dir, FLAGS.image_file)
+        detector.predict_image(img_list, FLAGS.run_benchmark, repeats=10)
+
+        if not FLAGS.run_benchmark:
+            detector.det_times.info(average=True)
+        else:
+            mode = FLAGS.run_mode
+            model_dir = FLAGS.model_dir
+            model_info = {
+                'model_name': model_dir.strip('/').split('/')[-1],
+                'precision': mode.split('_')[-1]
+            }
+            bench_log(detector, img_list, model_info, name='MOT')
+
+
+if __name__ == '__main__':
+    paddle.enable_static()
+    parser = argsparser()
+    FLAGS = parser.parse_args()
+    print_arguments(FLAGS)
+    FLAGS.device = FLAGS.device.upper()
+    assert FLAGS.device in ['CPU', 'GPU', 'XPU'
+                            ], "device should be CPU, GPU or XPU"
+
+    main()

pptracking/python/mot_sde_infer.py

@@ -0,0 +1,882 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import time
+import yaml
+import cv2
+import re
+import glob
+import numpy as np
+from collections import defaultdict
+import paddle
+
+from benchmark_utils import PaddleInferBenchmark
+from preprocess import decode_image
+
+# add python path
+import sys
+parent_path = os.path.abspath(os.path.join(__file__, *(['..'])))
+sys.path.insert(0, parent_path)
+
+from det_infer import Detector, get_test_images, print_arguments, bench_log, PredictConfig, load_predictor
+from mot_utils import argsparser, Timer, get_current_memory_mb, video2frames, _is_valid_video
+from mot.tracker import JDETracker, DeepSORTTracker, OCSORTTracker
+from mot.utils import MOTTimer, write_mot_results, get_crops, clip_box, flow_statistic
+from mot.visualize import plot_tracking, plot_tracking_dict
+
+from mot.mtmct.utils import parse_bias
+from mot.mtmct.postprocess import trajectory_fusion, sub_cluster, gen_res, print_mtmct_result
+from mot.mtmct.postprocess import get_mtmct_matching_results, save_mtmct_crops, save_mtmct_vis_results
+
+
+class SDE_Detector(Detector):
+    """
+    Args:
+        model_dir (str): root path of model.pdiparams, model.pdmodel and infer_cfg.yml
+        tracker_config (str): tracker config path
+        device (str): Choose the device you want to run, it can be: CPU/GPU/XPU, default is CPU
+        run_mode (str): mode of running(paddle/trt_fp32/trt_fp16)
+        batch_size (int): size of pre batch in inference
+        trt_min_shape (int): min shape for dynamic shape in trt
+        trt_max_shape (int): max shape for dynamic shape in trt
+        trt_opt_shape (int): opt shape for dynamic shape in trt
+        trt_calib_mode (bool): If the model is produced by TRT offline quantitative
+            calibration, trt_calib_mode need to set True
+        cpu_threads (int): cpu threads
+        enable_mkldnn (bool): whether to open MKLDNN
+        output_dir (string): The path of output, default as 'output'
+        threshold (float): Score threshold of the detected bbox, default as 0.5
+        save_images (bool): Whether to save visualization image results, default as False
+        save_mot_txts (bool): Whether to save tracking results (txt), default as False
+        draw_center_traj (bool): Whether drawing the trajectory of center, default as False
+        secs_interval (int): The seconds interval to count after tracking, default as 10
+        skip_frame_num (int): Skip frame num to get faster MOT results, default as -1
+        do_entrance_counting(bool): Whether counting the numbers of identifiers entering 
+            or getting out from the entrance, default as False，only support single class
+            counting in MOT, and the video should be taken by a static camera.
+        do_break_in_counting(bool): Whether counting the numbers of identifiers break in
+            the area, default as False，only support single class counting in MOT,
+            and the video should be taken by a static camera.
+        region_type (str): Area type for entrance counting or break in counting, 'horizontal'
+            and 'vertical' used when do entrance counting. 'custom' used when do break in counting. 
+            Note that only support single-class MOT, and the video should be taken by a static camera.
+        region_polygon (list): Clockwise point coords (x0,y0,x1,y1...) of polygon of area when
+            do_break_in_counting. Note that only support single-class MOT and
+            the video should be taken by a static camera.
+        reid_model_dir (str): reid model dir, default None for ByteTrack, but set for DeepSORT
+        mtmct_dir (str): MTMCT dir, default None, set for doing MTMCT
+    """
+
+    def __init__(self,
+                 model_dir,
+                 tracker_config,
+                 device='CPU',
+                 run_mode='paddle',
+                 batch_size=1,
+                 trt_min_shape=1,
+                 trt_max_shape=1280,
+                 trt_opt_shape=640,
+                 trt_calib_mode=False,
+                 cpu_threads=1,
+                 enable_mkldnn=False,
+                 output_dir='output',
+                 threshold=0.5,
+                 save_images=False,
+                 save_mot_txts=False,
+                 draw_center_traj=False,
+                 secs_interval=10,
+                 skip_frame_num=-1,
+                 do_entrance_counting=False,
+                 do_break_in_counting=False,
+                 region_type='horizontal',
+                 region_polygon=[],
+                 reid_model_dir=None,
+                 mtmct_dir=None):
+        super(SDE_Detector, self).__init__(
+            model_dir=model_dir,
+            device=device,
+            run_mode=run_mode,
+            batch_size=batch_size,
+            trt_min_shape=trt_min_shape,
+            trt_max_shape=trt_max_shape,
+            trt_opt_shape=trt_opt_shape,
+            trt_calib_mode=trt_calib_mode,
+            cpu_threads=cpu_threads,
+            enable_mkldnn=enable_mkldnn,
+            output_dir=output_dir,
+            threshold=threshold, )
+        self.save_images = save_images
+        self.save_mot_txts = save_mot_txts
+        self.draw_center_traj = draw_center_traj
+        self.secs_interval = secs_interval
+        self.skip_frame_num = skip_frame_num
+        self.do_entrance_counting = do_entrance_counting
+        self.do_break_in_counting = do_break_in_counting
+        self.region_type = region_type
+        self.region_polygon = region_polygon
+        if self.region_type == 'custom':
+            assert len(
+                self.region_polygon
+            ) > 6, 'region_type is custom, region_polygon should be at least 3 pairs of point coords.'
+
+        assert batch_size == 1, "MOT model only supports batch_size=1."
+        self.det_times = Timer(with_tracker=True)
+        self.num_classes = len(self.pred_config.labels)
+        if self.skip_frame_num > 1:
+            self.previous_det_result = None
+
+        # reid config
+        self.use_reid = False if reid_model_dir is None else True
+        if self.use_reid:
+            self.reid_pred_config = self.set_config(reid_model_dir)
+            self.reid_predictor, self.config = load_predictor(
+                reid_model_dir,
+                run_mode=run_mode,
+                batch_size=50,  # reid_batch_size
+                min_subgraph_size=self.reid_pred_config.min_subgraph_size,
+                device=device,
+                use_dynamic_shape=self.reid_pred_config.use_dynamic_shape,
+                trt_min_shape=trt_min_shape,
+                trt_max_shape=trt_max_shape,
+                trt_opt_shape=trt_opt_shape,
+                trt_calib_mode=trt_calib_mode,
+                cpu_threads=cpu_threads,
+                enable_mkldnn=enable_mkldnn)
+        else:
+            self.reid_pred_config = None
+            self.reid_predictor = None
+
+        assert tracker_config is not None, 'Note that tracker_config should be set.'
+        self.tracker_config = tracker_config
+        tracker_cfg = yaml.safe_load(open(self.tracker_config))
+        cfg = tracker_cfg[tracker_cfg['type']]
+
+        # tracker config
+        self.use_deepsort_tracker = True if tracker_cfg[
+            'type'] == 'DeepSORTTracker' else False
+        self.use_ocsort_tracker = True if tracker_cfg[
+            'type'] == 'OCSORTTracker' else False
+
+        if self.use_deepsort_tracker:
+            if self.reid_pred_config is not None and hasattr(
+                    self.reid_pred_config, 'tracker'):
+                cfg = self.reid_pred_config.tracker
+            budget = cfg.get('budget', 100)
+            max_age = cfg.get('max_age', 30)
+            max_iou_distance = cfg.get('max_iou_distance', 0.7)
+            matching_threshold = cfg.get('matching_threshold', 0.2)
+            min_box_area = cfg.get('min_box_area', 0)
+            vertical_ratio = cfg.get('vertical_ratio', 0)
+
+            self.tracker = DeepSORTTracker(
+                budget=budget,
+                max_age=max_age,
+                max_iou_distance=max_iou_distance,
+                matching_threshold=matching_threshold,
+                min_box_area=min_box_area,
+                vertical_ratio=vertical_ratio, )
+
+        elif self.use_ocsort_tracker:
+            det_thresh = cfg.get('det_thresh', 0.4)
+            max_age = cfg.get('max_age', 30)
+            min_hits = cfg.get('min_hits', 3)
+            iou_threshold = cfg.get('iou_threshold', 0.3)
+            delta_t = cfg.get('delta_t', 3)
+            inertia = cfg.get('inertia', 0.2)
+            min_box_area = cfg.get('min_box_area', 0)
+            vertical_ratio = cfg.get('vertical_ratio', 0)
+            use_byte = cfg.get('use_byte', False)
+
+            self.tracker = OCSORTTracker(
+                det_thresh=det_thresh,
+                max_age=max_age,
+                min_hits=min_hits,
+                iou_threshold=iou_threshold,
+                delta_t=delta_t,
+                inertia=inertia,
+                min_box_area=min_box_area,
+                vertical_ratio=vertical_ratio,
+                use_byte=use_byte)
+        else:
+            # use ByteTracker
+            use_byte = cfg.get('use_byte', False)
+            det_thresh = cfg.get('det_thresh', 0.3)
+            min_box_area = cfg.get('min_box_area', 0)
+            vertical_ratio = cfg.get('vertical_ratio', 0)
+            match_thres = cfg.get('match_thres', 0.9)
+            conf_thres = cfg.get('conf_thres', 0.6)
+            low_conf_thres = cfg.get('low_conf_thres', 0.1)
+
+            self.tracker = JDETracker(
+                use_byte=use_byte,
+                det_thresh=det_thresh,
+                num_classes=self.num_classes,
+                min_box_area=min_box_area,
+                vertical_ratio=vertical_ratio,
+                match_thres=match_thres,
+                conf_thres=conf_thres,
+                low_conf_thres=low_conf_thres, )
+
+        self.do_mtmct = False if mtmct_dir is None else True
+        self.mtmct_dir = mtmct_dir
+
+    def postprocess(self, inputs, result):
+        # postprocess output of predictor
+        keep_idx = result['boxes'][:, 1] > self.threshold
+        result['boxes'] = result['boxes'][keep_idx]
+        np_boxes_num = [len(result['boxes'])]
+        if np_boxes_num[0] <= 0:
+            print('[WARNNING] No object detected.')
+            result = {'boxes': np.zeros([0, 6]), 'boxes_num': [0]}
+        result = {k: v for k, v in result.items() if v is not None}
+        return result
+
+    def reidprocess(self, det_results, repeats=1):
+        pred_dets = det_results['boxes']  # cls_id, score, x0, y0, x1, y1
+        pred_xyxys = pred_dets[:, 2:6]
+
+        ori_image = det_results['ori_image']
+        ori_image_shape = ori_image.shape[:2]
+        pred_xyxys, keep_idx = clip_box(pred_xyxys, ori_image_shape)
+
+        if len(keep_idx[0]) == 0:
+            det_results['boxes'] = np.zeros((1, 6), dtype=np.float32)
+            det_results['embeddings'] = None
+            return det_results
+
+        pred_dets = pred_dets[keep_idx[0]]
+        pred_xyxys = pred_dets[:, 2:6]
+
+        w, h = self.tracker.input_size
+        crops = get_crops(pred_xyxys, ori_image, w, h)
+
+        # to keep fast speed, only use topk crops
+        crops = crops[:50]  # reid_batch_size
+        det_results['crops'] = np.array(crops).astype('float32')
+        det_results['boxes'] = pred_dets[:50]
+
+        input_names = self.reid_predictor.get_input_names()
+        for i in range(len(input_names)):
+            input_tensor = self.reid_predictor.get_input_handle(input_names[i])
+            input_tensor.copy_from_cpu(det_results[input_names[i]])
+
+        # model prediction
+        for i in range(repeats):
+            self.reid_predictor.run()
+            output_names = self.reid_predictor.get_output_names()
+            feature_tensor = self.reid_predictor.get_output_handle(
+                output_names[0])
+            pred_embs = feature_tensor.copy_to_cpu()
+
+        det_results['embeddings'] = pred_embs
+        return det_results
+
+    def tracking(self, det_results):
+        pred_dets = det_results['boxes']  # cls_id, score, x0, y0, x1, y1
+        pred_embs = det_results.get('embeddings', None)
+
+        if self.use_deepsort_tracker:
+            # use DeepSORTTracker, only support singe class
+            self.tracker.predict()
+            online_targets = self.tracker.update(pred_dets, pred_embs)
+            online_tlwhs, online_scores, online_ids = [], [], []
+            if self.do_mtmct:
+                online_tlbrs, online_feats = [], []
+            for t in online_targets:
+                if not t.is_confirmed() or t.time_since_update > 1:
+                    continue
+                tlwh = t.to_tlwh()
+                tscore = t.score
+                tid = t.track_id
+                if self.tracker.vertical_ratio > 0 and tlwh[2] / tlwh[
+                        3] > self.tracker.vertical_ratio:
+                    continue
+                online_tlwhs.append(tlwh)
+                online_scores.append(tscore)
+                online_ids.append(tid)
+                if self.do_mtmct:
+                    online_tlbrs.append(t.to_tlbr())
+                    online_feats.append(t.feat)
+
+            tracking_outs = {
+                'online_tlwhs': online_tlwhs,
+                'online_scores': online_scores,
+                'online_ids': online_ids,
+            }
+            if self.do_mtmct:
+                seq_name = det_results['seq_name']
+                frame_id = det_results['frame_id']
+
+                tracking_outs['feat_data'] = {}
+                for _tlbr, _id, _feat in zip(online_tlbrs, online_ids,
+                                             online_feats):
+                    feat_data = {}
+                    feat_data['bbox'] = _tlbr
+                    feat_data['frame'] = f"{frame_id:06d}"
+                    feat_data['id'] = _id
+                    _imgname = f'{seq_name}_{_id}_{frame_id}.jpg'
+                    feat_data['imgname'] = _imgname
+                    feat_data['feat'] = _feat
+                    tracking_outs['feat_data'].update({_imgname: feat_data})
+            return tracking_outs
+
+        elif self.use_ocsort_tracker:
+            # use OCSORTTracker, only support singe class
+            online_targets = self.tracker.update(pred_dets, pred_embs)
+            online_tlwhs = defaultdict(list)
+            online_scores = defaultdict(list)
+            online_ids = defaultdict(list)
+            for t in online_targets:
+                tlwh = [t[0], t[1], t[2] - t[0], t[3] - t[1]]
+                tscore = float(t[4])
+                tid = int(t[5])
+                if tlwh[2] * tlwh[3] <= self.tracker.min_box_area: continue
+                if self.tracker.vertical_ratio > 0 and tlwh[2] / tlwh[
+                        3] > self.tracker.vertical_ratio:
+                    continue
+                if tlwh[2] * tlwh[3] > 0:
+                    online_tlwhs[0].append(tlwh)
+                    online_ids[0].append(tid)
+                    online_scores[0].append(tscore)
+            tracking_outs = {
+                'online_tlwhs': online_tlwhs,
+                'online_scores': online_scores,
+                'online_ids': online_ids,
+            }
+            return tracking_outs
+
+        else:
+            # use ByteTracker, support multiple class
+            online_tlwhs = defaultdict(list)
+            online_scores = defaultdict(list)
+            online_ids = defaultdict(list)
+            if self.do_mtmct:
+                online_tlbrs, online_feats = defaultdict(list), defaultdict(
+                    list)
+            online_targets_dict = self.tracker.update(pred_dets, pred_embs)
+            for cls_id in range(self.num_classes):
+                online_targets = online_targets_dict[cls_id]
+                for t in online_targets:
+                    tlwh = t.tlwh
+                    tid = t.track_id
+                    tscore = t.score
+                    if tlwh[2] * tlwh[3] <= self.tracker.min_box_area:
+                        continue
+                    if self.tracker.vertical_ratio > 0 and tlwh[2] / tlwh[
+                            3] > self.tracker.vertical_ratio:
+                        continue
+                    online_tlwhs[cls_id].append(tlwh)
+                    online_ids[cls_id].append(tid)
+                    online_scores[cls_id].append(tscore)
+                    if self.do_mtmct:
+                        online_tlbrs[cls_id].append(t.tlbr)
+                        online_feats[cls_id].append(t.curr_feat)
+
+            if self.do_mtmct:
+                assert self.num_classes == 1, 'MTMCT only support single class.'
+                tracking_outs = {
+                    'online_tlwhs': online_tlwhs[0],
+                    'online_scores': online_scores[0],
+                    'online_ids': online_ids[0],
+                }
+                seq_name = det_results['seq_name']
+                frame_id = det_results['frame_id']
+                tracking_outs['feat_data'] = {}
+                for _tlbr, _id, _feat in zip(online_tlbrs[0], online_ids[0],
+                                             online_feats[0]):
+                    feat_data = {}
+                    feat_data['bbox'] = _tlbr
+                    feat_data['frame'] = f"{frame_id:06d}"
+                    feat_data['id'] = _id
+                    _imgname = f'{seq_name}_{_id}_{frame_id}.jpg'
+                    feat_data['imgname'] = _imgname
+                    feat_data['feat'] = _feat
+                    tracking_outs['feat_data'].update({_imgname: feat_data})
+                return tracking_outs
+
+            else:
+                tracking_outs = {
+                    'online_tlwhs': online_tlwhs,
+                    'online_scores': online_scores,
+                    'online_ids': online_ids,
+                }
+                return tracking_outs
+
+    def predict_image(self,
+                      image_list,
+                      run_benchmark=False,
+                      repeats=1,
+                      visual=True,
+                      seq_name=None,
+                      reuse_det_result=False):
+        num_classes = self.num_classes
+        image_list.sort()
+        ids2names = self.pred_config.labels
+        if self.do_mtmct:
+            mot_features_dict = {}  # cid_tid_fid feats
+        else:
+            mot_results = []
+        for frame_id, img_file in enumerate(image_list):
+            if self.do_mtmct:
+                if frame_id % 10 == 0:
+                    print('Tracking frame: %d' % (frame_id))
+            batch_image_list = [img_file]  # bs=1 in MOT model
+            frame, _ = decode_image(img_file, {})
+            if run_benchmark:
+                # preprocess
+                inputs = self.preprocess(batch_image_list)  # warmup
+                self.det_times.preprocess_time_s.start()
+                inputs = self.preprocess(batch_image_list)
+                self.det_times.preprocess_time_s.end()
+
+                # model prediction
+                result_warmup = self.predict(repeats=repeats)  # warmup
+                self.det_times.inference_time_s.start()
+                result = self.predict(repeats=repeats)
+                self.det_times.inference_time_s.end(repeats=repeats)
+
+                # postprocess
+                result_warmup = self.postprocess(inputs, result)  # warmup
+                self.det_times.postprocess_time_s.start()
+                det_result = self.postprocess(inputs, result)
+                self.det_times.postprocess_time_s.end()
+
+                # tracking
+                if self.use_reid:
+                    det_result['frame_id'] = frame_id
+                    det_result['seq_name'] = seq_name
+                    det_result['ori_image'] = frame
+                    det_result = self.reidprocess(det_result)
+                result_warmup = self.tracking(det_result)
+                self.det_times.tracking_time_s.start()
+                if self.use_reid:
+                    det_result = self.reidprocess(det_result)
+                tracking_outs = self.tracking(det_result)
+                self.det_times.tracking_time_s.end()
+                self.det_times.img_num += 1
+
+                cm, gm, gu = get_current_memory_mb()
+                self.cpu_mem += cm
+                self.gpu_mem += gm
+                self.gpu_util += gu
+
+            else:
+                self.det_times.preprocess_time_s.start()
+                if not reuse_det_result:
+                    inputs = self.preprocess(batch_image_list)
+                self.det_times.preprocess_time_s.end()
+
+                self.det_times.inference_time_s.start()
+                if not reuse_det_result:
+                    result = self.predict()
+                self.det_times.inference_time_s.end()
+
+                self.det_times.postprocess_time_s.start()
+                if not reuse_det_result:
+                    det_result = self.postprocess(inputs, result)
+                    self.previous_det_result = det_result
+                else:
+                    assert self.previous_det_result is not None
+                    det_result = self.previous_det_result
+                self.det_times.postprocess_time_s.end()
+
+                # tracking process
+                self.det_times.tracking_time_s.start()
+                if self.use_reid:
+                    det_result['frame_id'] = frame_id
+                    det_result['seq_name'] = seq_name
+                    det_result['ori_image'] = frame
+                    det_result = self.reidprocess(det_result)
+                tracking_outs = self.tracking(det_result)
+                self.det_times.tracking_time_s.end()
+                self.det_times.img_num += 1
+
+            online_tlwhs = tracking_outs['online_tlwhs']
+            online_scores = tracking_outs['online_scores']
+            online_ids = tracking_outs['online_ids']
+
+            if self.do_mtmct:
+                feat_data_dict = tracking_outs['feat_data']
+                mot_features_dict = dict(mot_features_dict, **feat_data_dict)
+            else:
+                mot_results.append([online_tlwhs, online_scores, online_ids])
+
+            if visual:
+                if len(image_list) > 1 and frame_id % 10 == 0:
+                    print('Tracking frame {}'.format(frame_id))
+                frame, _ = decode_image(img_file, {})
+                if isinstance(online_tlwhs, defaultdict):
+                    im = plot_tracking_dict(
+                        frame,
+                        num_classes,
+                        online_tlwhs,
+                        online_ids,
+                        online_scores,
+                        frame_id=frame_id,
+                        ids2names=ids2names)
+                else:
+                    im = plot_tracking(
+                        frame,
+                        online_tlwhs,
+                        online_ids,
+                        online_scores,
+                        frame_id=frame_id,
+                        ids2names=ids2names)
+                save_dir = os.path.join(self.output_dir, seq_name)
+                if not os.path.exists(save_dir):
+                    os.makedirs(save_dir)
+                cv2.imwrite(
+                    os.path.join(save_dir, '{:05d}.jpg'.format(frame_id)), im)
+
+        if self.do_mtmct:
+            return mot_features_dict
+        else:
+            return mot_results
+
+    def predict_video(self, video_file, camera_id):
+        video_out_name = 'output.mp4'
+        if camera_id != -1:
+            capture = cv2.VideoCapture(camera_id)
+        else:
+            capture = cv2.VideoCapture(video_file)
+            video_out_name = os.path.split(video_file)[-1]
+        # Get Video info : resolution, fps, frame count
+        width = int(capture.get(cv2.CAP_PROP_FRAME_WIDTH))
+        height = int(capture.get(cv2.CAP_PROP_FRAME_HEIGHT))
+        fps = int(capture.get(cv2.CAP_PROP_FPS))
+        frame_count = int(capture.get(cv2.CAP_PROP_FRAME_COUNT))
+        print("fps: %d, frame_count: %d" % (fps, frame_count))
+
+        if not os.path.exists(self.output_dir):
+            os.makedirs(self.output_dir)
+        out_path = os.path.join(self.output_dir, video_out_name)
+        video_format = 'mp4v'
+        fourcc = cv2.VideoWriter_fourcc(*video_format)
+        writer = cv2.VideoWriter(out_path, fourcc, fps, (width, height))
+
+        frame_id = 0
+        timer = MOTTimer()
+        results = defaultdict(list)
+        num_classes = self.num_classes
+        data_type = 'mcmot' if num_classes > 1 else 'mot'
+        ids2names = self.pred_config.labels
+
+        center_traj = None
+        entrance = None
+        records = None
+        if self.draw_center_traj:
+            center_traj = [{} for i in range(num_classes)]
+        if num_classes == 1:
+            id_set = set()
+            interval_id_set = set()
+            in_id_list = list()
+            out_id_list = list()
+            prev_center = dict()
+            records = list()
+            if self.do_entrance_counting or self.do_break_in_counting:
+                if self.region_type == 'horizontal':
+                    entrance = [0, height / 2., width, height / 2.]
+                elif self.region_type == 'vertical':
+                    entrance = [width / 2, 0., width / 2, height]
+                elif self.region_type == 'custom':
+                    entrance = []
+                    assert len(
+                        self.region_polygon
+                    ) % 2 == 0, "region_polygon should be pairs of coords points when do break_in counting."
+                    for i in range(0, len(self.region_polygon), 2):
+                        entrance.append([
+                            self.region_polygon[i], self.region_polygon[i + 1]
+                        ])
+                    entrance.append([width, height])
+                else:
+                    raise ValueError("region_type:{} is not supported.".format(
+                        self.region_type))
+
+        video_fps = fps
+
+        while (1):
+            ret, frame = capture.read()
+            if not ret:
+                break
+            if frame_id % 10 == 0:
+                print('Tracking frame: %d' % (frame_id))
+
+            timer.tic()
+            mot_skip_frame_num = self.skip_frame_num
+            reuse_det_result = False
+            if mot_skip_frame_num > 1 and frame_id > 0 and frame_id % mot_skip_frame_num > 0:
+                reuse_det_result = True
+            seq_name = video_out_name.split('.')[0]
+            mot_results = self.predict_image(
+                [frame],
+                visual=False,
+                seq_name=seq_name,
+                reuse_det_result=reuse_det_result)
+            timer.toc()
+
+            # bs=1 in MOT model
+            online_tlwhs, online_scores, online_ids = mot_results[0]
+
+            # flow statistic for one class, and only for bytetracker
+            if num_classes == 1 and not self.use_deepsort_tracker and not self.use_ocsort_tracker:
+                result = (frame_id + 1, online_tlwhs[0], online_scores[0],
+                          online_ids[0])
+                statistic = flow_statistic(
+                    result,
+                    self.secs_interval,
+                    self.do_entrance_counting,
+                    self.do_break_in_counting,
+                    self.region_type,
+                    video_fps,
+                    entrance,
+                    id_set,
+                    interval_id_set,
+                    in_id_list,
+                    out_id_list,
+                    prev_center,
+                    records,
+                    data_type,
+                    ids2names=self.pred_config.labels)
+                records = statistic['records']
+
+            fps = 1. / timer.duration
+            if self.use_deepsort_tracker or self.use_ocsort_tracker:
+                # use DeepSORTTracker or OCSORTTracker, only support singe class
+                results[0].append(
+                    (frame_id + 1, online_tlwhs, online_scores, online_ids))
+                im = plot_tracking(
+                    frame,
+                    online_tlwhs,
+                    online_ids,
+                    online_scores,
+                    frame_id=frame_id,
+                    fps=fps,
+                    ids2names=ids2names,
+                    do_entrance_counting=self.do_entrance_counting,
+                    entrance=entrance)
+            else:
+                # use ByteTracker, support multiple class
+                for cls_id in range(num_classes):
+                    results[cls_id].append(
+                        (frame_id + 1, online_tlwhs[cls_id],
+                         online_scores[cls_id], online_ids[cls_id]))
+                im = plot_tracking_dict(
+                    frame,
+                    num_classes,
+                    online_tlwhs,
+                    online_ids,
+                    online_scores,
+                    frame_id=frame_id,
+                    fps=fps,
+                    ids2names=ids2names,
+                    do_entrance_counting=self.do_entrance_counting,
+                    entrance=entrance,
+                    records=records,
+                    center_traj=center_traj)
+
+            writer.write(im)
+            if camera_id != -1:
+                cv2.imshow('Mask Detection', im)
+                if cv2.waitKey(1) & 0xFF == ord('q'):
+                    break
+            frame_id += 1
+
+        if self.save_mot_txts:
+            result_filename = os.path.join(
+                self.output_dir, video_out_name.split('.')[-2] + '.txt')
+            write_mot_results(result_filename, results)
+
+            result_filename = os.path.join(
+                self.output_dir,
+                video_out_name.split('.')[-2] + '_flow_statistic.txt')
+            f = open(result_filename, 'w')
+            for line in records:
+                f.write(line)
+            print('Flow statistic save in {}'.format(result_filename))
+            f.close()
+
+        writer.release()
+
+    def predict_mtmct(self, mtmct_dir, mtmct_cfg):
+        cameras_bias = mtmct_cfg['cameras_bias']
+        cid_bias = parse_bias(cameras_bias)
+        scene_cluster = list(cid_bias.keys())
+        # 1.zone releated parameters
+        use_zone = mtmct_cfg.get('use_zone', False)
+        zone_path = mtmct_cfg.get('zone_path', None)
+
+        # 2.tricks parameters, can be used for other mtmct dataset
+        use_ff = mtmct_cfg.get('use_ff', False)
+        use_rerank = mtmct_cfg.get('use_rerank', False)
+
+        # 3.camera releated parameters
+        use_camera = mtmct_cfg.get('use_camera', False)
+        use_st_filter = mtmct_cfg.get('use_st_filter', False)
+
+        # 4.zone releated parameters
+        use_roi = mtmct_cfg.get('use_roi', False)
+        roi_dir = mtmct_cfg.get('roi_dir', False)
+
+        mot_list_breaks = []
+        cid_tid_dict = dict()
+
+        output_dir = self.output_dir
+        if not os.path.exists(output_dir):
+            os.makedirs(output_dir)
+
+        seqs = os.listdir(mtmct_dir)
+        for seq in sorted(seqs):
+            fpath = os.path.join(mtmct_dir, seq)
+            if os.path.isfile(fpath) and _is_valid_video(fpath):
+                seq = seq.split('.')[-2]
+                print('ffmpeg processing of video {}'.format(fpath))
+                frames_path = video2frames(
+                    video_path=fpath, outpath=mtmct_dir, frame_rate=25)
+                fpath = os.path.join(mtmct_dir, seq)
+
+            if os.path.isdir(fpath) == False:
+                print('{} is not a image folder.'.format(fpath))
+                continue
+            if os.path.exists(os.path.join(fpath, 'img1')):
+                fpath = os.path.join(fpath, 'img1')
+            assert os.path.isdir(fpath), '{} should be a directory'.format(
+                fpath)
+            image_list = glob.glob(os.path.join(fpath, '*.jpg'))
+            image_list.sort()
+            assert len(image_list) > 0, '{} has no images.'.format(fpath)
+            print('start tracking seq: {}'.format(seq))
+
+            mot_features_dict = self.predict_image(
+                image_list, visual=False, seq_name=seq)
+
+            cid = int(re.sub('[a-z,A-Z]', "", seq))
+            tid_data, mot_list_break = trajectory_fusion(
+                mot_features_dict,
+                cid,
+                cid_bias,
+                use_zone=use_zone,
+                zone_path=zone_path)
+            mot_list_breaks.append(mot_list_break)
+            # single seq process
+            for line in tid_data:
+                tracklet = tid_data[line]
+                tid = tracklet['tid']
+                if (cid, tid) not in cid_tid_dict:
+                    cid_tid_dict[(cid, tid)] = tracklet
+
+        map_tid = sub_cluster(
+            cid_tid_dict,
+            scene_cluster,
+            use_ff=use_ff,
+            use_rerank=use_rerank,
+            use_camera=use_camera,
+            use_st_filter=use_st_filter)
+
+        pred_mtmct_file = os.path.join(output_dir, 'mtmct_result.txt')
+        if use_camera:
+            gen_res(pred_mtmct_file, scene_cluster, map_tid, mot_list_breaks)
+        else:
+            gen_res(
+                pred_mtmct_file,
+                scene_cluster,
+                map_tid,
+                mot_list_breaks,
+                use_roi=use_roi,
+                roi_dir=roi_dir)
+
+        camera_results, cid_tid_fid_res = get_mtmct_matching_results(
+            pred_mtmct_file)
+
+        crops_dir = os.path.join(output_dir, 'mtmct_crops')
+        save_mtmct_crops(
+            cid_tid_fid_res, images_dir=mtmct_dir, crops_dir=crops_dir)
+
+        save_dir = os.path.join(output_dir, 'mtmct_vis')
+        save_mtmct_vis_results(
+            camera_results,
+            images_dir=mtmct_dir,
+            save_dir=save_dir,
+            save_videos=FLAGS.save_images)
+
+
+def main():
+    deploy_file = os.path.join(FLAGS.model_dir, 'infer_cfg.yml')
+    with open(deploy_file) as f:
+        yml_conf = yaml.safe_load(f)
+    arch = yml_conf['arch']
+    detector = SDE_Detector(
+        FLAGS.model_dir,
+        tracker_config=FLAGS.tracker_config,
+        device=FLAGS.device,
+        run_mode=FLAGS.run_mode,
+        batch_size=1,
+        trt_min_shape=FLAGS.trt_min_shape,
+        trt_max_shape=FLAGS.trt_max_shape,
+        trt_opt_shape=FLAGS.trt_opt_shape,
+        trt_calib_mode=FLAGS.trt_calib_mode,
+        cpu_threads=FLAGS.cpu_threads,
+        enable_mkldnn=FLAGS.enable_mkldnn,
+        output_dir=FLAGS.output_dir,
+        threshold=FLAGS.threshold,
+        save_images=FLAGS.save_images,
+        save_mot_txts=FLAGS.save_mot_txts,
+        draw_center_traj=FLAGS.draw_center_traj,
+        secs_interval=FLAGS.secs_interval,
+        skip_frame_num=FLAGS.skip_frame_num,
+        do_entrance_counting=FLAGS.do_entrance_counting,
+        do_break_in_counting=FLAGS.do_break_in_counting,
+        region_type=FLAGS.region_type,
+        region_polygon=FLAGS.region_polygon,
+        reid_model_dir=FLAGS.reid_model_dir,
+        mtmct_dir=FLAGS.mtmct_dir, )
+
+    # predict from video file or camera video stream
+    if FLAGS.video_file is not None or FLAGS.camera_id != -1:
+        detector.predict_video(FLAGS.video_file, FLAGS.camera_id)
+    elif FLAGS.mtmct_dir is not None:
+        with open(FLAGS.mtmct_cfg) as f:
+            mtmct_cfg = yaml.safe_load(f)
+        detector.predict_mtmct(FLAGS.mtmct_dir, mtmct_cfg)
+    else:
+        # predict from image
+        if FLAGS.image_dir is None and FLAGS.image_file is not None:
+            assert FLAGS.batch_size == 1, "--batch_size should be 1 in MOT models."
+        img_list = get_test_images(FLAGS.image_dir, FLAGS.image_file)
+        seq_name = FLAGS.image_dir.split('/')[-1]
+        detector.predict_image(
+            img_list, FLAGS.run_benchmark, repeats=10, seq_name=seq_name)
+
+        if not FLAGS.run_benchmark:
+            detector.det_times.info(average=True)
+        else:
+            mode = FLAGS.run_mode
+            model_dir = FLAGS.model_dir
+            model_info = {
+                'model_name': model_dir.strip('/').split('/')[-1],
+                'precision': mode.split('_')[-1]
+            }
+            bench_log(detector, img_list, model_info, name='MOT')
+
+
+if __name__ == '__main__':
+    paddle.enable_static()
+    parser = argsparser()
+    FLAGS = parser.parse_args()
+    print_arguments(FLAGS)
+    FLAGS.device = FLAGS.device.upper()
+    assert FLAGS.device in ['CPU', 'GPU', 'XPU'
+                            ], "device should be CPU, GPU or XPU"
+
+    main()

pptracking/python/mot_utils.py

@@ -0,0 +1,349 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import time
+import os
+import sys
+import ast
+import argparse
+
+
+def argsparser():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--model_dir",
+        type=str,
+        default=None,
+        help=("Directory include:'model.pdiparams', 'model.pdmodel', "
+              "'infer_cfg.yml', created by tools/export_model.py."),
+        required=True)
+    parser.add_argument(
+        "--image_file", type=str, default=None, help="Path of image file.")
+    parser.add_argument(
+        "--image_dir",
+        type=str,
+        default=None,
+        help="Dir of image file, `image_file` has a higher priority.")
+    parser.add_argument(
+        "--batch_size", type=int, default=1, help="batch_size for inference.")
+    parser.add_argument(
+        "--video_file",
+        type=str,
+        default=None,
+        help="Path of video file, `video_file` or `camera_id` has a highest priority."
+    )
+    parser.add_argument(
+        "--camera_id",
+        type=int,
+        default=-1,
+        help="device id of camera to predict.")
+    parser.add_argument(
+        "--threshold", type=float, default=0.5, help="Threshold of score.")
+    parser.add_argument(
+        "--output_dir",
+        type=str,
+        default="output",
+        help="Directory of output visualization files.")
+    parser.add_argument(
+        "--run_mode",
+        type=str,
+        default='paddle',
+        help="mode of running(paddle/trt_fp32/trt_fp16/trt_int8)")
+    parser.add_argument(
+        "--device",
+        type=str,
+        default='cpu',
+        help="Choose the device you want to run, it can be: CPU/GPU/XPU, default is CPU."
+    )
+    parser.add_argument(
+        "--use_gpu",
+        type=ast.literal_eval,
+        default=False,
+        help="Deprecated, please use `--device`.")
+    parser.add_argument(
+        "--run_benchmark",
+        type=ast.literal_eval,
+        default=False,
+        help="Whether to predict a image_file repeatedly for benchmark")
+    parser.add_argument(
+        "--enable_mkldnn",
+        type=ast.literal_eval,
+        default=False,
+        help="Whether use mkldnn with CPU.")
+    parser.add_argument(
+        "--cpu_threads", type=int, default=1, help="Num of threads with CPU.")
+    parser.add_argument(
+        "--trt_min_shape", type=int, default=1, help="min_shape for TensorRT.")
+    parser.add_argument(
+        "--trt_max_shape",
+        type=int,
+        default=1280,
+        help="max_shape for TensorRT.")
+    parser.add_argument(
+        "--trt_opt_shape",
+        type=int,
+        default=640,
+        help="opt_shape for TensorRT.")
+    parser.add_argument(
+        "--trt_calib_mode",
+        type=bool,
+        default=False,
+        help="If the model is produced by TRT offline quantitative "
+        "calibration, trt_calib_mode need to set True.")
+    parser.add_argument(
+        '--save_images',
+        action='store_true',
+        help='Save visualization image results.')
+    parser.add_argument(
+        '--save_mot_txts',
+        action='store_true',
+        help='Save tracking results (txt).')
+    parser.add_argument(
+        '--save_mot_txt_per_img',
+        action='store_true',
+        help='Save tracking results (txt) for each image.')
+    parser.add_argument(
+        '--scaled',
+        type=bool,
+        default=False,
+        help="Whether coords after detector outputs are scaled, False in JDE YOLOv3 "
+        "True in general detector.")
+    parser.add_argument(
+        "--tracker_config", type=str, default=None, help=("tracker donfig"))
+    parser.add_argument(
+        "--reid_model_dir",
+        type=str,
+        default=None,
+        help=("Directory include:'model.pdiparams', 'model.pdmodel', "
+              "'infer_cfg.yml', created by tools/export_model.py."))
+    parser.add_argument(
+        "--reid_batch_size",
+        type=int,
+        default=50,
+        help="max batch_size for reid model inference.")
+    parser.add_argument(
+        '--use_dark',
+        type=ast.literal_eval,
+        default=True,
+        help='whether to use darkpose to get better keypoint position predict ')
+    parser.add_argument(
+        '--skip_frame_num',
+        type=int,
+        default=-1,
+        help='Skip frames to speed up the process of getting mot results.')
+    parser.add_argument(
+        "--do_entrance_counting",
+        action='store_true',
+        help="Whether counting the numbers of identifiers entering "
+        "or getting out from the entrance. Note that only support single-class MOT."
+    )
+    parser.add_argument(
+        "--do_break_in_counting",
+        action='store_true',
+        help="Whether counting the numbers of identifiers break in "
+        "the area. Note that only support single-class MOT and "
+        "the video should be taken by a static camera.")
+    parser.add_argument(
+        "--region_type",
+        type=str,
+        default='horizontal',
+        help="Area type for entrance counting or break in counting, 'horizontal' and "
+        "'vertical' used when do entrance counting. 'custom' used when do break in counting. "
+        "Note that only support single-class MOT, and the video should be taken by a static camera."
+    )
+    parser.add_argument(
+        '--region_polygon',
+        nargs='+',
+        type=int,
+        default=[],
+        help="Clockwise point coords (x0,y0,x1,y1...) of polygon of area when "
+        "do_break_in_counting. Note that only support single-class MOT and "
+        "the video should be taken by a static camera.")
+    parser.add_argument(
+        "--secs_interval",
+        type=int,
+        default=2,
+        help="The seconds interval to count after tracking")
+    parser.add_argument(
+        "--draw_center_traj",
+        action='store_true',
+        help="Whether drawing the trajectory of center")
+    parser.add_argument(
+        "--mtmct_dir",
+        type=str,
+        default=None,
+        help="The MTMCT scene video folder.")
+    parser.add_argument(
+        "--mtmct_cfg", type=str, default=None, help="The MTMCT config.")
+    return parser
+
+
+class Times(object):
+    def __init__(self):
+        self.time = 0.
+        # start time
+        self.st = 0.
+        # end time
+        self.et = 0.
+
+    def start(self):
+        self.st = time.time()
+
+    def end(self, repeats=1, accumulative=True):
+        self.et = time.time()
+        if accumulative:
+            self.time += (self.et - self.st) / repeats
+        else:
+            self.time = (self.et - self.st) / repeats
+
+    def reset(self):
+        self.time = 0.
+        self.st = 0.
+        self.et = 0.
+
+    def value(self):
+        return round(self.time, 4)
+
+
+class Timer(Times):
+    def __init__(self, with_tracker=False):
+        super(Timer, self).__init__()
+        self.with_tracker = with_tracker
+        self.preprocess_time_s = Times()
+        self.inference_time_s = Times()
+        self.postprocess_time_s = Times()
+        self.tracking_time_s = Times()
+        self.img_num = 0
+
+    def info(self, average=False):
+        pre_time = self.preprocess_time_s.value()
+        infer_time = self.inference_time_s.value()
+        post_time = self.postprocess_time_s.value()
+        track_time = self.tracking_time_s.value()
+
+        total_time = pre_time + infer_time + post_time
+        if self.with_tracker:
+            total_time = total_time + track_time
+        total_time = round(total_time, 4)
+        print("------------------ Inference Time Info ----------------------")
+        print("total_time(ms): {}, img_num: {}".format(total_time * 1000,
+                                                       self.img_num))
+        preprocess_time = round(pre_time / max(1, self.img_num),
+                                4) if average else pre_time
+        postprocess_time = round(post_time / max(1, self.img_num),
+                                 4) if average else post_time
+        inference_time = round(infer_time / max(1, self.img_num),
+                               4) if average else infer_time
+        tracking_time = round(track_time / max(1, self.img_num),
+                              4) if average else track_time
+
+        average_latency = total_time / max(1, self.img_num)
+        qps = 0
+        if total_time > 0:
+            qps = 1 / average_latency
+        print("average latency time(ms): {:.2f}, QPS: {:2f}".format(
+            average_latency * 1000, qps))
+        if self.with_tracker:
+            print(
+                "preprocess_time(ms): {:.2f}, inference_time(ms): {:.2f}, postprocess_time(ms): {:.2f}, tracking_time(ms): {:.2f}".
+                format(preprocess_time * 1000, inference_time * 1000,
+                       postprocess_time * 1000, tracking_time * 1000))
+        else:
+            print(
+                "preprocess_time(ms): {:.2f}, inference_time(ms): {:.2f}, postprocess_time(ms): {:.2f}".
+                format(preprocess_time * 1000, inference_time * 1000,
+                       postprocess_time * 1000))
+
+    def report(self, average=False):
+        dic = {}
+        pre_time = self.preprocess_time_s.value()
+        infer_time = self.inference_time_s.value()
+        post_time = self.postprocess_time_s.value()
+        track_time = self.tracking_time_s.value()
+
+        dic['preprocess_time_s'] = round(pre_time / max(1, self.img_num),
+                                         4) if average else pre_time
+        dic['inference_time_s'] = round(infer_time / max(1, self.img_num),
+                                        4) if average else infer_time
+        dic['postprocess_time_s'] = round(post_time / max(1, self.img_num),
+                                          4) if average else post_time
+        dic['img_num'] = self.img_num
+        total_time = pre_time + infer_time + post_time
+        if self.with_tracker:
+            dic['tracking_time_s'] = round(track_time / max(1, self.img_num),
+                                           4) if average else track_time
+            total_time = total_time + track_time
+        dic['total_time_s'] = round(total_time, 4)
+        return dic
+
+
+def get_current_memory_mb():
+    """
+    It is used to Obtain the memory usage of the CPU and GPU during the running of the program.
+    And this function Current program is time-consuming.
+    """
+    import pynvml
+    import psutil
+    import GPUtil
+    gpu_id = int(os.environ.get('CUDA_VISIBLE_DEVICES', 0))
+
+    pid = os.getpid()
+    p = psutil.Process(pid)
+    info = p.memory_full_info()
+    cpu_mem = info.uss / 1024. / 1024.
+    gpu_mem = 0
+    gpu_percent = 0
+    gpus = GPUtil.getGPUs()
+    if gpu_id is not None and len(gpus) > 0:
+        gpu_percent = gpus[gpu_id].load
+        pynvml.nvmlInit()
+        handle = pynvml.nvmlDeviceGetHandleByIndex(0)
+        meminfo = pynvml.nvmlDeviceGetMemoryInfo(handle)
+        gpu_mem = meminfo.used / 1024. / 1024.
+    return round(cpu_mem, 4), round(gpu_mem, 4), round(gpu_percent, 4)
+
+
+def video2frames(video_path, outpath, frame_rate=25, **kargs):
+    def _dict2str(kargs):
+        cmd_str = ''
+        for k, v in kargs.items():
+            cmd_str += (' ' + str(k) + ' ' + str(v))
+        return cmd_str
+
+    ffmpeg = ['ffmpeg ', ' -y -loglevel ', ' error ']
+    vid_name = os.path.basename(video_path).split('.')[0]
+    out_full_path = os.path.join(outpath, vid_name)
+
+    if not os.path.exists(out_full_path):
+        os.makedirs(out_full_path)
+
+    # video file name
+    outformat = os.path.join(out_full_path, '%05d.jpg')
+
+    cmd = ffmpeg
+    cmd = ffmpeg + [
+        ' -i ', video_path, ' -r ', str(frame_rate), ' -f image2 ', outformat
+    ]
+    cmd = ''.join(cmd) + _dict2str(kargs)
+
+    if os.system(cmd) != 0:
+        raise RuntimeError('ffmpeg process video: {} error'.format(video_path))
+        sys.exit(-1)
+
+    sys.stdout.flush()
+    return out_full_path
+
+
+def _is_valid_video(f, extensions=('.mp4', '.avi', '.mov', '.rmvb', '.flv')):
+    return f.lower().endswith(extensions)

pptracking/python/mtmct_cfg.yml

@@ -0,0 +1,17 @@
+# config for MTMCT
+MTMCT: True
+cameras_bias: # default for scene S01. For S06, should modify as 'c041: 0  c042: 0'
+  c003: 0
+  c004: 0
+# 1.zone releated parameters
+use_zone: False
+zone_path: dataset/mot/aic21mtmct_vehicle/S06/zone
+# 2.tricks parameters, can be used for other mtmct dataset
+use_ff: False
+use_rerank: False
+# 3.camera releated parameters
+use_camera: False
+use_st_filter: False
+# 4.zone releated parameters
+use_roi: False
+roi_dir: dataset/mot/aic21mtmct_vehicle/S06

pptracking/python/picodet_postprocess.py

@@ -0,0 +1,227 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+from scipy.special import softmax
+
+
+def hard_nms(box_scores, iou_threshold, top_k=-1, candidate_size=200):
+    """
+    Args:
+        box_scores (N, 5): boxes in corner-form and probabilities.
+        iou_threshold: intersection over union threshold.
+        top_k: keep top_k results. If k <= 0, keep all the results.
+        candidate_size: only consider the candidates with the highest scores.
+    Returns:
+         picked: a list of indexes of the kept boxes
+    """
+    scores = box_scores[:, -1]
+    boxes = box_scores[:, :-1]
+    picked = []
+    indexes = np.argsort(scores)
+    indexes = indexes[-candidate_size:]
+    while len(indexes) > 0:
+        current = indexes[-1]
+        picked.append(current)
+        if 0 < top_k == len(picked) or len(indexes) == 1:
+            break
+        current_box = boxes[current, :]
+        indexes = indexes[:-1]
+        rest_boxes = boxes[indexes, :]
+        iou = iou_of(
+            rest_boxes,
+            np.expand_dims(
+                current_box, axis=0), )
+        indexes = indexes[iou <= iou_threshold]
+
+    return box_scores[picked, :]
+
+
+def iou_of(boxes0, boxes1, eps=1e-5):
+    """Return intersection-over-union (Jaccard index) of boxes.
+    Args:
+        boxes0 (N, 4): ground truth boxes.
+        boxes1 (N or 1, 4): predicted boxes.
+        eps: a small number to avoid 0 as denominator.
+    Returns:
+        iou (N): IoU values.
+    """
+    overlap_left_top = np.maximum(boxes0[..., :2], boxes1[..., :2])
+    overlap_right_bottom = np.minimum(boxes0[..., 2:], boxes1[..., 2:])
+
+    overlap_area = area_of(overlap_left_top, overlap_right_bottom)
+    area0 = area_of(boxes0[..., :2], boxes0[..., 2:])
+    area1 = area_of(boxes1[..., :2], boxes1[..., 2:])
+    return overlap_area / (area0 + area1 - overlap_area + eps)
+
+
+def area_of(left_top, right_bottom):
+    """Compute the areas of rectangles given two corners.
+    Args:
+        left_top (N, 2): left top corner.
+        right_bottom (N, 2): right bottom corner.
+    Returns:
+        area (N): return the area.
+    """
+    hw = np.clip(right_bottom - left_top, 0.0, None)
+    return hw[..., 0] * hw[..., 1]
+
+
+class PicoDetPostProcess(object):
+    """
+    Args:
+        input_shape (int): network input image size
+        ori_shape (int): ori image shape of before padding
+        scale_factor (float): scale factor of ori image
+        enable_mkldnn (bool): whether to open MKLDNN
+    """
+
+    def __init__(self,
+                 input_shape,
+                 ori_shape,
+                 scale_factor,
+                 strides=[8, 16, 32, 64],
+                 score_threshold=0.4,
+                 nms_threshold=0.5,
+                 nms_top_k=1000,
+                 keep_top_k=100):
+        self.ori_shape = ori_shape
+        self.input_shape = input_shape
+        self.scale_factor = scale_factor
+        self.strides = strides
+        self.score_threshold = score_threshold
+        self.nms_threshold = nms_threshold
+        self.nms_top_k = nms_top_k
+        self.keep_top_k = keep_top_k
+
+    def warp_boxes(self, boxes, ori_shape):
+        """Apply transform to boxes
+        """
+        width, height = ori_shape[1], ori_shape[0]
+        n = len(boxes)
+        if n:
+            # warp points
+            xy = np.ones((n * 4, 3))
+            xy[:, :2] = boxes[:, [0, 1, 2, 3, 0, 3, 2, 1]].reshape(
+                n * 4, 2)  # x1y1, x2y2, x1y2, x2y1
+            # xy = xy @ M.T  # transform
+            xy = (xy[:, :2] / xy[:, 2:3]).reshape(n, 8)  # rescale
+            # create new boxes
+            x = xy[:, [0, 2, 4, 6]]
+            y = xy[:, [1, 3, 5, 7]]
+            xy = np.concatenate(
+                (x.min(1), y.min(1), x.max(1), y.max(1))).reshape(4, n).T
+            # clip boxes
+            xy[:, [0, 2]] = xy[:, [0, 2]].clip(0, width)
+            xy[:, [1, 3]] = xy[:, [1, 3]].clip(0, height)
+            return xy.astype(np.float32)
+        else:
+            return boxes
+
+    def __call__(self, scores, raw_boxes):
+        batch_size = raw_boxes[0].shape[0]
+        reg_max = int(raw_boxes[0].shape[-1] / 4 - 1)
+        out_boxes_num = []
+        out_boxes_list = []
+        for batch_id in range(batch_size):
+            # generate centers
+            decode_boxes = []
+            select_scores = []
+            for stride, box_distribute, score in zip(self.strides, raw_boxes,
+                                                     scores):
+                box_distribute = box_distribute[batch_id]
+                score = score[batch_id]
+                # centers
+                fm_h = self.input_shape[0] / stride
+                fm_w = self.input_shape[1] / stride
+                h_range = np.arange(fm_h)
+                w_range = np.arange(fm_w)
+                ww, hh = np.meshgrid(w_range, h_range)
+                ct_row = (hh.flatten() + 0.5) * stride
+                ct_col = (ww.flatten() + 0.5) * stride
+                center = np.stack((ct_col, ct_row, ct_col, ct_row), axis=1)
+
+                # box distribution to distance
+                reg_range = np.arange(reg_max + 1)
+                box_distance = box_distribute.reshape((-1, reg_max + 1))
+                box_distance = softmax(box_distance, axis=1)
+                box_distance = box_distance * np.expand_dims(reg_range, axis=0)
+                box_distance = np.sum(box_distance, axis=1).reshape((-1, 4))
+                box_distance = box_distance * stride
+
+                # top K candidate
+                topk_idx = np.argsort(score.max(axis=1))[::-1]
+                topk_idx = topk_idx[:self.nms_top_k]
+                center = center[topk_idx]
+                score = score[topk_idx]
+                box_distance = box_distance[topk_idx]
+
+                # decode box
+                decode_box = center + [-1, -1, 1, 1] * box_distance
+
+                select_scores.append(score)
+                decode_boxes.append(decode_box)
+
+            # nms
+            bboxes = np.concatenate(decode_boxes, axis=0)
+            confidences = np.concatenate(select_scores, axis=0)
+            picked_box_probs = []
+            picked_labels = []
+            for class_index in range(0, confidences.shape[1]):
+                probs = confidences[:, class_index]
+                mask = probs > self.score_threshold
+                probs = probs[mask]
+                if probs.shape[0] == 0:
+                    continue
+                subset_boxes = bboxes[mask, :]
+                box_probs = np.concatenate(
+                    [subset_boxes, probs.reshape(-1, 1)], axis=1)
+                box_probs = hard_nms(
+                    box_probs,
+                    iou_threshold=self.nms_threshold,
+                    top_k=self.keep_top_k, )
+                picked_box_probs.append(box_probs)
+                picked_labels.extend([class_index] * box_probs.shape[0])
+
+            if len(picked_box_probs) == 0:
+                out_boxes_list.append(np.empty((0, 4)))
+                out_boxes_num.append(0)
+
+            else:
+                picked_box_probs = np.concatenate(picked_box_probs)
+
+                # resize output boxes
+                picked_box_probs[:, :4] = self.warp_boxes(
+                    picked_box_probs[:, :4], self.ori_shape[batch_id])
+                im_scale = np.concatenate([
+                    self.scale_factor[batch_id][::-1],
+                    self.scale_factor[batch_id][::-1]
+                ])
+                picked_box_probs[:, :4] /= im_scale
+                # clas score box
+                out_boxes_list.append(
+                    np.concatenate(
+                        [
+                            np.expand_dims(
+                                np.array(picked_labels),
+                                axis=-1), np.expand_dims(
+                                    picked_box_probs[:, 4], axis=-1),
+                            picked_box_probs[:, :4]
+                        ],
+                        axis=1))
+                out_boxes_num.append(len(picked_labels))
+
+        out_boxes_list = np.concatenate(out_boxes_list, axis=0)
+        out_boxes_num = np.asarray(out_boxes_num).astype(np.int32)
+        return out_boxes_list, out_boxes_num

pptracking/python/preprocess.py

@@ -0,0 +1,286 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import cv2
+import numpy as np
+
+
+def decode_image(im_file, im_info):
+    """read rgb image
+    Args:
+        im_file (str|np.ndarray): input can be image path or np.ndarray
+        im_info (dict): info of image
+    Returns:
+        im (np.ndarray):  processed image (np.ndarray)
+        im_info (dict): info of processed image
+    """
+    if isinstance(im_file, str):
+        with open(im_file, 'rb') as f:
+            im_read = f.read()
+        data = np.frombuffer(im_read, dtype='uint8')
+        im = cv2.imdecode(data, 1)  # BGR mode, but need RGB mode
+        im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)
+    else:
+        im = im_file
+    im_info['im_shape'] = np.array(im.shape[:2], dtype=np.float32)
+    im_info['scale_factor'] = np.array([1., 1.], dtype=np.float32)
+    return im, im_info
+
+
+class Resize(object):
+    """resize image by target_size and max_size
+    Args:
+        target_size (int): the target size of image
+        keep_ratio (bool): whether keep_ratio or not, default true
+        interp (int): method of resize
+    """
+
+    def __init__(self, target_size, keep_ratio=True, interp=cv2.INTER_LINEAR):
+        if isinstance(target_size, int):
+            target_size = [target_size, target_size]
+        self.target_size = target_size
+        self.keep_ratio = keep_ratio
+        self.interp = interp
+
+    def __call__(self, im, im_info):
+        """
+        Args:
+            im (np.ndarray): image (np.ndarray)
+            im_info (dict): info of image
+        Returns:
+            im (np.ndarray):  processed image (np.ndarray)
+            im_info (dict): info of processed image
+        """
+        assert len(self.target_size) == 2
+        assert self.target_size[0] > 0 and self.target_size[1] > 0
+        im_channel = im.shape[2]
+        im_scale_y, im_scale_x = self.generate_scale(im)
+        im = cv2.resize(
+            im,
+            None,
+            None,
+            fx=im_scale_x,
+            fy=im_scale_y,
+            interpolation=self.interp)
+        im_info['im_shape'] = np.array(im.shape[:2]).astype('float32')
+        im_info['scale_factor'] = np.array(
+            [im_scale_y, im_scale_x]).astype('float32')
+        return im, im_info
+
+    def generate_scale(self, im):
+        """
+        Args:
+            im (np.ndarray): image (np.ndarray)
+        Returns:
+            im_scale_x: the resize ratio of X
+            im_scale_y: the resize ratio of Y
+        """
+        origin_shape = im.shape[:2]
+        im_c = im.shape[2]
+        if self.keep_ratio:
+            im_size_min = np.min(origin_shape)
+            im_size_max = np.max(origin_shape)
+            target_size_min = np.min(self.target_size)
+            target_size_max = np.max(self.target_size)
+            im_scale = float(target_size_min) / float(im_size_min)
+            if np.round(im_scale * im_size_max) > target_size_max:
+                im_scale = float(target_size_max) / float(im_size_max)
+            im_scale_x = im_scale
+            im_scale_y = im_scale
+        else:
+            resize_h, resize_w = self.target_size
+            im_scale_y = resize_h / float(origin_shape[0])
+            im_scale_x = resize_w / float(origin_shape[1])
+        return im_scale_y, im_scale_x
+
+
+class NormalizeImage(object):
+    """normalize image
+    Args:
+        mean (list): im - mean
+        std (list): im / std
+        is_scale (bool): whether need im / 255
+        is_channel_first (bool): if True: image shape is CHW, else: HWC
+    """
+
+    def __init__(self, mean, std, is_scale=True):
+        self.mean = mean
+        self.std = std
+        self.is_scale = is_scale
+
+    def __call__(self, im, im_info):
+        """
+        Args:
+            im (np.ndarray): image (np.ndarray)
+            im_info (dict): info of image
+        Returns:
+            im (np.ndarray):  processed image (np.ndarray)
+            im_info (dict): info of processed image
+        """
+        im = im.astype(np.float32, copy=False)
+        mean = np.array(self.mean)[np.newaxis, np.newaxis, :]
+        std = np.array(self.std)[np.newaxis, np.newaxis, :]
+
+        if self.is_scale:
+            im = im / 255.0
+        im -= mean
+        im /= std
+        return im, im_info
+
+
+class Permute(object):
+    """permute image
+    Args:
+        to_bgr (bool): whether convert RGB to BGR 
+        channel_first (bool): whether convert HWC to CHW
+    """
+
+    def __init__(self, ):
+        super(Permute, self).__init__()
+
+    def __call__(self, im, im_info):
+        """
+        Args:
+            im (np.ndarray): image (np.ndarray)
+            im_info (dict): info of image
+        Returns:
+            im (np.ndarray):  processed image (np.ndarray)
+            im_info (dict): info of processed image
+        """
+        im = im.transpose((2, 0, 1)).copy()
+        return im, im_info
+
+
+class PadStride(object):
+    """ padding image for model with FPN, instead PadBatch(pad_to_stride) in original config
+    Args:
+        stride (bool): model with FPN need image shape % stride == 0
+    """
+
+    def __init__(self, stride=0):
+        self.coarsest_stride = stride
+
+    def __call__(self, im, im_info):
+        """
+        Args:
+            im (np.ndarray): image (np.ndarray)
+            im_info (dict): info of image
+        Returns:
+            im (np.ndarray):  processed image (np.ndarray)
+            im_info (dict): info of processed image
+        """
+        coarsest_stride = self.coarsest_stride
+        if coarsest_stride <= 0:
+            return im, im_info
+        im_c, im_h, im_w = im.shape
+        pad_h = int(np.ceil(float(im_h) / coarsest_stride) * coarsest_stride)
+        pad_w = int(np.ceil(float(im_w) / coarsest_stride) * coarsest_stride)
+        padding_im = np.zeros((im_c, pad_h, pad_w), dtype=np.float32)
+        padding_im[:, :im_h, :im_w] = im
+        return padding_im, im_info
+
+
+class LetterBoxResize(object):
+    def __init__(self, target_size):
+        """
+        Resize image to target size, convert normalized xywh to pixel xyxy
+        format ([x_center, y_center, width, height] -> [x0, y0, x1, y1]).
+        Args:
+            target_size (int|list): image target size.
+        """
+        super(LetterBoxResize, self).__init__()
+        if isinstance(target_size, int):
+            target_size = [target_size, target_size]
+        self.target_size = target_size
+
+    def letterbox(self, img, height, width, color=(127.5, 127.5, 127.5)):
+        # letterbox: resize a rectangular image to a padded rectangular
+        shape = img.shape[:2]  # [height, width]
+        ratio_h = float(height) / shape[0]
+        ratio_w = float(width) / shape[1]
+        ratio = min(ratio_h, ratio_w)
+        new_shape = (round(shape[1] * ratio),
+                     round(shape[0] * ratio))  # [width, height]
+        padw = (width - new_shape[0]) / 2
+        padh = (height - new_shape[1]) / 2
+        top, bottom = round(padh - 0.1), round(padh + 0.1)
+        left, right = round(padw - 0.1), round(padw + 0.1)
+
+        img = cv2.resize(
+            img, new_shape, interpolation=cv2.INTER_AREA)  # resized, no border
+        img = cv2.copyMakeBorder(
+            img, top, bottom, left, right, cv2.BORDER_CONSTANT,
+            value=color)  # padded rectangular
+        return img, ratio, padw, padh
+
+    def __call__(self, im, im_info):
+        """
+        Args:
+            im (np.ndarray): image (np.ndarray)
+            im_info (dict): info of image
+        Returns:
+            im (np.ndarray):  processed image (np.ndarray)
+            im_info (dict): info of processed image
+        """
+        assert len(self.target_size) == 2
+        assert self.target_size[0] > 0 and self.target_size[1] > 0
+        height, width = self.target_size
+        h, w = im.shape[:2]
+        im, ratio, padw, padh = self.letterbox(im, height=height, width=width)
+
+        new_shape = [round(h * ratio), round(w * ratio)]
+        im_info['im_shape'] = np.array(new_shape, dtype=np.float32)
+        im_info['scale_factor'] = np.array([ratio, ratio], dtype=np.float32)
+        return im, im_info
+
+
+class Pad(object):
+    def __init__(self, size, fill_value=[114.0, 114.0, 114.0]):
+        """
+        Pad image to a specified size.
+        Args:
+            size (list[int]): image target size
+            fill_value (list[float]): rgb value of pad area, default (114.0, 114.0, 114.0)
+        """
+        super(Pad, self).__init__()
+        if isinstance(size, int):
+            size = [size, size]
+        self.size = size
+        self.fill_value = fill_value
+
+    def __call__(self, im, im_info):
+        im_h, im_w = im.shape[:2]
+        h, w = self.size
+        if h == im_h and w == im_w:
+            im = im.astype(np.float32)
+            return im, im_info
+
+        canvas = np.ones((h, w, 3), dtype=np.float32)
+        canvas *= np.array(self.fill_value, dtype=np.float32)
+        canvas[0:im_h, 0:im_w, :] = im.astype(np.float32)
+        im = canvas
+        return im, im_info
+
+
+def preprocess(im, preprocess_ops):
+    # process image by preprocess_ops
+    im_info = {
+        'scale_factor': np.array(
+            [1., 1.], dtype=np.float32),
+        'im_shape': None,
+    }
+    im, im_info = decode_image(im, im_info)
+    for operator in preprocess_ops:
+        im, im_info = operator(im, im_info)
+    return im, im_info

pptracking/python/tracker_config.yml

@@ -0,0 +1,43 @@
+# config of tracker for MOT SDE Detector, use 'OCSORTTracker' as default, 'JDETracker' here is just BYTETracker.
+# The tracker of MOT JDE Detector (such as FairMOT) is exported together with the model.
+# Here 'min_box_area' and 'vertical_ratio' are set for pedestrian, you can modify for other objects tracking.
+
+type: OCSORTTracker # choose one tracker in ['JDETracker', 'OCSORTTracker', 'DeepSORTTracker']
+# When using for MTMCT(Multi-Target Multi-Camera Tracking), you should modify to 'DeepSORTTracker'
+
+
+# just as BYTETracker, used for FairMOT in PP-Tracking project and for ByteTrack in PP-Humanv1 project
+JDETracker:
+  use_byte: True
+  det_thresh: 0.3
+  conf_thres: 0.6
+  low_conf_thres: 0.1
+  match_thres: 0.9
+  min_box_area: 0
+  vertical_ratio: 0 # 1.6 for pedestrian
+
+
+# used for OC-SORT in PP-Humanv2 project and PP-Vehicle project
+OCSORTTracker:
+  det_thresh: 0.4
+  max_age: 30
+  min_hits: 3
+  iou_threshold: 0.3
+  delta_t: 3
+  inertia: 0.2
+  min_box_area: 0
+  vertical_ratio: 0
+  use_byte: False
+
+
+# used for DeepSORT and MTMCT in PP-Tracking project
+DeepSORTTracker:
+  input_size: [64, 192] # An unique operation to scale the sub-image of the selected detected boxes to a fixed size
+  min_box_area: 0
+  vertical_ratio: -1
+  budget: 100
+  max_age: 70
+  n_init: 3
+  metric_type: cosine
+  matching_threshold: 0.2
+  max_iou_distance: 0.9

python/README.md

@@ -0,0 +1,104 @@
+# Python端预测部署
+
+在PaddlePaddle中预测引擎和训练引擎底层有着不同的优化方法, 预测引擎使用了AnalysisPredictor，专门针对推理进行了优化，是基于[C++预测库](https://www.paddlepaddle.org.cn/documentation/docs/zh/advanced_guide/inference_deployment/inference/native_infer.html)的Python接口，该引擎可以对模型进行多项图优化，减少不必要的内存拷贝。如果用户在部署已训练模型的过程中对性能有较高的要求，我们提供了独立于PaddleDetection的预测脚本，方便用户直接集成部署。
+
+
+Python端预测部署主要包含两个步骤：
+- 导出预测模型
+- 基于Python进行预测
+
+## 1. 导出预测模型
+
+PaddleDetection在训练过程包括网络的前向和优化器相关参数，而在部署过程中，我们只需要前向参数，具体参考:[导出模型](../EXPORT_MODEL.md)，例如
+
+```bash
+# 导出YOLOv3检测模型
+python tools/export_model.py -c configs/yolov3/yolov3_darknet53_270e_coco.yml --output_dir=./inference_model \
+ -o weights=https://paddledet.bj.bcebos.com/models/yolov3_darknet53_270e_coco.pdparams
+
+# 导出HigherHRNet(bottom-up)关键点检测模型
+python tools/export_model.py -c configs/keypoint/higherhrnet/higherhrnet_hrnet_w32_512.yml -o weights=https://paddledet.bj.bcebos.com/models/keypoint/higherhrnet_hrnet_w32_512.pdparams
+
+# 导出HRNet(top-down)关键点检测模型
+python tools/export_model.py -c configs/keypoint/hrnet/hrnet_w32_384x288.yml -o weights=https://paddledet.bj.bcebos.com/models/keypoint/hrnet_w32_384x288.pdparams
+
+# 导出FairMOT多目标跟踪模型
+python tools/export_model.py -c configs/mot/fairmot/fairmot_dla34_30e_1088x608.yml -o weights=https://paddledet.bj.bcebos.com/models/mot/fairmot_dla34_30e_1088x608.pdparams
+
+# 导出ByteTrack多目标跟踪模型(相当于只导出检测器)
+python tools/export_model.py -c configs/mot/bytetrack/detector/ppyoloe_crn_l_36e_640x640_mot17half.yml -o weights=https://paddledet.bj.bcebos.com/models/mot/ppyoloe_crn_l_36e_640x640_mot17half.pdparams
+```
+
+导出后目录下，包括`infer_cfg.yml`, `model.pdiparams`,  `model.pdiparams.info`, `model.pdmodel`四个文件。
+
+
+## 2. 基于Python的预测
+
+### 2.1 通用检测
+在终端输入以下命令进行预测：
+```bash
+python deploy/python/infer.py --model_dir=./output_inference/yolov3_darknet53_270e_coco --image_file=./demo/000000014439.jpg --device=GPU
+```
+
+### 2.2 关键点检测
+在终端输入以下命令进行预测：
+```bash
+# keypoint top-down(HRNet)/bottom-up(HigherHRNet)单独推理，该模式下top-down模型HRNet只支持单人截图预测
+python deploy/python/keypoint_infer.py --model_dir=output_inference/hrnet_w32_384x288/ --image_file=./demo/hrnet_demo.jpg --device=GPU --threshold=0.5
+python deploy/python/keypoint_infer.py --model_dir=output_inference/higherhrnet_hrnet_w32_512/ --image_file=./demo/000000014439_640x640.jpg --device=GPU --threshold=0.5
+
+# detector 检测 + keypoint top-down模型联合部署（联合推理只支持top-down关键点模型）
+python deploy/python/det_keypoint_unite_infer.py --det_model_dir=output_inference/yolov3_darknet53_270e_coco/ --keypoint_model_dir=output_inference/hrnet_w32_384x288/ --video_file={your video name}.mp4  --device=GPU
+```
+**注意:**
+ - 关键点检测模型导出和预测具体可参照[keypoint](../../configs/keypoint/README.md)，可分别在各个模型的文档中查找具体用法；
+ - 此目录下的关键点检测部署为基础前向功能，更多关键点检测功能可使用PP-Human项目，参照[pipeline](../pipeline/README.md)；
+
+
+### 2.3 多目标跟踪
+在终端输入以下命令进行预测：
+```bash
+# FairMOT跟踪
+python deploy/python/mot_jde_infer.py --model_dir=output_inference/fairmot_dla34_30e_1088x608 --video_file={your video name}.mp4 --device=GPU
+
+# ByteTrack跟踪
+python deploy/python/mot_sde_infer.py --model_dir=output_inference/ppyoloe_crn_l_36e_640x640_mot17half/ --tracker_config=deploy/python/tracker_config.yml --video_file={your video name}.mp4 --device=GPU --scaled=True
+
+# FairMOT多目标跟踪联合HRNet关键点检测（联合推理只支持top-down关键点模型）
+python deploy/python/mot_keypoint_unite_infer.py --mot_model_dir=output_inference/fairmot_dla34_30e_1088x608/ --keypoint_model_dir=output_inference/hrnet_w32_384x288/ --video_file={your video name}.mp4 --device=GPU
+```
+
+**注意:**
+ - 多目标跟踪模型导出和预测具体可参照[mot]](../../configs/mot/README.md)，可分别在各个模型的文档中查找具体用法；
+ - 此目录下的跟踪部署为基础前向功能以及联合关键点部署，更多跟踪功能可使用PP-Human项目，参照[pipeline](../pipeline/README.md)，或PP-Tracking项目(绘制轨迹、出入口流量计数)，参照[pptracking](../pptracking/README.md)；
+
+
+参数说明如下:
+
+| 参数 | 是否必须| 含义                                                                                          |
+|-------|-------|---------------------------------------------------------------------------------------------|
+| --model_dir | Yes| 上述导出的模型路径                                                                                   |
+| --image_file | Option | 需要预测的图片                                                                                     |
+| --image_dir  | Option | 要预测的图片文件夹路径                                                                                 |
+| --video_file | Option | 需要预测的视频                                                                                     |
+| --camera_id | Option | 用来预测的摄像头ID，默认为-1(表示不使用摄像头预测，可设置为：0 - (摄像头数目-1) )，预测过程中在可视化界面按`q`退出输出预测结果到：output/output.mp4 |
+| --device | Option | 运行时的设备，可选择`CPU/GPU/XPU`，默认为`CPU`                                                            |
+| --run_mode | Option | 使用GPU时，默认为paddle, 可选（paddle/trt_fp32/trt_fp16/trt_int8）                                     |
+| --batch_size | Option | 预测时的batch size，在指定`image_dir`时有效，默认为1                                                       |
+| --threshold | Option| 预测得分的阈值，默认为0.5                                                                              |
+| --output_dir | Option| 可视化结果保存的根目录，默认为output/                                                                      |
+| --run_benchmark | Option| 是否运行benchmark，同时需指定`--image_file`或`--image_dir`，默认为False                                    |
+| --enable_mkldnn | Option | CPU预测中是否开启MKLDNN加速，默认为False                                                                 |
+| --cpu_threads | Option| 设置cpu线程数，默认为1                                                                               |
+| --trt_calib_mode | Option| TensorRT是否使用校准功能，默认为False。使用TensorRT的int8功能时，需设置为True，使用PaddleSlim量化后的模型时需要设置为False         |
+| --save_images | Option| 是否保存可视化结果                                                                                   |
+| --save_results | Option| 是否在文件夹下将图片的预测结果以JSON的形式保存                                                                   |
+
+
+说明：
+
+- 参数优先级顺序：`camera_id` > `video_file` > `image_dir` > `image_file`。
+- run_mode：paddle代表使用AnalysisPredictor，精度float32来推理，其他参数指用AnalysisPredictor，TensorRT不同精度来推理。
+- 如果安装的PaddlePaddle不支持基于TensorRT进行预测，需要自行编译，详细可参考[预测库编译教程](https://paddleinference.paddlepaddle.org.cn/user_guides/source_compile.html)。
+- --run_benchmark如果设置为True，则需要安装依赖`pip install pynvml psutil GPUtil`。
+- 如果需要使用导出模型在coco数据集上进行评估，请在推理时添加`--save_results`和`--use_coco_category`参数用以保存coco评估所需要的json文件

python/benchmark_utils.py

@@ -0,0 +1,291 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import logging
+
+import paddle
+import paddle.inference as paddle_infer
+
+from pathlib import Path
+
+CUR_DIR = os.path.dirname(os.path.abspath(__file__))
+LOG_PATH_ROOT = f"{CUR_DIR}/../../output"
+
+
+class PaddleInferBenchmark(object):
+    def __init__(self,
+                 config,
+                 model_info: dict={},
+                 data_info: dict={},
+                 perf_info: dict={},
+                 resource_info: dict={},
+                 **kwargs):
+        """
+        Construct PaddleInferBenchmark Class to format logs.
+        args:
+            config(paddle.inference.Config): paddle inference config
+            model_info(dict): basic model info
+                {'model_name': 'resnet50'
+                 'precision': 'fp32'}
+            data_info(dict): input data info
+                {'batch_size': 1
+                 'shape': '3,224,224'
+                 'data_num': 1000}
+            perf_info(dict): performance result
+                {'preprocess_time_s': 1.0
+                'inference_time_s': 2.0
+                'postprocess_time_s': 1.0
+                'total_time_s': 4.0}
+            resource_info(dict): 
+                cpu and gpu resources
+                {'cpu_rss': 100
+                 'gpu_rss': 100
+                 'gpu_util': 60}
+        """
+        # PaddleInferBenchmark Log Version
+        self.log_version = "1.0.3"
+
+        # Paddle Version
+        self.paddle_version = paddle.__version__
+        self.paddle_commit = paddle.__git_commit__
+        paddle_infer_info = paddle_infer.get_version()
+        self.paddle_branch = paddle_infer_info.strip().split(': ')[-1]
+
+        # model info
+        self.model_info = model_info
+
+        # data info
+        self.data_info = data_info
+
+        # perf info
+        self.perf_info = perf_info
+
+        try:
+            # required value
+            self.model_name = model_info['model_name']
+            self.precision = model_info['precision']
+
+            self.batch_size = data_info['batch_size']
+            self.shape = data_info['shape']
+            self.data_num = data_info['data_num']
+
+            self.inference_time_s = round(perf_info['inference_time_s'], 4)
+        except:
+            self.print_help()
+            raise ValueError(
+                "Set argument wrong, please check input argument and its type")
+
+        self.preprocess_time_s = perf_info.get('preprocess_time_s', 0)
+        self.postprocess_time_s = perf_info.get('postprocess_time_s', 0)
+        self.with_tracker = True if 'tracking_time_s' in perf_info else False
+        self.tracking_time_s = perf_info.get('tracking_time_s', 0)
+        self.total_time_s = perf_info.get('total_time_s', 0)
+
+        self.inference_time_s_90 = perf_info.get("inference_time_s_90", "")
+        self.inference_time_s_99 = perf_info.get("inference_time_s_99", "")
+        self.succ_rate = perf_info.get("succ_rate", "")
+        self.qps = perf_info.get("qps", "")
+
+        # conf info
+        self.config_status = self.parse_config(config)
+
+        # mem info
+        if isinstance(resource_info, dict):
+            self.cpu_rss_mb = int(resource_info.get('cpu_rss_mb', 0))
+            self.cpu_vms_mb = int(resource_info.get('cpu_vms_mb', 0))
+            self.cpu_shared_mb = int(resource_info.get('cpu_shared_mb', 0))
+            self.cpu_dirty_mb = int(resource_info.get('cpu_dirty_mb', 0))
+            self.cpu_util = round(resource_info.get('cpu_util', 0), 2)
+
+            self.gpu_rss_mb = int(resource_info.get('gpu_rss_mb', 0))
+            self.gpu_util = round(resource_info.get('gpu_util', 0), 2)
+            self.gpu_mem_util = round(resource_info.get('gpu_mem_util', 0), 2)
+        else:
+            self.cpu_rss_mb = 0
+            self.cpu_vms_mb = 0
+            self.cpu_shared_mb = 0
+            self.cpu_dirty_mb = 0
+            self.cpu_util = 0
+
+            self.gpu_rss_mb = 0
+            self.gpu_util = 0
+            self.gpu_mem_util = 0
+
+        # init benchmark logger
+        self.benchmark_logger()
+
+    def benchmark_logger(self):
+        """
+        benchmark logger
+        """
+        # remove other logging handler
+        for handler in logging.root.handlers[:]:
+            logging.root.removeHandler(handler)
+
+        # Init logger
+        FORMAT = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+        log_output = f"{LOG_PATH_ROOT}/{self.model_name}.log"
+        Path(f"{LOG_PATH_ROOT}").mkdir(parents=True, exist_ok=True)
+        logging.basicConfig(
+            level=logging.INFO,
+            format=FORMAT,
+            handlers=[
+                logging.FileHandler(
+                    filename=log_output, mode='w'),
+                logging.StreamHandler(),
+            ])
+        self.logger = logging.getLogger(__name__)
+        self.logger.info(
+            f"Paddle Inference benchmark log will be saved to {log_output}")
+
+    def parse_config(self, config) -> dict:
+        """
+        parse paddle predictor config
+        args:
+            config(paddle.inference.Config): paddle inference config
+        return:
+            config_status(dict): dict style config info
+        """
+        if isinstance(config, paddle_infer.Config):
+            config_status = {}
+            config_status['runtime_device'] = "gpu" if config.use_gpu(
+            ) else "cpu"
+            config_status['ir_optim'] = config.ir_optim()
+            config_status['enable_tensorrt'] = config.tensorrt_engine_enabled()
+            config_status['precision'] = self.precision
+            config_status['enable_mkldnn'] = config.mkldnn_enabled()
+            config_status[
+                'cpu_math_library_num_threads'] = config.cpu_math_library_num_threads(
+                )
+        elif isinstance(config, dict):
+            config_status['runtime_device'] = config.get('runtime_device', "")
+            config_status['ir_optim'] = config.get('ir_optim', "")
+            config_status['enable_tensorrt'] = config.get('enable_tensorrt',
+                                                          "")
+            config_status['precision'] = config.get('precision', "")
+            config_status['enable_mkldnn'] = config.get('enable_mkldnn', "")
+            config_status['cpu_math_library_num_threads'] = config.get(
+                'cpu_math_library_num_threads', "")
+        else:
+            self.print_help()
+            raise ValueError(
+                "Set argument config wrong, please check input argument and its type"
+            )
+        return config_status
+
+    def report(self, identifier=None):
+        """
+        print log report
+        args:
+            identifier(string): identify log
+        """
+        if identifier:
+            identifier = f"[{identifier}]"
+        else:
+            identifier = ""
+
+        self.logger.info("\n")
+        self.logger.info(
+            "---------------------- Paddle info ----------------------")
+        self.logger.info(f"{identifier} paddle_version: {self.paddle_version}")
+        self.logger.info(f"{identifier} paddle_commit: {self.paddle_commit}")
+        self.logger.info(f"{identifier} paddle_branch: {self.paddle_branch}")
+        self.logger.info(f"{identifier} log_api_version: {self.log_version}")
+        self.logger.info(
+            "----------------------- Conf info -----------------------")
+        self.logger.info(
+            f"{identifier} runtime_device: {self.config_status['runtime_device']}"
+        )
+        self.logger.info(
+            f"{identifier} ir_optim: {self.config_status['ir_optim']}")
+        self.logger.info(f"{identifier} enable_memory_optim: {True}")
+        self.logger.info(
+            f"{identifier} enable_tensorrt: {self.config_status['enable_tensorrt']}"
+        )
+        self.logger.info(
+            f"{identifier} enable_mkldnn: {self.config_status['enable_mkldnn']}"
+        )
+        self.logger.info(
+            f"{identifier} cpu_math_library_num_threads: {self.config_status['cpu_math_library_num_threads']}"
+        )
+        self.logger.info(
+            "----------------------- Model info ----------------------")
+        self.logger.info(f"{identifier} model_name: {self.model_name}")
+        self.logger.info(f"{identifier} precision: {self.precision}")
+        self.logger.info(
+            "----------------------- Data info -----------------------")
+        self.logger.info(f"{identifier} batch_size: {self.batch_size}")
+        self.logger.info(f"{identifier} input_shape: {self.shape}")
+        self.logger.info(f"{identifier} data_num: {self.data_num}")
+        self.logger.info(
+            "----------------------- Perf info -----------------------")
+        self.logger.info(
+            f"{identifier} cpu_rss(MB): {self.cpu_rss_mb}, cpu_vms: {self.cpu_vms_mb}, cpu_shared_mb: {self.cpu_shared_mb}, cpu_dirty_mb: {self.cpu_dirty_mb}, cpu_util: {self.cpu_util}%"
+        )
+        self.logger.info(
+            f"{identifier} gpu_rss(MB): {self.gpu_rss_mb}, gpu_util: {self.gpu_util}%, gpu_mem_util: {self.gpu_mem_util}%"
+        )
+        self.logger.info(
+            f"{identifier} total time spent(s): {self.total_time_s}")
+
+        if self.with_tracker:
+            self.logger.info(
+                f"{identifier} preprocess_time(ms): {round(self.preprocess_time_s*1000, 1)}, "
+                f"inference_time(ms): {round(self.inference_time_s*1000, 1)}, "
+                f"postprocess_time(ms): {round(self.postprocess_time_s*1000, 1)}, "
+                f"tracking_time(ms): {round(self.tracking_time_s*1000, 1)}")
+        else:
+            self.logger.info(
+                f"{identifier} preprocess_time(ms): {round(self.preprocess_time_s*1000, 1)}, "
+                f"inference_time(ms): {round(self.inference_time_s*1000, 1)}, "
+                f"postprocess_time(ms): {round(self.postprocess_time_s*1000, 1)}"
+            )
+        if self.inference_time_s_90:
+            self.looger.info(
+                f"{identifier} 90%_cost: {self.inference_time_s_90}, 99%_cost: {self.inference_time_s_99}, succ_rate: {self.succ_rate}"
+            )
+        if self.qps:
+            self.logger.info(f"{identifier} QPS: {self.qps}")
+
+    def print_help(self):
+        """
+        print function help
+        """
+        print("""Usage: 
+            ==== Print inference benchmark logs. ====
+            config = paddle.inference.Config()
+            model_info = {'model_name': 'resnet50'
+                          'precision': 'fp32'}
+            data_info = {'batch_size': 1
+                         'shape': '3,224,224'
+                         'data_num': 1000}
+            perf_info = {'preprocess_time_s': 1.0
+                         'inference_time_s': 2.0
+                         'postprocess_time_s': 1.0
+                         'total_time_s': 4.0}
+            resource_info = {'cpu_rss_mb': 100
+                             'gpu_rss_mb': 100
+                             'gpu_util': 60}
+            log = PaddleInferBenchmark(config, model_info, data_info, perf_info, resource_info)
+            log('Test')
+            """)
+
+    def __call__(self, identifier=None):
+        """
+        __call__
+        args:
+            identifier(string): identify log
+        """
+        self.report(identifier)

python/det_keypoint_unite_infer.py

@@ -0,0 +1,377 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import json
+import cv2
+import math
+import numpy as np
+import paddle
+import yaml
+
+from det_keypoint_unite_utils import argsparser
+from preprocess import decode_image
+from infer import Detector, DetectorPicoDet, PredictConfig, print_arguments, get_test_images, bench_log
+from keypoint_infer import KeyPointDetector, PredictConfig_KeyPoint
+from visualize import visualize_pose
+from benchmark_utils import PaddleInferBenchmark
+from utils import get_current_memory_mb
+from keypoint_postprocess import translate_to_ori_images
+
+KEYPOINT_SUPPORT_MODELS = {
+    'HigherHRNet': 'keypoint_bottomup',
+    'HRNet': 'keypoint_topdown'
+}
+
+
+def predict_with_given_det(image, det_res, keypoint_detector,
+                           keypoint_batch_size, run_benchmark):
+    keypoint_res = {}
+
+    rec_images, records, det_rects = keypoint_detector.get_person_from_rect(
+        image, det_res)
+
+    if len(det_rects) == 0:
+        keypoint_res['keypoint'] = [[], []]
+        return keypoint_res
+
+    keypoint_vector = []
+    score_vector = []
+
+    rect_vector = det_rects
+    keypoint_results = keypoint_detector.predict_image(
+        rec_images, run_benchmark, repeats=10, visual=False)
+    keypoint_vector, score_vector = translate_to_ori_images(keypoint_results,
+                                                            np.array(records))
+    keypoint_res['keypoint'] = [
+        keypoint_vector.tolist(), score_vector.tolist()
+    ] if len(keypoint_vector) > 0 else [[], []]
+    keypoint_res['bbox'] = rect_vector
+    return keypoint_res
+
+
+def topdown_unite_predict(detector,
+                          topdown_keypoint_detector,
+                          image_list,
+                          keypoint_batch_size=1,
+                          save_res=False):
+    det_timer = detector.get_timer()
+    store_res = []
+    for i, img_file in enumerate(image_list):
+        # Decode image in advance in det + pose prediction
+        det_timer.preprocess_time_s.start()
+        image, _ = decode_image(img_file, {})
+        det_timer.preprocess_time_s.end()
+
+        if FLAGS.run_benchmark:
+            results = detector.predict_image(
+                [image], run_benchmark=True, repeats=10)
+
+            cm, gm, gu = get_current_memory_mb()
+            detector.cpu_mem += cm
+            detector.gpu_mem += gm
+            detector.gpu_util += gu
+        else:
+            results = detector.predict_image([image], visual=False)
+        results = detector.filter_box(results, FLAGS.det_threshold)
+        if results['boxes_num'] > 0:
+            keypoint_res = predict_with_given_det(
+                image, results, topdown_keypoint_detector, keypoint_batch_size,
+                FLAGS.run_benchmark)
+
+            if save_res:
+                save_name = img_file if isinstance(img_file, str) else i
+                store_res.append([
+                    save_name, keypoint_res['bbox'], [
+                        keypoint_res['keypoint'][0],
+                        keypoint_res['keypoint'][1]
+                    ]
+                ])
+        else:
+            results["keypoint"] = [[], []]
+            keypoint_res = results
+        if FLAGS.run_benchmark:
+            cm, gm, gu = get_current_memory_mb()
+            topdown_keypoint_detector.cpu_mem += cm
+            topdown_keypoint_detector.gpu_mem += gm
+            topdown_keypoint_detector.gpu_util += gu
+        else:
+            if not os.path.exists(FLAGS.output_dir):
+                os.makedirs(FLAGS.output_dir)
+            visualize_pose(
+                img_file,
+                keypoint_res,
+                visual_thresh=FLAGS.keypoint_threshold,
+                save_dir=FLAGS.output_dir)
+    if save_res:
+        """
+        1) store_res: a list of image_data
+        2) image_data: [imageid, rects, [keypoints, scores]]
+        3) rects: list of rect [xmin, ymin, xmax, ymax]
+        4) keypoints: 17(joint numbers)*[x, y, conf], total 51 data in list
+        5) scores: mean of all joint conf
+        """
+        with open("det_keypoint_unite_image_results.json", 'w') as wf:
+            json.dump(store_res, wf, indent=4)
+
+
+def topdown_unite_predict_video(detector,
+                                topdown_keypoint_detector,
+                                camera_id,
+                                keypoint_batch_size=1,
+                                save_res=False):
+    video_name = 'output.mp4'
+    if camera_id != -1:
+        capture = cv2.VideoCapture(camera_id)
+    else:
+        capture = cv2.VideoCapture(FLAGS.video_file)
+        video_name = os.path.split(FLAGS.video_file)[-1]
+    # Get Video info : resolution, fps, frame count
+    width = int(capture.get(cv2.CAP_PROP_FRAME_WIDTH))
+    height = int(capture.get(cv2.CAP_PROP_FRAME_HEIGHT))
+    fps = int(capture.get(cv2.CAP_PROP_FPS))
+    frame_count = int(capture.get(cv2.CAP_PROP_FRAME_COUNT))
+    print("fps: %d, frame_count: %d" % (fps, frame_count))
+
+    if not os.path.exists(FLAGS.output_dir):
+        os.makedirs(FLAGS.output_dir)
+    out_path = os.path.join(FLAGS.output_dir, video_name)
+    fourcc = cv2.VideoWriter_fourcc(*'mp4v')
+    writer = cv2.VideoWriter(out_path, fourcc, fps, (width, height))
+    index = 0
+    store_res = []
+    keypoint_smoothing = KeypointSmoothing(
+        width, height, filter_type=FLAGS.filter_type, beta=0.05)
+
+    while (1):
+        ret, frame = capture.read()
+        if not ret:
+            break
+        index += 1
+        print('detect frame: %d' % (index))
+
+        frame2 = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+
+        results = detector.predict_image([frame2], visual=False)
+        results = detector.filter_box(results, FLAGS.det_threshold)
+        if results['boxes_num'] == 0:
+            writer.write(frame)
+            continue
+
+        keypoint_res = predict_with_given_det(
+            frame2, results, topdown_keypoint_detector, keypoint_batch_size,
+            FLAGS.run_benchmark)
+
+        if FLAGS.smooth and len(keypoint_res['keypoint'][0]) == 1:
+            current_keypoints = np.array(keypoint_res['keypoint'][0][0])
+            smooth_keypoints = keypoint_smoothing.smooth_process(
+                current_keypoints)
+
+            keypoint_res['keypoint'][0][0] = smooth_keypoints.tolist()
+
+        im = visualize_pose(
+            frame,
+            keypoint_res,
+            visual_thresh=FLAGS.keypoint_threshold,
+            returnimg=True)
+
+        if save_res:
+            store_res.append([
+                index, keypoint_res['bbox'],
+                [keypoint_res['keypoint'][0], keypoint_res['keypoint'][1]]
+            ])
+
+        writer.write(im)
+        if camera_id != -1:
+            cv2.imshow('Mask Detection', im)
+            if cv2.waitKey(1) & 0xFF == ord('q'):
+                break
+    writer.release()
+    print('output_video saved to: {}'.format(out_path))
+    if save_res:
+        """
+        1) store_res: a list of frame_data
+        2) frame_data: [frameid, rects, [keypoints, scores]]
+        3) rects: list of rect [xmin, ymin, xmax, ymax]
+        4) keypoints: 17(joint numbers)*[x, y, conf], total 51 data in list
+        5) scores: mean of all joint conf
+        """
+        with open("det_keypoint_unite_video_results.json", 'w') as wf:
+            json.dump(store_res, wf, indent=4)
+
+
+class KeypointSmoothing(object):
+    # The following code are modified from:
+    # https://github.com/jaantollander/OneEuroFilter
+
+    def __init__(self,
+                 width,
+                 height,
+                 filter_type,
+                 alpha=0.5,
+                 fc_d=0.1,
+                 fc_min=0.1,
+                 beta=0.1,
+                 thres_mult=0.3):
+        super(KeypointSmoothing, self).__init__()
+        self.image_width = width
+        self.image_height = height
+        self.threshold = np.array([
+            0.005, 0.005, 0.005, 0.005, 0.005, 0.01, 0.01, 0.01, 0.01, 0.01,
+            0.01, 0.01, 0.01, 0.01, 0.01, 0.01, 0.01
+        ]) * thres_mult
+        self.filter_type = filter_type
+        self.alpha = alpha
+        self.dx_prev_hat = None
+        self.x_prev_hat = None
+        self.fc_d = fc_d
+        self.fc_min = fc_min
+        self.beta = beta
+
+        if self.filter_type == 'OneEuro':
+            self.smooth_func = self.one_euro_filter
+        elif self.filter_type == 'EMA':
+            self.smooth_func = self.ema_filter
+        else:
+            raise ValueError('filter type must be one_euro or ema')
+
+    def smooth_process(self, current_keypoints):
+        if self.x_prev_hat is None:
+            self.x_prev_hat = current_keypoints[:, :2]
+            self.dx_prev_hat = np.zeros(current_keypoints[:, :2].shape)
+            return current_keypoints
+        else:
+            result = current_keypoints
+            num_keypoints = len(current_keypoints)
+            for i in range(num_keypoints):
+                result[i, :2] = self.smooth(current_keypoints[i, :2],
+                                            self.threshold[i], i)
+            return result
+
+    def smooth(self, current_keypoint, threshold, index):
+        distance = np.sqrt(
+            np.square((current_keypoint[0] - self.x_prev_hat[index][0]) /
+                      self.image_width) + np.square((current_keypoint[
+                          1] - self.x_prev_hat[index][1]) / self.image_height))
+        if distance < threshold:
+            result = self.x_prev_hat[index]
+        else:
+            result = self.smooth_func(current_keypoint, self.x_prev_hat[index],
+                                      index)
+
+        return result
+
+    def one_euro_filter(self, x_cur, x_pre, index):
+        te = 1
+        self.alpha = self.smoothing_factor(te, self.fc_d)
+        dx_cur = (x_cur - x_pre) / te
+        dx_cur_hat = self.exponential_smoothing(dx_cur,
+                                                self.dx_prev_hat[index])
+
+        fc = self.fc_min + self.beta * np.abs(dx_cur_hat)
+        self.alpha = self.smoothing_factor(te, fc)
+        x_cur_hat = self.exponential_smoothing(x_cur, x_pre)
+        self.dx_prev_hat[index] = dx_cur_hat
+        self.x_prev_hat[index] = x_cur_hat
+        return x_cur_hat
+
+    def ema_filter(self, x_cur, x_pre, index):
+        x_cur_hat = self.exponential_smoothing(x_cur, x_pre)
+        self.x_prev_hat[index] = x_cur_hat
+        return x_cur_hat
+
+    def smoothing_factor(self, te, fc):
+        r = 2 * math.pi * fc * te
+        return r / (r + 1)
+
+    def exponential_smoothing(self, x_cur, x_pre, index=0):
+        return self.alpha * x_cur + (1 - self.alpha) * x_pre
+
+
+def main():
+    deploy_file = os.path.join(FLAGS.det_model_dir, 'infer_cfg.yml')
+    with open(deploy_file) as f:
+        yml_conf = yaml.safe_load(f)
+    arch = yml_conf['arch']
+    detector_func = 'Detector'
+    if arch == 'PicoDet':
+        detector_func = 'DetectorPicoDet'
+
+    detector = eval(detector_func)(FLAGS.det_model_dir,
+                                   device=FLAGS.device,
+                                   run_mode=FLAGS.run_mode,
+                                   trt_min_shape=FLAGS.trt_min_shape,
+                                   trt_max_shape=FLAGS.trt_max_shape,
+                                   trt_opt_shape=FLAGS.trt_opt_shape,
+                                   trt_calib_mode=FLAGS.trt_calib_mode,
+                                   cpu_threads=FLAGS.cpu_threads,
+                                   enable_mkldnn=FLAGS.enable_mkldnn,
+                                   threshold=FLAGS.det_threshold)
+
+    topdown_keypoint_detector = KeyPointDetector(
+        FLAGS.keypoint_model_dir,
+        device=FLAGS.device,
+        run_mode=FLAGS.run_mode,
+        batch_size=FLAGS.keypoint_batch_size,
+        trt_min_shape=FLAGS.trt_min_shape,
+        trt_max_shape=FLAGS.trt_max_shape,
+        trt_opt_shape=FLAGS.trt_opt_shape,
+        trt_calib_mode=FLAGS.trt_calib_mode,
+        cpu_threads=FLAGS.cpu_threads,
+        enable_mkldnn=FLAGS.enable_mkldnn,
+        use_dark=FLAGS.use_dark)
+    keypoint_arch = topdown_keypoint_detector.pred_config.arch
+    assert KEYPOINT_SUPPORT_MODELS[
+        keypoint_arch] == 'keypoint_topdown', 'Detection-Keypoint unite inference only supports topdown models.'
+
+    # predict from video file or camera video stream
+    if FLAGS.video_file is not None or FLAGS.camera_id != -1:
+        topdown_unite_predict_video(detector, topdown_keypoint_detector,
+                                    FLAGS.camera_id, FLAGS.keypoint_batch_size,
+                                    FLAGS.save_res)
+    else:
+        # predict from image
+        img_list = get_test_images(FLAGS.image_dir, FLAGS.image_file)
+        topdown_unite_predict(detector, topdown_keypoint_detector, img_list,
+                              FLAGS.keypoint_batch_size, FLAGS.save_res)
+        if not FLAGS.run_benchmark:
+            detector.det_times.info(average=True)
+            topdown_keypoint_detector.det_times.info(average=True)
+        else:
+            mode = FLAGS.run_mode
+            det_model_dir = FLAGS.det_model_dir
+            det_model_info = {
+                'model_name': det_model_dir.strip('/').split('/')[-1],
+                'precision': mode.split('_')[-1]
+            }
+            bench_log(detector, img_list, det_model_info, name='Det')
+            keypoint_model_dir = FLAGS.keypoint_model_dir
+            keypoint_model_info = {
+                'model_name': keypoint_model_dir.strip('/').split('/')[-1],
+                'precision': mode.split('_')[-1]
+            }
+            bench_log(topdown_keypoint_detector, img_list, keypoint_model_info,
+                      FLAGS.keypoint_batch_size, 'KeyPoint')
+
+
+if __name__ == '__main__':
+    paddle.enable_static()
+    parser = argsparser()
+    FLAGS = parser.parse_args()
+    print_arguments(FLAGS)
+    FLAGS.device = FLAGS.device.upper()
+    assert FLAGS.device in ['CPU', 'GPU', 'XPU'
+                            ], "device should be CPU, GPU or XPU"
+
+    main()

python/det_keypoint_unite_utils.py

@@ -0,0 +1,141 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import ast
+import argparse
+
+
+def argsparser():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--det_model_dir",
+        type=str,
+        default=None,
+        help=("Directory include:'model.pdiparams', 'model.pdmodel', "
+              "'infer_cfg.yml', created by tools/export_model.py."),
+        required=True)
+    parser.add_argument(
+        "--keypoint_model_dir",
+        type=str,
+        default=None,
+        help=("Directory include:'model.pdiparams', 'model.pdmodel', "
+              "'infer_cfg.yml', created by tools/export_model.py."),
+        required=True)
+    parser.add_argument(
+        "--image_file", type=str, default=None, help="Path of image file.")
+    parser.add_argument(
+        "--image_dir",
+        type=str,
+        default=None,
+        help="Dir of image file, `image_file` has a higher priority.")
+    parser.add_argument(
+        "--keypoint_batch_size",
+        type=int,
+        default=8,
+        help=("batch_size for keypoint inference. In detection-keypoint unit"
+              "inference, the batch size in detection is 1. Then collate det "
+              "result in batch for keypoint inference."))
+    parser.add_argument(
+        "--video_file",
+        type=str,
+        default=None,
+        help="Path of video file, `video_file` or `camera_id` has a highest priority."
+    )
+    parser.add_argument(
+        "--camera_id",
+        type=int,
+        default=-1,
+        help="device id of camera to predict.")
+    parser.add_argument(
+        "--det_threshold", type=float, default=0.5, help="Threshold of score.")
+    parser.add_argument(
+        "--keypoint_threshold",
+        type=float,
+        default=0.5,
+        help="Threshold of score.")
+    parser.add_argument(
+        "--output_dir",
+        type=str,
+        default="output",
+        help="Directory of output visualization files.")
+    parser.add_argument(
+        "--run_mode",
+        type=str,
+        default='paddle',
+        help="mode of running(paddle/trt_fp32/trt_fp16/trt_int8)")
+    parser.add_argument(
+        "--device",
+        type=str,
+        default='cpu',
+        help="Choose the device you want to run, it can be: CPU/GPU/XPU, default is CPU."
+    )
+    parser.add_argument(
+        "--run_benchmark",
+        type=ast.literal_eval,
+        default=False,
+        help="Whether to predict a image_file repeatedly for benchmark")
+    parser.add_argument(
+        "--enable_mkldnn",
+        type=ast.literal_eval,
+        default=False,
+        help="Whether use mkldnn with CPU.")
+    parser.add_argument(
+        "--cpu_threads", type=int, default=1, help="Num of threads with CPU.")
+    parser.add_argument(
+        "--trt_min_shape", type=int, default=1, help="min_shape for TensorRT.")
+    parser.add_argument(
+        "--trt_max_shape",
+        type=int,
+        default=1280,
+        help="max_shape for TensorRT.")
+    parser.add_argument(
+        "--trt_opt_shape",
+        type=int,
+        default=640,
+        help="opt_shape for TensorRT.")
+    parser.add_argument(
+        "--trt_calib_mode",
+        type=bool,
+        default=False,
+        help="If the model is produced by TRT offline quantitative "
+        "calibration, trt_calib_mode need to set True.")
+    parser.add_argument(
+        '--use_dark',
+        type=ast.literal_eval,
+        default=True,
+        help='whether to use darkpose to get better keypoint position predict ')
+    parser.add_argument(
+        '--save_res',
+        type=bool,
+        default=False,
+        help=(
+            "whether to save predict results to json file"
+            "1) store_res: a list of image_data"
+            "2) image_data: [imageid, rects, [keypoints, scores]]"
+            "3) rects: list of rect [xmin, ymin, xmax, ymax]"
+            "4) keypoints: 17(joint numbers)*[x, y, conf], total 51 data in list"
+            "5) scores: mean of all joint conf"))
+    parser.add_argument(
+        '--smooth',
+        type=ast.literal_eval,
+        default=False,
+        help='smoothing keypoints for each frame, new incoming keypoints will be more stable.'
+    )
+    parser.add_argument(
+        '--filter_type',
+        type=str,
+        default='OneEuro',
+        help='when set --smooth True, choose filter type you want to use, it can be [OneEuro] or [EMA].'
+    )
+    return parser

python/infer.py

@@ -0,0 +1,1035 @@
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import yaml
+import glob
+import json
+from pathlib import Path
+from functools import reduce
+
+import cv2
+import numpy as np
+import math
+import paddle
+from paddle.inference import Config
+from paddle.inference import create_predictor
+
+import sys
+# add deploy path of PadleDetection to sys.path
+parent_path = os.path.abspath(os.path.join(__file__, *(['..'])))
+sys.path.insert(0, parent_path)
+
+from benchmark_utils import PaddleInferBenchmark
+from picodet_postprocess import PicoDetPostProcess
+from preprocess import preprocess, Resize, NormalizeImage, Permute, PadStride, LetterBoxResize, WarpAffine, Pad, decode_image
+from keypoint_preprocess import EvalAffine, TopDownEvalAffine, expand_crop
+from visualize import visualize_box_mask
+from utils import argsparser, Timer, get_current_memory_mb, multiclass_nms, coco_clsid2catid
+
+# Global dictionary
+SUPPORT_MODELS = {
+    'YOLO', 'RCNN', 'SSD', 'Face', 'FCOS', 'SOLOv2', 'TTFNet', 'S2ANet', 'JDE',
+    'FairMOT', 'DeepSORT', 'GFL', 'PicoDet', 'CenterNet', 'TOOD', 'RetinaNet',
+    'StrongBaseline', 'STGCN', 'YOLOX', 'PPHGNet', 'PPLCNet'
+}
+
+
+def bench_log(detector, img_list, model_info, batch_size=1, name=None):
+    mems = {
+        'cpu_rss_mb': detector.cpu_mem / len(img_list),
+        'gpu_rss_mb': detector.gpu_mem / len(img_list),
+        'gpu_util': detector.gpu_util * 100 / len(img_list)
+    }
+    perf_info = detector.det_times.report(average=True)
+    data_info = {
+        'batch_size': batch_size,
+        'shape': "dynamic_shape",
+        'data_num': perf_info['img_num']
+    }
+    log = PaddleInferBenchmark(detector.config, model_info, data_info,
+                               perf_info, mems)
+    log(name)
+
+
+class Detector(object):
+    """
+    Args:
+        pred_config (object): config of model, defined by `Config(model_dir)`
+        model_dir (str): root path of model.pdiparams, model.pdmodel and infer_cfg.yml
+        device (str): Choose the device you want to run, it can be: CPU/GPU/XPU, default is CPU
+        run_mode (str): mode of running(paddle/trt_fp32/trt_fp16)
+        batch_size (int): size of pre batch in inference
+        trt_min_shape (int): min shape for dynamic shape in trt
+        trt_max_shape (int): max shape for dynamic shape in trt
+        trt_opt_shape (int): opt shape for dynamic shape in trt
+        trt_calib_mode (bool): If the model is produced by TRT offline quantitative
+            calibration, trt_calib_mode need to set True
+        cpu_threads (int): cpu threads
+        enable_mkldnn (bool): whether to open MKLDNN
+        enable_mkldnn_bfloat16 (bool): whether to turn on mkldnn bfloat16
+        output_dir (str): The path of output
+        threshold (float): The threshold of score for visualization
+        delete_shuffle_pass (bool): whether to remove shuffle_channel_detect_pass in TensorRT. 
+                                    Used by action model.
+    """
+
+    def __init__(self,
+                 model_dir,
+                 device='CPU',
+                 run_mode='paddle',
+                 batch_size=1,
+                 trt_min_shape=1,
+                 trt_max_shape=1280,
+                 trt_opt_shape=640,
+                 trt_calib_mode=False,
+                 cpu_threads=1,
+                 enable_mkldnn=False,
+                 enable_mkldnn_bfloat16=False,
+                 output_dir='output',
+                 threshold=0.5,
+                 delete_shuffle_pass=False):
+        self.pred_config = self.set_config(model_dir)
+        self.predictor, self.config = load_predictor(
+            model_dir,
+            run_mode=run_mode,
+            batch_size=batch_size,
+            min_subgraph_size=self.pred_config.min_subgraph_size,
+            device=device,
+            use_dynamic_shape=self.pred_config.use_dynamic_shape,
+            trt_min_shape=trt_min_shape,
+            trt_max_shape=trt_max_shape,
+            trt_opt_shape=trt_opt_shape,
+            trt_calib_mode=trt_calib_mode,
+            cpu_threads=cpu_threads,
+            enable_mkldnn=enable_mkldnn,
+            enable_mkldnn_bfloat16=enable_mkldnn_bfloat16,
+            delete_shuffle_pass=delete_shuffle_pass)
+        self.det_times = Timer()
+        self.cpu_mem, self.gpu_mem, self.gpu_util = 0, 0, 0
+        self.batch_size = batch_size
+        self.output_dir = output_dir
+        self.threshold = threshold
+
+    def set_config(self, model_dir):
+        return PredictConfig(model_dir)
+
+    def preprocess(self, image_list):
+        preprocess_ops = []
+        for op_info in self.pred_config.preprocess_infos:
+            new_op_info = op_info.copy()
+            op_type = new_op_info.pop('type')
+            preprocess_ops.append(eval(op_type)(**new_op_info))
+
+        input_im_lst = []
+        input_im_info_lst = []
+        for im_path in image_list:
+            im, im_info = preprocess(im_path, preprocess_ops)
+            input_im_lst.append(im)
+            input_im_info_lst.append(im_info)
+        inputs = create_inputs(input_im_lst, input_im_info_lst)
+        input_names = self.predictor.get_input_names()
+        for i in range(len(input_names)):
+            input_tensor = self.predictor.get_input_handle(input_names[i])
+            if input_names[i] == 'x':
+                input_tensor.copy_from_cpu(inputs['image'])
+            else:
+                input_tensor.copy_from_cpu(inputs[input_names[i]])
+
+        return inputs
+
+    def postprocess(self, inputs, result):
+        # postprocess output of predictor
+        np_boxes_num = result['boxes_num']
+        assert isinstance(np_boxes_num, np.ndarray), \
+            '`np_boxes_num` should be a `numpy.ndarray`'
+
+        if np_boxes_num.sum() <= 0:
+            print('[WARNNING] No object detected.')
+            result = {'boxes': np.zeros([0, 6]), 'boxes_num': np_boxes_num}
+        result = {k: v for k, v in result.items() if v is not None}
+        return result
+
+    def filter_box(self, result, threshold):
+        np_boxes_num = result['boxes_num']
+        boxes = result['boxes']
+        start_idx = 0
+        filter_boxes = []
+        filter_num = []
+        for i in range(len(np_boxes_num)):
+            boxes_num = np_boxes_num[i]
+            boxes_i = boxes[start_idx:start_idx + boxes_num, :]
+            idx = boxes_i[:, 1] > threshold
+            filter_boxes_i = boxes_i[idx, :]
+            filter_boxes.append(filter_boxes_i)
+            filter_num.append(filter_boxes_i.shape[0])
+            start_idx += boxes_num
+        boxes = np.concatenate(filter_boxes)
+        filter_num = np.array(filter_num)
+        filter_res = {'boxes': boxes, 'boxes_num': filter_num}
+        return filter_res
+
+    def predict(self, repeats=1):
+        '''
+        Args:
+            repeats (int): repeats number for prediction
+        Returns:
+            result (dict): include 'boxes': np.ndarray: shape:[N,6], N: number of box,
+                            matix element:[class, score, x_min, y_min, x_max, y_max]
+                            MaskRCNN's result include 'masks': np.ndarray:
+                            shape: [N, im_h, im_w]
+        '''
+        # model prediction
+        np_boxes_num, np_boxes, np_masks = np.array([0]), None, None
+        for i in range(repeats):
+            self.predictor.run()
+            output_names = self.predictor.get_output_names()
+            boxes_tensor = self.predictor.get_output_handle(output_names[0])
+            np_boxes = boxes_tensor.copy_to_cpu()
+            boxes_num = self.predictor.get_output_handle(output_names[1])
+            np_boxes_num = boxes_num.copy_to_cpu()
+            if self.pred_config.mask:
+                masks_tensor = self.predictor.get_output_handle(output_names[
+                    2])
+                np_masks = masks_tensor.copy_to_cpu()
+        result = dict(boxes=np_boxes, masks=np_masks, boxes_num=np_boxes_num)
+        return result
+
+    def merge_batch_result(self, batch_result):
+        if len(batch_result) == 1:
+            return batch_result[0]
+        res_key = batch_result[0].keys()
+        results = {k: [] for k in res_key}
+        for res in batch_result:
+            for k, v in res.items():
+                results[k].append(v)
+        for k, v in results.items():
+            if k not in ['masks', 'segm']:
+                results[k] = np.concatenate(v)
+        return results
+
+    def get_timer(self):
+        return self.det_times
+
+    def predict_image_slice(self,
+                            img_list,
+                            slice_size=[640, 640],
+                            overlap_ratio=[0.25, 0.25],
+                            combine_method='nms',
+                            match_threshold=0.6,
+                            match_metric='ios',
+                            run_benchmark=False,
+                            repeats=1,
+                            visual=True,
+                            save_results=False):
+        # slice infer only support bs=1
+        results = []
+        try:
+            import sahi
+            from sahi.slicing import slice_image
+        except Exception as e:
+            print(
+                'sahi not found, plaese install sahi. '
+                'for example: `pip install sahi`, see https://github.com/obss/sahi.'
+            )
+            raise e
+        num_classes = len(self.pred_config.labels)
+        for i in range(len(img_list)):
+            ori_image = img_list[i]
+            slice_image_result = sahi.slicing.slice_image(
+                image=ori_image,
+                slice_height=slice_size[0],
+                slice_width=slice_size[1],
+                overlap_height_ratio=overlap_ratio[0],
+                overlap_width_ratio=overlap_ratio[1])
+            sub_img_num = len(slice_image_result)
+            merged_bboxs = []
+            print('sub_img_num', sub_img_num)
+
+            batch_image_list = [
+                slice_image_result.images[_ind] for _ind in range(sub_img_num)
+            ]
+            if run_benchmark:
+                # preprocess
+                inputs = self.preprocess(batch_image_list)  # warmup
+                self.det_times.preprocess_time_s.start()
+                inputs = self.preprocess(batch_image_list)
+                self.det_times.preprocess_time_s.end()
+
+                # model prediction
+                result = self.predict(repeats=50)  # warmup
+                self.det_times.inference_time_s.start()
+                result = self.predict(repeats=repeats)
+                self.det_times.inference_time_s.end(repeats=repeats)
+
+                # postprocess
+                result_warmup = self.postprocess(inputs, result)  # warmup
+                self.det_times.postprocess_time_s.start()
+                result = self.postprocess(inputs, result)
+                self.det_times.postprocess_time_s.end()
+                self.det_times.img_num += 1
+
+                cm, gm, gu = get_current_memory_mb()
+                self.cpu_mem += cm
+                self.gpu_mem += gm
+                self.gpu_util += gu
+            else:
+                # preprocess
+                self.det_times.preprocess_time_s.start()
+                inputs = self.preprocess(batch_image_list)
+                self.det_times.preprocess_time_s.end()
+
+                # model prediction
+                self.det_times.inference_time_s.start()
+                result = self.predict()
+                self.det_times.inference_time_s.end()
+
+                # postprocess
+                self.det_times.postprocess_time_s.start()
+                result = self.postprocess(inputs, result)
+                self.det_times.postprocess_time_s.end()
+                self.det_times.img_num += 1
+
+            st, ed = 0, result['boxes_num'][0]  # start_index, end_index
+            for _ind in range(sub_img_num):
+                boxes_num = result['boxes_num'][_ind]
+                ed = boxes_num
+                shift_amount = slice_image_result.starting_pixels[_ind]
+                result['boxes'][st:ed][:, 2:4] = result['boxes'][
+                    st:ed][:, 2:4] + shift_amount
+                result['boxes'][st:ed][:, 4:6] = result['boxes'][
+                    st:ed][:, 4:6] + shift_amount
+                merged_bboxs.append(result['boxes'][st:ed])
+                st = ed
+
+            merged_results = {'boxes': []}
+            if combine_method == 'nms':
+                final_boxes = multiclass_nms(
+                    np.concatenate(merged_bboxs), num_classes, match_threshold,
+                    match_metric)
+                merged_results['boxes'] = np.concatenate(final_boxes)
+            elif combine_method == 'concat':
+                merged_results['boxes'] = np.concatenate(merged_bboxs)
+            else:
+                raise ValueError(
+                    "Now only support 'nms' or 'concat' to fuse detection results."
+                )
+            merged_results['boxes_num'] = np.array(
+                [len(merged_results['boxes'])], dtype=np.int32)
+
+            if visual:
+                visualize(
+                    [ori_image],  # should be list
+                    merged_results,
+                    self.pred_config.labels,
+                    output_dir=self.output_dir,
+                    threshold=self.threshold)
+
+            results.append(merged_results)
+            print('Test iter {}'.format(i))
+
+        results = self.merge_batch_result(results)
+        if save_results:
+            Path(self.output_dir).mkdir(exist_ok=True)
+            self.save_coco_results(
+                img_list, results, use_coco_category=FLAGS.use_coco_category)
+        return results
+
+    def predict_image(self,
+                      image_list,
+                      run_benchmark=False,
+                      repeats=1,
+                      visual=True,
+                      save_results=False):
+        batch_loop_cnt = math.ceil(float(len(image_list)) / self.batch_size)
+        results = []
+        for i in range(batch_loop_cnt):
+            start_index = i * self.batch_size
+            end_index = min((i + 1) * self.batch_size, len(image_list))
+            batch_image_list = image_list[start_index:end_index]
+            if run_benchmark:
+                # preprocess
+                inputs = self.preprocess(batch_image_list)  # warmup
+                self.det_times.preprocess_time_s.start()
+                inputs = self.preprocess(batch_image_list)
+                self.det_times.preprocess_time_s.end()
+
+                # model prediction
+                result = self.predict(repeats=50)  # warmup
+                self.det_times.inference_time_s.start()
+                result = self.predict(repeats=repeats)
+                self.det_times.inference_time_s.end(repeats=repeats)
+
+                # postprocess
+                result_warmup = self.postprocess(inputs, result)  # warmup
+                self.det_times.postprocess_time_s.start()
+                result = self.postprocess(inputs, result)
+                self.det_times.postprocess_time_s.end()
+                self.det_times.img_num += len(batch_image_list)
+
+                cm, gm, gu = get_current_memory_mb()
+                self.cpu_mem += cm
+                self.gpu_mem += gm
+                self.gpu_util += gu
+            else:
+                # preprocess
+                self.det_times.preprocess_time_s.start()
+                inputs = self.preprocess(batch_image_list)
+                self.det_times.preprocess_time_s.end()
+
+                # model prediction
+                self.det_times.inference_time_s.start()
+                result = self.predict()
+                self.det_times.inference_time_s.end()
+
+                # postprocess
+                self.det_times.postprocess_time_s.start()
+                result = self.postprocess(inputs, result)
+                self.det_times.postprocess_time_s.end()
+                self.det_times.img_num += len(batch_image_list)
+
+                if visual:
+                    visualize(
+                        batch_image_list,
+                        result,
+                        self.pred_config.labels,
+                        output_dir=self.output_dir,
+                        threshold=self.threshold)
+            results.append(result)
+            print('Test iter {}'.format(i))
+        results = self.merge_batch_result(results)
+        if save_results:
+            Path(self.output_dir).mkdir(exist_ok=True)
+            self.save_coco_results(
+                image_list, results, use_coco_category=FLAGS.use_coco_category)
+        return results
+
+    def predict_video(self, video_file, camera_id):
+        video_out_name = 'output.mp4'
+        if camera_id != -1:
+            capture = cv2.VideoCapture(camera_id)
+        else:
+            capture = cv2.VideoCapture(video_file)
+            video_out_name = os.path.split(video_file)[-1]
+        # Get Video info : resolution, fps, frame count
+        width = int(capture.get(cv2.CAP_PROP_FRAME_WIDTH))
+        height = int(capture.get(cv2.CAP_PROP_FRAME_HEIGHT))
+        fps = int(capture.get(cv2.CAP_PROP_FPS))
+        frame_count = int(capture.get(cv2.CAP_PROP_FRAME_COUNT))
+        print("fps: %d, frame_count: %d" % (fps, frame_count))
+
+        if not os.path.exists(self.output_dir):
+            os.makedirs(self.output_dir)
+        out_path = os.path.join(self.output_dir, video_out_name)
+        fourcc = cv2.VideoWriter_fourcc(*'mp4v')
+        writer = cv2.VideoWriter(out_path, fourcc, fps, (width, height))
+        index = 1
+        while (1):
+            ret, frame = capture.read()
+            if not ret:
+                break
+            print('detect frame: %d' % (index))
+            index += 1
+            results = self.predict_image([frame[:, :, ::-1]], visual=False)
+
+            im = visualize_box_mask(
+                frame,
+                results,
+                self.pred_config.labels,
+                threshold=self.threshold)
+            im = np.array(im)
+            writer.write(im)
+            if camera_id != -1:
+                cv2.imshow('Mask Detection', im)
+                if cv2.waitKey(1) & 0xFF == ord('q'):
+                    break
+        writer.release()
+
+    def save_coco_results(self, image_list, results, use_coco_category=False):
+        bbox_results = []
+        mask_results = []
+        idx = 0
+        print("Start saving coco json files...")
+        for i, box_num in enumerate(results['boxes_num']):
+            file_name = os.path.split(image_list[i])[-1]
+            if use_coco_category:
+                img_id = int(os.path.splitext(file_name)[0])
+            else:
+                img_id = i
+
+            if 'boxes' in results:
+                boxes = results['boxes'][idx:idx + box_num].tolist()
+                bbox_results.extend([{
+                    'image_id': img_id,
+                    'category_id': coco_clsid2catid[int(box[0])] \
+                        if use_coco_category else int(box[0]),
+                    'file_name': file_name,
+                    'bbox': [box[2], box[3], box[4] - box[2],
+                         box[5] - box[3]],  # xyxy -> xywh
+                    'score': box[1]} for box in boxes])
+
+            if 'masks' in results:
+                import pycocotools.mask as mask_util
+
+                boxes = results['boxes'][idx:idx + box_num].tolist()
+                masks = results['masks'][i][:box_num].astype(np.uint8)
+                seg_res = []
+                for box, mask in zip(boxes, masks):
+                    rle = mask_util.encode(
+                        np.array(
+                            mask[:, :, None], dtype=np.uint8, order="F"))[0]
+                    if 'counts' in rle:
+                        rle['counts'] = rle['counts'].decode("utf8")
+                    seg_res.append({
+                        'image_id': img_id,
+                        'category_id': coco_clsid2catid[int(box[0])] \
+                        if use_coco_category else int(box[0]),
+                        'file_name': file_name,
+                        'segmentation': rle,
+                        'score': box[1]})
+                mask_results.extend(seg_res)
+
+            idx += box_num
+
+        if bbox_results:
+            bbox_file = os.path.join(self.output_dir, "bbox.json")
+            with open(bbox_file, 'w') as f:
+                json.dump(bbox_results, f)
+            print(f"The bbox result is saved to {bbox_file}")
+        if mask_results:
+            mask_file = os.path.join(self.output_dir, "mask.json")
+            with open(mask_file, 'w') as f:
+                json.dump(mask_results, f)
+            print(f"The mask result is saved to {mask_file}")
+
+
+class DetectorSOLOv2(Detector):
+    """
+    Args:
+        model_dir (str): root path of model.pdiparams, model.pdmodel and infer_cfg.yml
+        device (str): Choose the device you want to run, it can be: CPU/GPU/XPU, default is CPU
+        run_mode (str): mode of running(paddle/trt_fp32/trt_fp16)
+        batch_size (int): size of pre batch in inference
+        trt_min_shape (int): min shape for dynamic shape in trt
+        trt_max_shape (int): max shape for dynamic shape in trt
+        trt_opt_shape (int): opt shape for dynamic shape in trt
+        trt_calib_mode (bool): If the model is produced by TRT offline quantitative
+            calibration, trt_calib_mode need to set True
+        cpu_threads (int): cpu threads
+        enable_mkldnn (bool): whether to open MKLDNN 
+        enable_mkldnn_bfloat16 (bool): Whether to turn on mkldnn bfloat16
+        output_dir (str): The path of output
+        threshold (float): The threshold of score for visualization
+       
+    """
+
+    def __init__(
+            self,
+            model_dir,
+            device='CPU',
+            run_mode='paddle',
+            batch_size=1,
+            trt_min_shape=1,
+            trt_max_shape=1280,
+            trt_opt_shape=640,
+            trt_calib_mode=False,
+            cpu_threads=1,
+            enable_mkldnn=False,
+            enable_mkldnn_bfloat16=False,
+            output_dir='./',
+            threshold=0.5, ):
+        super(DetectorSOLOv2, self).__init__(
+            model_dir=model_dir,
+            device=device,
+            run_mode=run_mode,
+            batch_size=batch_size,
+            trt_min_shape=trt_min_shape,
+            trt_max_shape=trt_max_shape,
+            trt_opt_shape=trt_opt_shape,
+            trt_calib_mode=trt_calib_mode,
+            cpu_threads=cpu_threads,
+            enable_mkldnn=enable_mkldnn,
+            enable_mkldnn_bfloat16=enable_mkldnn_bfloat16,
+            output_dir=output_dir,
+            threshold=threshold, )
+
+    def predict(self, repeats=1):
+        '''
+        Args:
+            repeats (int): repeat number for prediction
+        Returns:
+            result (dict): 'segm': np.ndarray,shape:[N, im_h, im_w]
+                            'cate_label': label of segm, shape:[N]
+                            'cate_score': confidence score of segm, shape:[N]
+        '''
+        np_label, np_score, np_segms = None, None, None
+        for i in range(repeats):
+            self.predictor.run()
+            output_names = self.predictor.get_output_names()
+            np_boxes_num = self.predictor.get_output_handle(output_names[
+                0]).copy_to_cpu()
+            np_label = self.predictor.get_output_handle(output_names[
+                1]).copy_to_cpu()
+            np_score = self.predictor.get_output_handle(output_names[
+                2]).copy_to_cpu()
+            np_segms = self.predictor.get_output_handle(output_names[
+                3]).copy_to_cpu()
+
+        result = dict(
+            segm=np_segms,
+            label=np_label,
+            score=np_score,
+            boxes_num=np_boxes_num)
+        return result
+
+
+class DetectorPicoDet(Detector):
+    """
+    Args:
+        model_dir (str): root path of model.pdiparams, model.pdmodel and infer_cfg.yml
+        device (str): Choose the device you want to run, it can be: CPU/GPU/XPU, default is CPU
+        run_mode (str): mode of running(paddle/trt_fp32/trt_fp16)
+        batch_size (int): size of pre batch in inference
+        trt_min_shape (int): min shape for dynamic shape in trt
+        trt_max_shape (int): max shape for dynamic shape in trt
+        trt_opt_shape (int): opt shape for dynamic shape in trt
+        trt_calib_mode (bool): If the model is produced by TRT offline quantitative
+            calibration, trt_calib_mode need to set True
+        cpu_threads (int): cpu threads
+        enable_mkldnn (bool): whether to turn on MKLDNN
+        enable_mkldnn_bfloat16 (bool): whether to turn on MKLDNN_BFLOAT16
+    """
+
+    def __init__(
+            self,
+            model_dir,
+            device='CPU',
+            run_mode='paddle',
+            batch_size=1,
+            trt_min_shape=1,
+            trt_max_shape=1280,
+            trt_opt_shape=640,
+            trt_calib_mode=False,
+            cpu_threads=1,
+            enable_mkldnn=False,
+            enable_mkldnn_bfloat16=False,
+            output_dir='./',
+            threshold=0.5, ):
+        super(DetectorPicoDet, self).__init__(
+            model_dir=model_dir,
+            device=device,
+            run_mode=run_mode,
+            batch_size=batch_size,
+            trt_min_shape=trt_min_shape,
+            trt_max_shape=trt_max_shape,
+            trt_opt_shape=trt_opt_shape,
+            trt_calib_mode=trt_calib_mode,
+            cpu_threads=cpu_threads,
+            enable_mkldnn=enable_mkldnn,
+            enable_mkldnn_bfloat16=enable_mkldnn_bfloat16,
+            output_dir=output_dir,
+            threshold=threshold, )
+
+    def postprocess(self, inputs, result):
+        # postprocess output of predictor
+        np_score_list = result['boxes']
+        np_boxes_list = result['boxes_num']
+        postprocessor = PicoDetPostProcess(
+            inputs['image'].shape[2:],
+            inputs['im_shape'],
+            inputs['scale_factor'],
+            strides=self.pred_config.fpn_stride,
+            nms_threshold=self.pred_config.nms['nms_threshold'])
+        np_boxes, np_boxes_num = postprocessor(np_score_list, np_boxes_list)
+        result = dict(boxes=np_boxes, boxes_num=np_boxes_num)
+        return result
+
+    def predict(self, repeats=1):
+        '''
+        Args:
+            repeats (int): repeat number for prediction
+        Returns:
+            result (dict): include 'boxes': np.ndarray: shape:[N,6], N: number of box,
+                            matix element:[class, score, x_min, y_min, x_max, y_max]
+        '''
+        np_score_list, np_boxes_list = [], []
+        for i in range(repeats):
+            self.predictor.run()
+            np_score_list.clear()
+            np_boxes_list.clear()
+            output_names = self.predictor.get_output_names()
+            num_outs = int(len(output_names) / 2)
+            for out_idx in range(num_outs):
+                np_score_list.append(
+                    self.predictor.get_output_handle(output_names[out_idx])
+                    .copy_to_cpu())
+                np_boxes_list.append(
+                    self.predictor.get_output_handle(output_names[
+                        out_idx + num_outs]).copy_to_cpu())
+        result = dict(boxes=np_score_list, boxes_num=np_boxes_list)
+        return result
+
+
+def create_inputs(imgs, im_info):
+    """generate input for different model type
+    Args:
+        imgs (list(numpy)): list of images (np.ndarray)
+        im_info (list(dict)): list of image info
+    Returns:
+        inputs (dict): input of model
+    """
+    inputs = {}
+
+    im_shape = []
+    scale_factor = []
+    if len(imgs) == 1:
+        inputs['image'] = np.array((imgs[0], )).astype('float32')
+        inputs['im_shape'] = np.array(
+            (im_info[0]['im_shape'], )).astype('float32')
+        inputs['scale_factor'] = np.array(
+            (im_info[0]['scale_factor'], )).astype('float32')
+        return inputs
+
+    for e in im_info:
+        im_shape.append(np.array((e['im_shape'], )).astype('float32'))
+        scale_factor.append(np.array((e['scale_factor'], )).astype('float32'))
+
+    inputs['im_shape'] = np.concatenate(im_shape, axis=0)
+    inputs['scale_factor'] = np.concatenate(scale_factor, axis=0)
+
+    imgs_shape = [[e.shape[1], e.shape[2]] for e in imgs]
+    max_shape_h = max([e[0] for e in imgs_shape])
+    max_shape_w = max([e[1] for e in imgs_shape])
+    padding_imgs = []
+    for img in imgs:
+        im_c, im_h, im_w = img.shape[:]
+        padding_im = np.zeros(
+            (im_c, max_shape_h, max_shape_w), dtype=np.float32)
+        padding_im[:, :im_h, :im_w] = img
+        padding_imgs.append(padding_im)
+    inputs['image'] = np.stack(padding_imgs, axis=0)
+    return inputs
+
+
+class PredictConfig():
+    """set config of preprocess, postprocess and visualize
+    Args:
+        model_dir (str): root path of model.yml
+    """
+
+    def __init__(self, model_dir):
+        # parsing Yaml config for Preprocess
+        deploy_file = os.path.join(model_dir, 'infer_cfg.yml')
+        with open(deploy_file) as f:
+            yml_conf = yaml.safe_load(f)
+        self.check_model(yml_conf)
+        self.arch = yml_conf['arch']
+        self.preprocess_infos = yml_conf['Preprocess']
+        self.min_subgraph_size = yml_conf['min_subgraph_size']
+        self.labels = yml_conf['label_list']
+        self.mask = False
+        self.use_dynamic_shape = yml_conf['use_dynamic_shape']
+        if 'mask' in yml_conf:
+            self.mask = yml_conf['mask']
+        self.tracker = None
+        if 'tracker' in yml_conf:
+            self.tracker = yml_conf['tracker']
+        if 'NMS' in yml_conf:
+            self.nms = yml_conf['NMS']
+        if 'fpn_stride' in yml_conf:
+            self.fpn_stride = yml_conf['fpn_stride']
+        if self.arch == 'RCNN' and yml_conf.get('export_onnx', False):
+            print(
+                'The RCNN export model is used for ONNX and it only supports batch_size = 1'
+            )
+        self.print_config()
+
+    def check_model(self, yml_conf):
+        """
+        Raises:
+            ValueError: loaded model not in supported model type 
+        """
+        for support_model in SUPPORT_MODELS:
+            if support_model in yml_conf['arch']:
+                return True
+        raise ValueError("Unsupported arch: {}, expect {}".format(yml_conf[
+            'arch'], SUPPORT_MODELS))
+
+    def print_config(self):
+        print('-----------  Model Configuration -----------')
+        print('%s: %s' % ('Model Arch', self.arch))
+        print('%s: ' % ('Transform Order'))
+        for op_info in self.preprocess_infos:
+            print('--%s: %s' % ('transform op', op_info['type']))
+        print('--------------------------------------------')
+
+
+def load_predictor(model_dir,
+                   run_mode='paddle',
+                   batch_size=1,
+                   device='CPU',
+                   min_subgraph_size=3,
+                   use_dynamic_shape=False,
+                   trt_min_shape=1,
+                   trt_max_shape=1280,
+                   trt_opt_shape=640,
+                   trt_calib_mode=False,
+                   cpu_threads=1,
+                   enable_mkldnn=False,
+                   enable_mkldnn_bfloat16=False,
+                   delete_shuffle_pass=False):
+    """set AnalysisConfig, generate AnalysisPredictor
+    Args:
+        model_dir (str): root path of __model__ and __params__
+        device (str): Choose the device you want to run, it can be: CPU/GPU/XPU, default is CPU
+        run_mode (str): mode of running(paddle/trt_fp32/trt_fp16/trt_int8)
+        use_dynamic_shape (bool): use dynamic shape or not
+        trt_min_shape (int): min shape for dynamic shape in trt
+        trt_max_shape (int): max shape for dynamic shape in trt
+        trt_opt_shape (int): opt shape for dynamic shape in trt
+        trt_calib_mode (bool): If the model is produced by TRT offline quantitative
+            calibration, trt_calib_mode need to set True
+        delete_shuffle_pass (bool): whether to remove shuffle_channel_detect_pass in TensorRT. 
+                                    Used by action model.
+    Returns:
+        predictor (PaddlePredictor): AnalysisPredictor
+    Raises:
+        ValueError: predict by TensorRT need device == 'GPU'.
+    """
+    if device != 'GPU' and run_mode != 'paddle':
+        raise ValueError(
+            "Predict by TensorRT mode: {}, expect device=='GPU', but device == {}"
+            .format(run_mode, device))
+    infer_model = os.path.join(model_dir, 'model.pdmodel')
+    infer_params = os.path.join(model_dir, 'model.pdiparams')
+    if not os.path.exists(infer_model):
+        infer_model = os.path.join(model_dir, 'inference.pdmodel')
+        infer_params = os.path.join(model_dir, 'inference.pdiparams')
+        if not os.path.exists(infer_model):
+            raise ValueError("Cannot find any inference model in dir: {},".
+                             format(model_dir))
+    config = Config(infer_model, infer_params)
+    if device == 'GPU':
+        # initial GPU memory(M), device ID
+        config.enable_use_gpu(200, 0)
+        # optimize graph and fuse op
+        config.switch_ir_optim(True)
+    elif device == 'XPU':
+        config.enable_lite_engine()
+        config.enable_xpu(10 * 1024 * 1024)
+    else:
+        config.disable_gpu()
+        config.set_cpu_math_library_num_threads(cpu_threads)
+        if enable_mkldnn:
+            try:
+                # cache 10 different shapes for mkldnn to avoid memory leak
+                config.set_mkldnn_cache_capacity(10)
+                config.enable_mkldnn()
+                if enable_mkldnn_bfloat16:
+                    config.enable_mkldnn_bfloat16()
+            except Exception as e:
+                print(
+                    "The current environment does not support `mkldnn`, so disable mkldnn."
+                )
+                pass
+
+    precision_map = {
+        'trt_int8': Config.Precision.Int8,
+        'trt_fp32': Config.Precision.Float32,
+        'trt_fp16': Config.Precision.Half
+    }
+    if run_mode in precision_map.keys():
+        config.enable_tensorrt_engine(
+            workspace_size=(1 << 25) * batch_size,
+            max_batch_size=batch_size,
+            min_subgraph_size=min_subgraph_size,
+            precision_mode=precision_map[run_mode],
+            use_static=False,
+            use_calib_mode=trt_calib_mode)
+
+        if use_dynamic_shape:
+            min_input_shape = {
+                'image': [batch_size, 3, trt_min_shape, trt_min_shape]
+            }
+            max_input_shape = {
+                'image': [batch_size, 3, trt_max_shape, trt_max_shape]
+            }
+            opt_input_shape = {
+                'image': [batch_size, 3, trt_opt_shape, trt_opt_shape]
+            }
+            config.set_trt_dynamic_shape_info(min_input_shape, max_input_shape,
+                                              opt_input_shape)
+            print('trt set dynamic shape done!')
+
+    # disable print log when predict
+    config.disable_glog_info()
+    # enable shared memory
+    config.enable_memory_optim()
+    # disable feed, fetch OP, needed by zero_copy_run
+    config.switch_use_feed_fetch_ops(False)
+    if delete_shuffle_pass:
+        config.delete_pass("shuffle_channel_detect_pass")
+    predictor = create_predictor(config)
+    return predictor, config
+
+
+def get_test_images(infer_dir, infer_img):
+    """
+    Get image path list in TEST mode
+    """
+    assert infer_img is not None or infer_dir is not None, \
+        "--image_file or --image_dir should be set"
+    assert infer_img is None or os.path.isfile(infer_img), \
+            "{} is not a file".format(infer_img)
+    assert infer_dir is None or os.path.isdir(infer_dir), \
+            "{} is not a directory".format(infer_dir)
+
+    # infer_img has a higher priority
+    if infer_img and os.path.isfile(infer_img):
+        return [infer_img]
+
+    images = set()
+    infer_dir = os.path.abspath(infer_dir)
+    assert os.path.isdir(infer_dir), \
+        "infer_dir {} is not a directory".format(infer_dir)
+    exts = ['jpg', 'jpeg', 'png', 'bmp']
+    exts += [ext.upper() for ext in exts]
+    for ext in exts:
+        images.update(glob.glob('{}/*.{}'.format(infer_dir, ext)))
+    images = list(images)
+
+    assert len(images) > 0, "no image found in {}".format(infer_dir)
+    print("Found {} inference images in total.".format(len(images)))
+
+    return images
+
+
+def visualize(image_list, result, labels, output_dir='output/', threshold=0.5):
+    # visualize the predict result
+    start_idx = 0
+    for idx, image_file in enumerate(image_list):
+        im_bboxes_num = result['boxes_num'][idx]
+        im_results = {}
+        if 'boxes' in result:
+            im_results['boxes'] = result['boxes'][start_idx:start_idx +
+                                                  im_bboxes_num, :]
+        if 'masks' in result:
+            im_results['masks'] = result['masks'][start_idx:start_idx +
+                                                  im_bboxes_num, :]
+        if 'segm' in result:
+            im_results['segm'] = result['segm'][start_idx:start_idx +
+                                                im_bboxes_num, :]
+        if 'label' in result:
+            im_results['label'] = result['label'][start_idx:start_idx +
+                                                  im_bboxes_num]
+        if 'score' in result:
+            im_results['score'] = result['score'][start_idx:start_idx +
+                                                  im_bboxes_num]
+
+        start_idx += im_bboxes_num
+        im = visualize_box_mask(
+            image_file, im_results, labels, threshold=threshold)
+        img_name = os.path.split(image_file)[-1]
+        if not os.path.exists(output_dir):
+            os.makedirs(output_dir)
+        out_path = os.path.join(output_dir, img_name)
+        im.save(out_path, quality=95)
+        print("save result to: " + out_path)
+
+
+def print_arguments(args):
+    print('-----------  Running Arguments -----------')
+    for arg, value in sorted(vars(args).items()):
+        print('%s: %s' % (arg, value))
+    print('------------------------------------------')
+
+
+def main():
+    deploy_file = os.path.join(FLAGS.model_dir, 'infer_cfg.yml')
+    with open(deploy_file) as f:
+        yml_conf = yaml.safe_load(f)
+    arch = yml_conf['arch']
+    detector_func = 'Detector'
+    if arch == 'SOLOv2':
+        detector_func = 'DetectorSOLOv2'
+    elif arch == 'PicoDet':
+        detector_func = 'DetectorPicoDet'
+
+    detector = eval(detector_func)(
+        FLAGS.model_dir,
+        device=FLAGS.device,
+        run_mode=FLAGS.run_mode,
+        batch_size=FLAGS.batch_size,
+        trt_min_shape=FLAGS.trt_min_shape,
+        trt_max_shape=FLAGS.trt_max_shape,
+        trt_opt_shape=FLAGS.trt_opt_shape,
+        trt_calib_mode=FLAGS.trt_calib_mode,
+        cpu_threads=FLAGS.cpu_threads,
+        enable_mkldnn=FLAGS.enable_mkldnn,
+        enable_mkldnn_bfloat16=FLAGS.enable_mkldnn_bfloat16,
+        threshold=FLAGS.threshold,
+        output_dir=FLAGS.output_dir)
+
+    # predict from video file or camera video stream
+    if FLAGS.video_file is not None or FLAGS.camera_id != -1:
+        detector.predict_video(FLAGS.video_file, FLAGS.camera_id)
+    else:
+        # predict from image
+        if FLAGS.image_dir is None and FLAGS.image_file is not None:
+            assert FLAGS.batch_size == 1, "batch_size should be 1, when image_file is not None"
+        img_list = get_test_images(FLAGS.image_dir, FLAGS.image_file)
+        if FLAGS.slice_infer:
+            detector.predict_image_slice(
+                img_list,
+                FLAGS.slice_size,
+                FLAGS.overlap_ratio,
+                FLAGS.combine_method,
+                FLAGS.match_threshold,
+                FLAGS.match_metric,
+                visual=FLAGS.save_images,
+                save_results=FLAGS.save_results)
+        else:
+            detector.predict_image(
+                img_list,
+                FLAGS.run_benchmark,
+                repeats=100,
+                visual=FLAGS.save_images,
+                save_results=FLAGS.save_results)
+        if not FLAGS.run_benchmark:
+            detector.det_times.info(average=True)
+        else:
+            mode = FLAGS.run_mode
+            model_dir = FLAGS.model_dir
+            model_info = {
+                'model_name': model_dir.strip('/').split('/')[-1],
+                'precision': mode.split('_')[-1]
+            }
+            bench_log(detector, img_list, model_info, name='DET')
+
+
+if __name__ == '__main__':
+    paddle.enable_static()
+    parser = argsparser()
+    FLAGS = parser.parse_args()
+    print_arguments(FLAGS)
+    FLAGS.device = FLAGS.device.upper()
+    assert FLAGS.device in ['CPU', 'GPU', 'XPU'
+                            ], "device should be CPU, GPU or XPU"
+    assert not FLAGS.use_gpu, "use_gpu has been deprecated, please use --device"
+
+    assert not (
+        FLAGS.enable_mkldnn == False and FLAGS.enable_mkldnn_bfloat16 == True
+    ), 'To enable mkldnn bfloat, please turn on both enable_mkldnn and enable_mkldnn_bfloat16'
+
+    main()