Merge pull request #21 from davidpagnon/ik

- trc Z values not set to zeros but to coherent values based on visible side
- c3d export
- renamed a bunch of parameters

Author: davidpagnon

README.md

@@ -12,12 +12,16 @@
 
 
 [project]
-video_input = 'demo.mp4' # 'webcam' or '<video_path.ext>', or ['video1_path.mp4', 'video2_path.avi>', ...]
-                        # On Windows, replace '\' with '/'
-                        # Beware that images won't be saved if paths contain non ASCII characters.
-person_height = 1.70    # Height of the person in meters (for pixels -> meters conversion)
-load_trc = ''           # If you do not want to recalculate pose, load it from a trc file (in px, not in m)
-compare = false         # Not implemented yet
+video_input = 'demo.mp4'   # 'webcam' or '<video_path.ext>', or ['video1_path.mp4', 'video2_path.avi>', ...]
+                           # On Windows, replace '\' with '/'
+                           # Beware that images won't be saved if paths contain non ASCII characters.
+px_to_m_from_person_id = 2    # Person to use for pixels to meters conversion (not used if a calibration file is provided)
+px_to_m_person_height = 1.65  # Height of the reference person in meters (for pixels -> meters conversion). 
+visible_side = ['front', 'none', 'auto']  # Choose visible side among ['right', 'left', 'front', 'back', 'auto', 'none']. String or list of strings.
+               # if 'auto', will be either 'left', 'right', or 'front' depending on the direction of the motion
+               # if 'none', no processing will be performed on the corresponding person
+load_trc_px = ''  # If you do not want to recalculate pose, load it from a trc file (in px, not in m)
+compare = false      # Not implemented yet
 
 # Video parameters
 time_range = []   # [] for the whole video, or [start_time, end_time] (in seconds), or [[start_time1, end_time1], [start_time2, end_time2], ...]
@@ -67,15 +71,19 @@ mode = 'balanced' # 'lightweight', 'balanced', 'performance', or """{dictionary}
 # mode = """{'pose_class':'RTMO', 
 #          'pose_model':'https://download.openmmlab.com/mmpose/v1/projects/rtmo/onnx_sdk/rtmo-m_16xb16-600e_body7-640x640-39e78cc4_20231211.zip', 
 #          'pose_input_size':[640, 640]}"""
+# Example with animal pose estimation:
+# mode = """{'pose_class':'RTMPose',
+#          'pose_model':'https://download.openmmlab.com/mmpose/v1/projects/rtmposev1/onnx_sdk/rtmpose-m_simcc-ap10k_pt-aic-coco_210e-256x256-7a041aa1_20230206.zip',
+#          'pose_input_size':[256,256]}"""
 
 det_frequency = 4       # Run person detection only every N frames, and inbetween track previously detected bounding boxes (keypoint detection is still run on all frames). 
                         # Equal to or greater than 1, can be as high as you want in simple uncrowded cases. Much faster, but might be less accurate. 
 device = 'auto' # 'auto', 'CPU', 'CUDA', 'MPS', 'ROCM'
 backend = 'auto' # 'auto', 'openvino', 'onnxruntime', 'opencv'
 tracking_mode = 'sports2d' # 'sports2d' or 'deepsort'. 'deepsort' is slower but more robust in difficult configurations
-deepsort_params = """{'max_age':30, 'n_init':3, 'nms_max_overlap':0.8, 'max_cosine_distance':0.3, 'nn_budget':200, 'max_iou_distance':0.8, 'embedder_gpu': True}""" # """{dictionary between 3 double quotes}"""
-                  # More robust in crowded scenes but Can be tricky to parametrize. More information there: https://github.com/levan92/deep_sort_realtime/blob/master/deep_sort_realtime/deepsort_tracker.py#L51
-                  # Note: For faster and more robust tracking, use {'embedder_gpu': True, embedder':'torchreid'}, which uses the GPU and runs osnet_ain_x1_0 by default. requires `pip install torch torchvision torchreid gdown tensorboard`
+deepsort_params = """{'max_age':30, 'n_init':3, 'max_cosine_distance':0.3, 'max_iou_distance':0.8, 'embedder_gpu': True, embedder':'torchreid'}""" # """{dictionary between 3 double quotes}"""
+                  # More robust in crowded scenes but tricky to parametrize. More information there: https://github.com/levan92/deep_sort_realtime/blob/master/deep_sort_realtime/deepsort_tracker.py#L51
+                  # Requires `pip install torch torchvision torchreid gdown tensorboard`
 
 
 # Processing parameters
@@ -87,13 +95,15 @@ keypoint_number_threshold = 0.3     # Person will be ignored if the number of go
 [px_to_meters_conversion]
 # Pixel to meters conversion
 to_meters = true
-# If conversion from a calibration file
-calib_file = ''         # Calibration in the Pose2Sim format. 'calib_demo.toml', or '' if not available
-# If conversion from person_height
-calib_on_person_id = 0  # Person to use for calibration
+make_c3d = true
+save_calib = true # Coming soon!
+
+# If conversion from px_to_m_person_height
 floor_angle = 'auto'    # 'auto' or a value in degrees, eg 2.3. If 'auto', estimated from the line formed by the toes when they are on the ground (where speed = 0)
 xy_origin = ['auto']    # ['auto'] or [px_x,px_y]. N.B.: px_y points downwards. If ['auto'], direction estimated from the start to the end of the line formed by the toes when they are on the ground
-save_calib = true
+
+# If conversion from a calibration file
+calib_file = ''         # Calibration in the Pose2Sim format. 'calib_demo.toml', or '' if not available
 
 fastest_frames_to_remove_percent = 0.1 # Frames with high speed are considered as outliers
 close_to_zero_speed_px = 50 # Sum for all keypoints: about 50 px/frame or 0.2 m/frame
@@ -136,15 +146,20 @@ filter_type = 'butterworth'   # butterworth, gaussian, LOESS, median
    kernel_size = 3
 
 
-[inverse-kinematics]
+[kinematics]
 do_ik = false # Do scaling and inverse kinematics?
-person_orientation = ['front', 'none', 'left'] # Choose among 'auto', 'none', 'front', 'back', 'left', 'right'
-                     # if 'none', no IK will be performed on the corresponding person
-                     # if 'auto', will be either 'left' or 'right' depending on the direction of the motion
-                     # Example with one person on one video: ['front']
-                     # Or ['front', 'none', 'left'] with 3 persons on one video
+use_augmentation = true  # true or false (lowercase) # Set to true if you want to use the model with augmented markers
+use_contacts_muscles = true # true or false (lowercase) # If true, contact spheres and muscles are added to the model
+
 osim_setup_path = '../OpenSim_setup' # Path to the OpenSim setup folder
-close_to_zero_speed_m = 0.2 # Sum for all keypoints: about 50 px/frame or 0.2 m/frame 
+right_left_symmetry = true # true or false (lowercase) # Set to false only if you have good reasons to think the participant is not symmetrical (e.g. prosthetic limb)
+# default_height = 1.7 # meters # If automatic height calculation did not work, this value is used to scale the model
+remove_individual_scaling_setup = true # true or false (lowercase) # If true, the individual scaling setup files are removed to avoid cluttering
+remove_individual_ik_setup = true # true or false (lowercase) # If true, the individual IK setup files are removed to avoid cluttering
+fastest_frames_to_remove_percent = 0.1 # Frames with high speed are considered as outliers
+close_to_zero_speed_m = 0.2 # Sum for all keypoints: about 50 px/frame or 0.2 m/frame
+large_hip_knee_angles = 45 # Hip and knee angles below this value are considered as imprecise
+trimmed_extrema_percent = 0.5 # Proportion of the most extreme segment values to remove before calculating their mean)
 
 
 [logging]

Sports2D/Demo/Config_demo.toml

@@ -122,8 +122,10 @@
 
 ## CONSTANTS
 DEFAULT_CONFIG =   {'project': {'video_input': ['demo.mp4'],
-                                'person_height': 1.70,
-                                'load_trc': '',
+                                'px_to_m_from_person_id': 2,
+                                'px_to_m_person_height': 1.65,
+                                'visible_side': ['front', 'auto'],
+                                'load_trc_px': '',
                                 'compare': False,
                                 'time_range': [],
                                 'video_dir': '',
@@ -153,8 +155,8 @@
                                 },
                     'px_to_meters_conversion': {
                                 'to_meters': True,
+                                'make_c3d': True,
                                 'calib_file': '',
-                                'calib_on_person_id': 0,
                                 'floor_angle': 'auto',
                                 'xy_origin': ['auto'],
                                 'save_calib': True,
@@ -205,18 +207,27 @@
                                         'loess': {'nb_values_used': 5},
                                         'median': {'kernel_size': 3}
                                         },
-                    'inverse-kinematics':{'do_ik': False,
-                                          'person_orientation': ['front', '', 'left'],
-                                          'osim_setup_path': '../OpenSim_setup',
-                                          'close_to_zero_speed_m': 0.2
+                    'kinematics':{'do_ik': False,
+                                          'use_augmentation': False,
+                                          'use_contacts_muscles': False,
+                                          'right_left_symmetry': True,
+                                          'default_height': 1.70,
+                                          'remove_individual_scaling_setup': True,
+                                          'remove_individual_ik_setup': True,
+                                          'fastest_frames_to_remove_percent': 0.1,
+                                          'close_to_zero_speed_m': 0.2,
+                                          'large_hip_knee_angles': 45,
+                                          'trimmed_extrema_percent': 0.5,
+                                          'osim_setup_path': '../OpenSim_setup'
                                           },
                     'logging': {'use_custom_logging': False}
                     }
 
 CONFIG_HELP =   {'config': ["C", "path to a toml configuration file"],
                 'video_input': ["i", "webcam, or video_path.mp4, or video1_path.avi video2_path.mp4 ... Beware that images won't be saved if paths contain non ASCII characters"],
-                'person_height': ["H", "height of the person in meters. 1.70 if not specified"],
-                'load_trc': ["", "load trc file to avaid running pose estimation again. false if not specified"],
+                'px_to_m_person_height': ["H", "height of the person in meters. 1.70 if not specified"],
+                'visible_side': ["", "front, back, left, right, auto, or none. 'front auto' if not specified. If 'auto', will be either left or right depending on the direction of the motion. If 'none', no IK for this person"],
+                'load_trc_px': ["", "load trc file to avaid running pose estimation again. false if not specified"],
                 'compare': ["", "visually compare motion with trc file. false if not specified"],
                 'webcam_id': ["w", "webcam ID. 0 if not specified"],
                 'time_range': ["t", "start_time end_time. In seconds. Whole video if not specified. start_time1 end_time1 start_time2 end_time2 ... if multiple videos with different time ranges"],
@@ -240,19 +251,21 @@
                 'backend': ["", "Backend for pose estimation can be 'auto', 'cpu', 'cuda', 'mps' (for MacOS), or 'rocm' (for AMD GPUs)"],
                 'device': ["", "Device for pose estimatino can be 'auto', 'openvino', 'onnxruntime', 'opencv'"],
                 'to_meters': ["M", "convert pixels to meters. true if not specified"],
-                'calib_on_person_id': ["", "person ID to calibrate on. 0 if not specified"],
+                'make_c3d': ["", "Convert trc to c3d file. true if not specified"],
+                'px_to_m_from_person_id': ["", "person ID to calibrate on. 0 if not specified"],
                 'floor_angle': ["", "angle of the floor. 'auto' if not specified"],
                 'xy_origin': ["", "origin of the xy plane. 'auto' if not specified"],
                 'calib_file': ["", "path to calibration file. '' if not specified, eg no calibration file"],
                 'save_calib': ["", "save calibration file. true if not specified"],
                 'do_ik': ["", "do inverse kinematics. false if not specified"],
-                'osim_setup_path': ["", "path to OpenSim setup. '../OpenSim_setup' if not specified"],
-                'person_orientation': ["", "front, back, left, right, auto, or none. 'front none left' if not specified. If 'auto', will be either left or right depending on the direction of the motion."],
+                'use_augmentation': ["", "Use LSTM marker augmentation. false if not specified"],
+                'use_contacts_muscles': ["", "Use model with contact spheres and muscles. false if not specified"],
                 'close_to_zero_speed_m': ["","Sum for all keypoints: about 50 px/frame or 0.2 m/frame"], 
                 'multiperson': ["", "multiperson involves tracking: will be faster if set to false. true if not specified"],
                 'tracking_mode': ["", "sports2d or rtmlib. sports2d is generally much more accurate and comparable in speed. sports2d if not specified"],
                 'deepsort_params': ["", 'Deepsort tracking parameters: """{dictionary between 3 double quotes}""". \n\
-                                    More information there: https://github.com/levan92/deep_sort_realtime/blob/master/deep_sort_realtime/deepsort_tracker.py#L51'],                 # 
+                                    Default: max_age:30, n_init:3, nms_max_overlap:0.8, max_cosine_distance:0.3, nn_budget:200, max_iou_distance:0.8, embedder_gpu: True\n\
+                                    More information there: https://github.com/levan92/deep_sort_realtime/blob/master/deep_sort_realtime/deepsort_tracker.py#L51'],
                 'input_size': ["", "width, height. 1280, 720 if not specified. Lower resolution will be faster but less precise"],
                 'keypoint_likelihood_threshold': ["", "detected keypoints are not retained if likelihood is below this threshold. 0.3 if not specified"],
                 'average_likelihood_threshold': ["", "detected persons are not retained if average keypoint likelihood is below this threshold. 0.5 if not specified"],
@@ -274,6 +287,15 @@
                 'sigma_kernel': ["", "sigma of the gaussian filter. 1 if not specified"],
                 'nb_values_used': ["", "number of values used for the loess filter. 5 if not specified"],
                 'kernel_size': ["", "kernel size of the median filter. 3 if not specified"],
+                'osim_setup_path': ["", "path to OpenSim setup. '../OpenSim_setup' if not specified"],
+                'right_left_symmetry': ["", "right left symmetry. true if not specified"],
+                'default_height': ["", "default height for scaling. 1.70 if not specified"],
+                'remove_individual_scaling_setup': ["", "remove individual scaling setup files generated during scaling. true if not specified"],
+                'remove_individual_ik_setup': ["", "remove individual IK setup files generated during IK. true if not specified"],
+                'fastest_frames_to_remove_percent': ["", "Frames with high speed are considered as outliers. Defaults to 0.1"],
+                'close_to_zero_speed_m': ["","Sum for all keypoints: about 50 px/frame or 0.2 m/frame"],
+                'large_hip_knee_angles': ["", "Hip and knee angles below this value are considered as imprecise and ignored. Defaults to 45"],
+                'trimmed_extrema_percent': ["", "Proportion of the most extreme segment values to remove before calculating their mean. Defaults to 50"],
                 'use_custom_logging': ["", "use custom logging. false if not specified"]
                 }