diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c5d069c3..afe094f4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -6,6 +6,7 @@ name: build on: push: branches: + - alpha_test # remove it after public - main paths-ignore: - 'README.md' @@ -23,56 +24,54 @@ concurrency: cancel-in-progress: true jobs: - build_cuda101: + build_cuda102: runs-on: ubuntu-18.04 - strategy: - matrix: - python-version: [3.7] - torch: [1.7.0, 1.8.0] - include: - - torch: 1.7.0 - torchvision: 0.8.1 - - torch: 1.8.0 - torchvision: 0.9.0 defaults: run: shell: bash -l {0} - + container: + image: openxrlab/xrmocap_runtime:ubuntu1804_x64_cu102_py38_torch181_mmcv153 + credentials: + username: ${{secrets.DOCKERHUB_USERNAME}} + password: ${{secrets.DOCKERHUB_PWD}} steps: - uses: actions/checkout@v2 - - uses: conda-incubator/setup-miniconda@v2 - with: - activate-environment: xrmocap_env - python-version: ${{matrix.python-version}} - auto-activate-base: false - - name: Prepare test data + - name: Show conda env run: | - # TODO: prepare test data or remove this step - - name: Upgrade pip - run: pip install pip --upgrade - - name: Install ffmpeg - run: | - conda install ffmpeg - ffmpeg -version - - name: Install PyTorch + source /root/miniconda3/etc/profile.d/conda.sh && conda deactivate + conda info -e + conda activate openxrlab + conda list | grep torch + conda list | grep mmcv + - name: Prepare test data run: | - conda install pytorch==${{matrix.torch}} torchvision==${{matrix.torchvision}} cudatoolkit=10.1 -c pytorch - - name: Install MMCV + sh scripts/download_test_data.sh + - name: Prepare weight run: | - pip install mmcv-full -f https://download.openmmlab.com/mmcv/dist/cpu/torch${{matrix.torch}}/index.html - python -c 'import mmcv; print(mmcv.__version__)' - - name: Install other dependencies - run: pip install -r requirements.txt + sh scripts/download_test_data.sh + mkdir weight + cd weight + wget https://download.openmmlab.com/mmdetection/v2.0/faster_rcnn/faster_rcnn_r50_fpn_1x_coco/faster_rcnn_r50_fpn_1x_coco_20200130-047c8118.pth + wget https://download.openmmlab.com/mmpose/top_down/hrnet/hrnet_w48_coco_wholebody_384x288_dark-f5726563_20200918.pth + wget https://download.openmmlab.com/mmtracking/mot/faster_rcnn/faster-rcnn_r50_fpn_4e_mot17-half-64ee2ed4.pth + wget https://download.openmmlab.com/mmtracking/mot/reid/tracktor_reid_r50_iter25245-a452f51f.pth + cd .. - name: Build and install - run: rm -rf .eggs && pip install -e . + run: | + rm -rf xrmocap.egg-info + source /root/miniconda3/etc/profile.d/conda.sh && conda activate openxrlab + pip install xrprimer + pip install -e . - name: Run unittests and generate coverage report run: | + source /root/miniconda3/etc/profile.d/conda.sh && conda activate openxrlab coverage run --source xrmocap -m pytest tests/ coverage xml coverage report -m - name: Upload coverage to Codecov - uses: codecov/codecov-action@v2 + uses: codecov/codecov-action@v3 with: + token: ${{ secrets.CODECOV_TOKEN }} # deprecated after public files: ./coverage.xml flags: unittests env_vars: OS,PYTHON diff --git a/.github/workflows/publish-to-pypi.yml b/.github/workflows/publish-to-pypi.yml index 85724d7a..e5aaccd8 100644 --- a/.github/workflows/publish-to-pypi.yml +++ b/.github/workflows/publish-to-pypi.yml @@ -19,4 +19,4 @@ jobs: - name: Publish distribution to PyPI run: | pip install twine - twine upload dist/* -u __token__ -p ${{ secrets.pypi_password }} + twine upload dist/* -u __token__ -p ${{ secrets.PYPI_PASSWORD }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1a67c91d..77ff723f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -37,6 +37,7 @@ repos: rev: v2.1.0 hooks: - id: codespell + args: ["--ignore-words-list", "mot"] - repo: https://github.com/myint/docformatter.git rev: v1.3.1 hooks: diff --git a/Dockerfile b/Dockerfile index 1eef16ab..51c25673 100644 --- a/Dockerfile +++ b/Dockerfile @@ -42,21 +42,19 @@ RUN . /root/miniconda3/etc/profile.d/conda.sh && \ conda activate openxrlab && \ pip install pre-commit interrogate coverage pytest && \ pip install mmcv-full==1.5.3 -f https://download.openmmlab.com/mmcv/dist/cu102/torch1.8.1/index.html && \ - pip install xrprimer -i https://repo.sensetime.com/repository/pypi/simple && \ + pip install xrprimer && \ pip cache purge # Install build requirements RUN . /root/miniconda3/etc/profile.d/conda.sh && \ conda activate openxrlab && \ - wget http://10.4.11.59:18080/resources/XRlab/requirements/xrmocap/build.txt && \ - pip install -r build.txt && rm build.txt && \ + pip install -r https://github.com/openxrlab/xrmocap/blob/main/requirements/build.txt && \ pip cache purge # Install test requirements RUN . /root/miniconda3/etc/profile.d/conda.sh && \ conda activate openxrlab && \ - wget http://10.4.11.59:18080/resources/XRlab/requirements/xrmocap/test.txt && \ - pip install -r test.txt && rm test.txt && \ + pip install -r https://github.com/openxrlab/xrmocap/blob/main/requirements/test.txt && \ pip cache purge # Install mmhuman3d @@ -73,3 +71,11 @@ RUN . /root/miniconda3/etc/profile.d/conda.sh && \ pip uninstall opencv-python opencv-python-headless -y && \ pip install opencv-python-headless && \ pip cache purge + +# Clone xrmocap and install +RUN . /root/miniconda3/etc/profile.d/conda.sh && \ + conda activate openxrlab && \ + cd /workspace && \ + git clone https://github.com/openxrlab/xrmocap.git && \ + cd xrmocap && pip install -e . && \ + pip cache purge diff --git a/README.md b/README.md index b3833943..fb85f6ff 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,13 @@ # XRMocap +
+ +[![actions](https://github.com/openxrlab/xrmocap/workflows/build/badge.svg)](https://github.com/openxrlab/xrmocap/actions) +[![codecov](https://codecov.io/gh/openxrlab/xrmocap/branch/main/graph/badge.svg)](https://codecov.io/gh/openxrlab/xrmocap) +[![PyPI](https://img.shields.io/pypi/v/xrmocap)](https://pypi.org/project/xrmocap/) +[![LICENSE](https://img.shields.io/github/license/openxrlab/xrmocap.svg)](https://github.com/openxrlab/xrmocap/blob/main/LICENSE) + +
## Introduction @@ -9,7 +17,7 @@ XRMoCap is an open-source PyTorch-based codebase for the use of multi-view motio If you are interested in single-view motion capture, please refer to [mmhuman3d](https://github.com/open-mmlab/mmhuman3d) for more details. -**TODO:[Insert demo here]** +https://user-images.githubusercontent.com/26729379/187710195-ba4660ce-c736-4820-8450-104f82e5cc99.mp4 A detailed introduction can be found in [introduction.md](./docs/en/tutorials/introduction.md). @@ -62,7 +70,7 @@ Supported datasets: ## Getting Started -Please see [getting_started.md](docs/getting_started.md) for the basic usage of XRMoCap. +Please see [getting_started.md](docs/en/getting_started.md) for the basic usage of XRMoCap. ## License diff --git a/configs/mvp/README.md b/configs/mvp/README.md index 667f1a17..e2752bf7 100644 --- a/configs/mvp/README.md +++ b/configs/mvp/README.md @@ -19,32 +19,35 @@ We provide the config files for MvP: [Direct multi-view multi-person 3d pose est ## Results and Models -We evaluate MvP on 3 popular benchmarks. - -[Explain common setting and metrics] +We evaluate MvP on 3 popular benchmarks, report the Percentage of Correct Parts (PCP) on Shelf and Campus dataset, Mean Per Joint Position Error (MPJPE), mAP and recall on CMU Panoptic dataset. ### Campus -[Explain specific setting, metrics and other details (finetuned from pretrained cmu panoptic xxx)] +MvP for Campus fine-tuned from the model weights pre-trained with 3 selected views in CMU Panoptic dataset is provided. Fine-tuning with the model pre-train with CMU Panoptic HD camera view 3, 6, 12 gives the best final performance on Campus dataset. | Config | Campus | Download | |:------:|:-------:|:--------:| -| - | - | [model]() | [log]() | +| [mvp_campus.py](./campus_config/mvp_campus.py) | 96.7 | [model](https://openxrlab-share.oss-cn-hongkong.aliyuncs.com/xrmocap/weight/mvp/xrmocap_mvp_campus-e6093968_20220831.pth) | ### Shelf -[Explain specific setting, metrics and other details (finetuned from pretrained cmu panoptic xxx)] +MvP for Shelf fine-tuned from the model weights pre-trained with 5 selected views in CMU Panoptic dataset is provided. The 5 selected views, HD camera view 3, 6, 12, 13 and 23 are the same views used in VoxelPose. | Config | Shelf | Download | |:------:|:-------:|:--------:| -| - | - | [model]() | [log]() | +| [mvp_shelf.py](./shelf_config/mvp_shelf.py) | 97.1 | [model](https://openxrlab-share.oss-cn-hongkong.aliyuncs.com/xrmocap/weight/mvp/xrmocap_mvp_shelf-22d1b5ed_20220831.pth) | ### CMU Panoptic -[Explain view selection, training details etc.] +MvP for CMU Panoptic trained from stcratch with pre-trained Pose ResNet50 backbone is provided. The provided model weights were trained and evaluated with the 5 selected views same as VoxelPose (HD camera view 3, 6, 12, 13, 23). A checkpoint trained with 3 selected views (HD camera view 3, 12, 23) is also provided as the pre-trained model weights for Campus dataset fine-tuning. + +| Config | AP25 | AP100 | Recall@500 | MPJPE(mm) |Download | +|:------:|:----:|:----:|:---------:|:--------:|:--------:| +| [mvp_panoptic.py](./panoptic_config/mvp_panoptic.py) | 91.5 | 97.9 | 99.85 |16.45 | [model](https://openxrlab-share.oss-cn-hongkong.aliyuncs.com/xrmocap/weight/mvp/xrmocap_mvp_panoptic_5view-1b673cdf_20220831.pth) | +| [mvp_panoptic_3cam.py](./panoptic_config/mvp_panoptic_3cam.py) | 54.7 | 95.1 | 98.83 |30.55 | [model](https://openxrlab-share.oss-cn-hongkong.aliyuncs.com/xrmocap/weight/mvp/xrmocap_mvp_panoptic_3view_3_12_23-4b391740_20220831.pth) | + +### Pose ResNet50 Backbone -| Config | AP25 | MPJPE(mm) | Download | -|:------:|:----:|:---------:|:--------:| -| - | - | - | [model]() | [log]() | +All the checkpoints provided above were trained on top of the pre-trained [Pose ResNet50](https://openxrlab-share.oss-cn-hongkong.aliyuncs.com/xrmocap/weight/mvp/xrmocap_pose_resnet50_panoptic-5a2e53c9_20220831.pth) backbone weights. diff --git a/configs/mvp/campus_config/mvp_campus.py b/configs/mvp/campus_config/mvp_campus.py index 8e56be30..20969e88 100644 --- a/configs/mvp/campus_config/mvp_campus.py +++ b/configs/mvp/campus_config/mvp_campus.py @@ -33,7 +33,7 @@ end_epoch=25, pretrained_backbone='', model_root='./weight', - finetune_model='convert_model_best_31223.pth.tar', + finetune_model='xrmocap_mvp_panoptic_3view_3_12_23.pth.tar', resume=False, lr_decay_epoch=[10], inference_conf_thr=[0.0], diff --git a/configs/mvp/panoptic_config/mvp_panoptic.py b/configs/mvp/panoptic_config/mvp_panoptic.py index 3ed9175b..dcfd1cf6 100644 --- a/configs/mvp/panoptic_config/mvp_panoptic.py +++ b/configs/mvp/panoptic_config/mvp_panoptic.py @@ -30,7 +30,7 @@ weight_decay=1e-4, optimizer='adam', end_epoch=200, - pretrained_backbone='convert_pose_resnet50_panoptic.pth.tar', + pretrained_backbone='xrmocap_pose_resnet50_panoptic.pth.tar', model_root='./weight', finetune_model=None, resume=False, diff --git a/configs/mvp/panoptic_config/mvp_panoptic_3cam.py b/configs/mvp/panoptic_config/mvp_panoptic_3cam.py index b0566013..65f74159 100644 --- a/configs/mvp/panoptic_config/mvp_panoptic_3cam.py +++ b/configs/mvp/panoptic_config/mvp_panoptic_3cam.py @@ -30,7 +30,7 @@ weight_decay=1e-4, optimizer='adam', end_epoch=200, - pretrained_backbone='convert_pose_resnet50_panoptic.pth.tar', + pretrained_backbone='xrmocap_pose_resnet50_panoptic.pth.tar', model_root='./weight', finetune_model=None, resume=False, diff --git a/configs/mvp/shelf_config/mvp_shelf.py b/configs/mvp/shelf_config/mvp_shelf.py index bae122a6..1eecadf4 100644 --- a/configs/mvp/shelf_config/mvp_shelf.py +++ b/configs/mvp/shelf_config/mvp_shelf.py @@ -33,7 +33,7 @@ end_epoch=30, pretrained_backbone='', model_root='./weight', - finetune_model='convert_model_best_5view.pth.tar', + finetune_model='xrmocap_mvp_panoptic_5view.pth.tar', resume=False, lr_decay_epoch=[30], inference_conf_thr=[0.0], diff --git a/configs/mvp/shelf_config/mvp_shelf_50.py b/configs/mvp/shelf_config/mvp_shelf_50.py new file mode 100644 index 00000000..386d547b --- /dev/null +++ b/configs/mvp/shelf_config/mvp_shelf_50.py @@ -0,0 +1,166 @@ +# finetune with the weight pretrained from panoptic dataset + +__dataset__ = 'shelf' +__train_dataset__ = __dataset__ +__test_dataset__ = __dataset__ +__resnet_n_layer__ = 50 +__net_n_kps__ = 15 +__dataset_n_kps__ = 14 +__n_instance__ = 10 +__n_cameras__ = 5 +__image_size__ = [800, 608] +__space_size__ = [8000.0, 8000.0, 2000.0] +__space_center__ = [450.0, -320.0, 800.0] +__decoder_size__ = 256 +__projattn_pose_embed_mode__ = 'use_rayconv' +__pred_conf_threshold__ = 0.5 +__root_idx__ = [2, 3] + +model = 'multi_view_pose_transformer' +output_dir = 'output' +dataset = __dataset__ +backbone_layers = __resnet_n_layer__ + +trainer_setup = dict( + type='MVPTrainer', + workers=4, + train_dataset=__train_dataset__, + test_dataset=__test_dataset__, + lr=0.0002, + lr_linear_proj_mult=0.1, + weight_decay=1e-4, + optimizer='adamw', + end_epoch=30, + pretrained_backbone='', + model_root='./weight', + finetune_model='xrmocap_mvp_panoptic_5view.pth.tar', + resume=False, + lr_decay_epoch=[30], + inference_conf_thr=[0.0], + train_batch_size=1, + test_batch_size=1, + test_model_file='model_best.pth.tar', + clip_max_norm=0.1, + print_freq=100, + cudnn_setup=dict( + benchmark=True, + deterministic=False, + enable=True, + ), + dataset_setup=dict( + train_dataset_setup=dict( + type='MVPDataset', + test_mode=False, + meta_path='./xrmocap_data/meta/shelf/shelf_trainset', + ), + test_dataset_setup=dict( + type='MVPDataset', + test_mode=True, + meta_path='./xrmocap_data/Shelf_50/xrmocap_meta_testset_small/', + ), + base_dataset_setup=dict( + type='MVPDataset', + dataset=__dataset__, + data_root='./xrmocap_data/Shelf_50/Shelf/', + img_pipeline=[ + dict(type='LoadImageCV2'), + dict(type='BGR2RGB'), + dict( + type='WarpAffine', + image_size=__image_size__, + flag='inter_linear'), + dict(type='ToTensor'), + dict( + type='Normalize', + mean=[0.485, 0.456, 0.406], + std=[0.229, 0.224, 0.225]) + ], + image_size=__image_size__, + heatmap_size=[200, 152], + metric_unit='millimeter', + shuffled=False, + gt_kps3d_convention='campus', # same convention for shelf, campus + cam_world2cam=True, + n_max_person=__n_instance__, + n_views=__n_cameras__, + n_kps=__dataset_n_kps__, + ), + ), + mvp_setup=dict( + type='MviewPoseTransformer', + n_kps=__net_n_kps__, + n_instance=__n_instance__, + image_size=__image_size__, + space_size=__space_size__, + space_center=__space_center__, + d_model=__decoder_size__, + use_feat_level=[0, 1, 2], + n_cameras=__n_cameras__, + query_embed_type='person_kp', + with_pose_refine=True, + loss_weight_loss_ce=0.0, + loss_per_kp=5., + aux_loss=True, + pred_conf_threshold=__pred_conf_threshold__, + pred_class_fuse='mean', + projattn_pose_embed_mode=__projattn_pose_embed_mode__, + query_adaptation=True, + convert_kp_format_indexes=[ + 14, 13, 12, 6, 7, 8, 11, 10, 9, 3, 4, 5, 0, 1 + ], + backbone_setup=dict( + type='PoseResNet', + n_layers=__resnet_n_layer__, + n_kps=__net_n_kps__, + deconv_with_bias=False, + n_deconv_layers=3, + n_deconv_filters=[256, 256, 256], + n_deconv_kernels=[4, 4, 4], + final_conv_kernel=1, + ), + decoder_layer_setup=dict( + type='MvPDecoderLayer', + space_size=__space_size__, + space_center=__space_center__, + image_size=__image_size__, + d_model=__decoder_size__, + dim_feedforward=1024, + dropout=0.1, + activation='relu', + n_feature_levels=1, + n_heads=8, + dec_n_points=4, + detach_refpoints_cameraprj=True, + fuse_view_feats='cat_proj', + n_views=__n_cameras__, + projattn_pose_embed_mode=__projattn_pose_embed_mode__, + ), + decoder_setup=dict( + type='MvPDecoder', + n_decoder_layer=6, + return_intermediate=True, + ), + pos_encoding_setup=dict( + type='PositionEmbeddingSine', + normalize=True, + temperature=10000, + ), + pose_embed_setup=dict( + type='MLP', d_model=__decoder_size__, pose_embed_layer=3), + matcher_setup=dict( + type='HungarianMatcher', + match_coord='norm', + ), + criterion_setup=dict( + type='SetCriterion', + image_size=__image_size__, + n_person=__n_instance__, + loss_kp_type='l1', + focal_alpha=0.25, + space_size=__space_size__, + space_center=__space_center__, + use_loss_pose_perprojection=True, + loss_pose_normalize=False, + pred_conf_threshold=__pred_conf_threshold__, + ), + )) diff --git a/configs/mvpose/README.md b/configs/mvpose/README.md index 2992f097..e4a9dafd 100644 --- a/configs/mvpose/README.md +++ b/configs/mvpose/README.md @@ -1,5 +1,9 @@ # MVPose (Single frame) +- [Introduction](#introduction) +- [Prepare models and datasets](#prepare-models-and-datasets) +- [Results and Models](#results-and-models) + ## Introduction We provide the config files for MVPose (Single frame): [Fast and robust multi-person 3d pose estimation from multiple views](https://zju3dv.github.io/mvpose/). @@ -15,35 +19,102 @@ We provide the config files for MVPose (Single frame): [Fast and robust multi-pe year={2019} } ``` +## Prepare models and datasets + +- **Prepare models**: + +``` +sh scripts/download_weight.sh +``` +You can find `resnet50_reid_camstyle.pth.tar` in `weight` file. + +- **Prepare the datasets**: + +Convert original dataset to our unified meta-data, with data converters controlled by configs, +you can find more details in [dataset_preparation.md](../../docs/en/dataset_preparation.md). + +The final file structure would be like: + +```text +xrmocap +├── xrmocap +├── docs +├── tools +├── configs +└── weight + └── resnet50_reid_camstyle.pth.tar +└── xrmocap_data + ├── Shelf + ├── Camera0 + ├── ... + ├── Camera4 + ├── xrmocap_meta_testset_fasterrcnn + └── xrmocap_meta_testset + ├── CampusSeq1 + └── panoptic + ├── xrmocap_meta_ian5 + ├── hd_00_03 + ├── ... + ├── hd_00_23 + ├── camera_parameters + ├── keypoints3d_GT.npz + └── perception_2d.npz + ├── xrmocap_meta_pizza1 + ├── xrmocap_meta_band4 + └── xrmocap_meta_haggling1 +``` ## Results and Models -We evaluate MVPose (Single frame) on 3 popular benchmarks. +We evaluate MVPose (Single frame) on 3 popular benchmarks, report the Percentage of Correct Parts (PCP) on Shelf/Campus/CMU-Panoptic datasets. + +You can find the recommended configs in `configs/mvpose/*/eval_keypoints3d.py`, where `__bbox_thr__` is the threshold of bbox2d, you can set a high threshold to ignore incorrect 2D perception data, `n_cam_min` is the amount of views required for triangulation, which defaults to 2. -[Explain common setting and metrics] ### Campus -[Explain specific setting, metrics and other details] +The 2D perception data we use is generated by fasterrcnn, and you can download it from [here](/docs/en/dataset_preparation.md#download-converted-meta-data). What's more, we set `__bbox_thr__=0.9` and `n_cam_min=2`. -| Config | Campus | Download | -|:------:|:-------:|:--------:| -| - | - | [model]() | [log]() | +| Config | Actor 0 | Actor 1 | Actor 2 | Average | Download | +|:------:|:-------:|:--------:|:--------:|:--------:|:--------:| +| [eval_keypoints3d.py](./campus_config/eval_keypoints3d.py) | 94.17 | 88.19 | 98.38 | 93.58 | [log](https://openxrlab-share.oss-cn-hongkong.aliyuncs.com/xrmocap/logs/MVPose/campus.zip) | ### Shelf -[Explain specific setting, metrics and other details] +The 2D perception data we use is generated by fasterrcnn, and you can download it from [here](/docs/en/dataset_preparation.md#download-converted-meta-data). What's more, we set `__bbox_thr__=0.9` and `n_cam_min=3`. -| Config | Shelf | Download | -|:------:|:-------:|:--------:| -| - | - | [model]() | [log]() | +| Config | Actor 0 | Actor 1 | Actor 2 | Average | Download | +|:------:|:-------:|:--------:|:--------:|:--------:|:--------:| +| [eval_keypoints3d.py](./shelf_config/eval_keypoints3d.py) | 97.53 | 94.05 | 97.89 | 96.49 | [log](https://openxrlab-share.oss-cn-hongkong.aliyuncs.com/xrmocap/logs/MVPose/shelf.zip) | ### CMU Panoptic -[Explain view selection, training details etc.] +The 2D perception data we use is generated by mmpose, and you can download it from [here](/docs/en/dataset_preparation.md#download-converted-meta-data). The selection principle of the camera is to cover as much information as possible about the human body, so we selected cameras 3, 6, 12, 13 and 23. You can find more details in [config files](panoptic_config/eval_keypoints3d.py). + +The CMU Panoptic dataset contains four sequences that share the same config file. For different sequences, you need to change the `__meta_path__`. In addition, there is a child in `160906_ian5` sequence, and the number of cameras that capture him decreases, so we set `__bbox_thr__=0.85` and `n_cam_min=2`. For other sequences, we set `__bbox_thr__=0.9` and `n_cam_min=3`. + +- **160906_band4** + +| Config | Actor 0 | Actor 1 | Actor 2 | Average | Download | +|:-------:|:--------:|:--------:|:--------:|:--------:|:--------:| +| [eval_keypoints3d.py](./panoptic_config/eval_keypoints3d.py) | 98.49 | 79.39 | 92.53 | 90.14 | [log](https://openxrlab-share.oss-cn-hongkong.aliyuncs.com/xrmocap/logs/MVPose/panoptic.zip) | + +- **160906_ian5** + +| Config | Actor 0 | Actor 1 | Average | Download | +|:-------:|:--------:|:--------:|:--------:|:--------:| +| [eval_keypoints3d.py](./panoptic_config/eval_keypoints3d.py) | 96.30 | 71.08 | 83.69 | [log](https://openxrlab-share.oss-cn-hongkong.aliyuncs.com/xrmocap/logs/MVPose/panoptic.zip) | + +- **160906_pizza1** + +| Config | Actor 0 | Actor 1 | Actor 2 | Actor 3 | Actor 4 | Actor 5 | Actor 6 | Average | Download | +|:-------:|:--------:|:--------:|:--------:|:--------:|:--------:|:--------:|:--------:|:--------:|:--------:| +| [eval_keypoints3d.py](./panoptic_config/eval_keypoints3d.py) | 98.88 | 85.69 | 67.78 | 93.93 | 90.69 | 62.81 | 91.56 | 84.48 | [log](https://openxrlab-share.oss-cn-hongkong.aliyuncs.com/xrmocap/logs/MVPose/panoptic.zip) | + +- **160422_haggling1** -| Config | AP25 | MPJPE(mm) | Download | -|:------:|:----:|:---------:|:--------:| -| - | - | - | [model]() | [log]() | +| Config | Actor 0 | Actor 1 | Actor 2 | Actor 3 | Actor 4 | Actor 5 | Actor 6 | Actor 7 | Actor 8 | Actor 9 | Actor 10 | Actor 11 | Actor 12 | Actor 13 | Actor 14 | Actor 15 | Actor 16 | Actor 17 | Average | Download | +|:-------:|:--------:|:--------:|:--------:|:--------:|:--------:|:-------:|:--------:|:--------:|:--------:|:--------:|:--------:|:-------:|:--------:|:--------:|:--------:|:--------:|:--------:|:--------:|:--------:|:--------:| +| [eval_keypoints3d.py](./panoptic_config/eval_keypoints3d.py) | 97.41 | 97.51 | 95.02 | 97.78 | 95.67 | 88.57 | 86.36 | 43.62 | 96.79 | 90.65 | 97.55 | 91.74 | 81.64 | 89.09 | 97.13 | 85.47 | 97.97 | 88.64 | 89.92 | [log](https://openxrlab-share.oss-cn-hongkong.aliyuncs.com/xrmocap/logs/MVPose/panoptic.zip) | diff --git a/configs/mvpose/campus_config/eval_keypoints3d.py b/configs/mvpose/campus_config/eval_keypoints3d.py index 811a204d..073a144a 100644 --- a/configs/mvpose/campus_config/eval_keypoints3d.py +++ b/configs/mvpose/campus_config/eval_keypoints3d.py @@ -1,11 +1,11 @@ type = 'TopDownAssociationEvaluation' __data_root__ = './xrmocap_data/CampusSeq1' -__meta_path__ = __data_root__ + '/xrmocap_meta_testset' +__meta_path__ = __data_root__ + '/xrmocap_meta_testset_fasterrcnn' __bbox_thr__ = 0.9 logger = None -output_dir = './output/campus' +output_dir = './output/mvpose/campus/fasterrcnn' pred_kps3d_convention = 'coco' eval_kps3d_convention = 'campus' selected_limbs_name = [ diff --git a/configs/mvpose/panoptic_config/eval_keypoints3d.py b/configs/mvpose/panoptic_config/eval_keypoints3d.py index 9cbc710c..7e803601 100644 --- a/configs/mvpose/panoptic_config/eval_keypoints3d.py +++ b/configs/mvpose/panoptic_config/eval_keypoints3d.py @@ -2,10 +2,10 @@ __data_root__ = './xrmocap_data/CMU-panoptic' __meta_path__ = __data_root__ + '/xrmocap_meta_testset' -__bbox_thr__ = 0.99 +__bbox_thr__ = 0.9 logger = None -output_dir = './output/cmu-panoptic' +output_dir = './output/mvpose/cmu-panoptic' pred_kps3d_convention = 'coco' eval_kps3d_convention = 'campus' selected_limbs_name = [ diff --git a/configs/mvpose/shelf_config/eval_keypoints3d.py b/configs/mvpose/shelf_config/eval_keypoints3d.py index 915f4eb4..41881a14 100644 --- a/configs/mvpose/shelf_config/eval_keypoints3d.py +++ b/configs/mvpose/shelf_config/eval_keypoints3d.py @@ -1,11 +1,11 @@ type = 'TopDownAssociationEvaluation' __data_root__ = './xrmocap_data/Shelf' -__meta_path__ = __data_root__ + '/xrmocap_meta_testset' +__meta_path__ = __data_root__ + '/xrmocap_meta_testset_fasterrcnn' __bbox_thr__ = 0.9 logger = None -output_dir = './output/shelf' +output_dir = './output/mvpose/shelf/fasterrcnn' pred_kps3d_convention = 'coco' eval_kps3d_convention = 'campus' selected_limbs_name = [ diff --git a/configs/mvpose_tracking/README.md b/configs/mvpose_tracking/README.md index 2e7de6d8..6c2988cc 100644 --- a/configs/mvpose_tracking/README.md +++ b/configs/mvpose_tracking/README.md @@ -1,5 +1,9 @@ # MVPose (Temporal tracking and filtering) +- [Introduction](#introduction) +- [Prepare models and datasets](#prepare-models-and-datasets) +- [Results and Models](#results-and-models) + ## Introduction We provide the config files for MVPose (Temporal tracking and filtering): [Fast and robust multi-person 3d pose estimation and tracking from multiple views](https://zju3dv.github.io/mvpose/). @@ -14,34 +18,101 @@ We provide the config files for MVPose (Temporal tracking and filtering): [Fast } ``` +## Prepare models and datasets + +- **Prepare models**: + +``` +sh scripts/download_weight.sh +``` +You can find `resnet50_reid_camstyle.pth.tar` in `weight` file. + +- **Prepare the datasets**: + +Convert original dataset to our unified meta-data, with data converters controlled by configs, +you can find more details in [dataset_preparation.md](../../docs/en/dataset_preparation.md). + +The final file structure would be like: + +```text +xrmocap +├── xrmocap +├── docs +├── tools +├── configs +└── weight + └── resnet50_reid_camstyle.pth.tar +└── xrmocap_data + ├── Shelf + ├── Camera0 + ├── ... + ├── Camera4 + ├── xrmocap_meta_testset_fasterrcnn + └── xrmocap_meta_testset + ├── CampusSeq1 + └── panoptic + ├── xrmocap_meta_ian5 + ├── hd_00_03 + ├── ... + ├── hd_00_23 + ├── camera_parameters + ├── keypoints3d_GT.npz + └── perception_2d.npz + ├── xrmocap_meta_pizza1 + ├── xrmocap_meta_band4 + └── xrmocap_meta_haggling1 +``` + ## Results and Models -We evaluate MVPose (Temporal tracking and filtering) on 3 popular benchmarks. +We evaluate MVPose (Temporal tracking and filtering) on 3 popular benchmarks, report the Percentage of Correct Parts (PCP) on Shelf/Campus/CMU-Panoptic datasets. -[Explain common setting and metrics] +You can find the recommended configs in `configs/mvpose_tracking/*/eval_keypoints3d.py`, where `interval` is the global matching interval, that is, the maximum number of frames for Kalman filtering. If the interval is set too large, the accuracy of the estimation will be degraded, so we recommen within 50 frames. `__bbox_thr__` is the threshold of bbox2d, you can set a high threshold to ignore incorrect 2D perception data, and we recommen setting it to 0.9. `best_distance` is the threshold at which the current-frame keypoints2d successfully matches the last-frame keypoints2d, for the different dataset, it needs to be adjusted. `n_cam_min` is the amount of views required for triangulation, which defaults to 2. ### Campus -[Explain specific setting, metrics and other details] +The 2D perception data we use is generated by fasterrcnn, and you can download it from [here](/docs/en/dataset_preparation.md#download-converted-meta-data). What's more, we set `__bbox_thr__=0.9`, `n_cam_min=2` and `interval=10`. -| Config | Campus | Download | -|:------:|:-------:|:--------:| -| - | - | [model]() | [log]() | +| Config | Actor 0 | Actor 1 | Actor 2 | Average | Download | +|:------:|:-------:|:--------:|:------:|:-------:|:--------:| +| [eval_keypoints3d.py](./campus_config/eval_keypoints3d.py) | 94.79 | 88.46 | 98.38 | 93.88 | [log](https://openxrlab-share.oss-cn-hongkong.aliyuncs.com/xrmocap/logs/MVPoseTracking/campus.zip) | ### Shelf -[Explain specific setting, metrics and other details] +The 2D perception data we use is generated by fasterrcnn, and you can download it from [here](/docs/en/dataset_preparation.md#download-converted-meta-data). What's more, we set `__bbox_thr__=0.9`, `n_cam_min=3` and `interval=5`. -| Config | Shelf | Download | -|:------:|:-------:|:--------:| -| - | - | [model]() | [log]() | +| Config | Actor 0 | Actor 1 | Actor 2 | Average | Download | +|:------:|:-------:|:--------:|:------:|:-------:|:--------:| +| [eval_keypoints3d.py](./shelf_config/eval_keypoints3d.py) | 98.28 | 95.41 | 97.83 | 97.17 | [log](https://openxrlab-share.oss-cn-hongkong.aliyuncs.com/xrmocap/logs/MVPoseTracking/shelf.zip) | ### CMU Panoptic -[Explain view selection, training details etc.] +The 2D perception data we use is generated by mmpose, and you can download it from [here](/docs/en/dataset_preparation.md#download-converted-meta-data). The selection principle of the camera is to cover as much information as possible about the human body, so we selected cameras 3, 6, 12, 13 and 23. You can find more details in the [config](panoptic_config/eval_keypoints3d.py). + +The CMU Panoptic dataset contains four sequences that share the same config file. For different sequences, you need to change the `__meta_path__`. In addition, there is a child in `160906_ian5` sequence, and the number of cameras that capture him decreases, so we set `__bbox_thr__=0.85` and `n_cam_min=2`. For other sequences, we set `__bbox_thr__=0.9` and `n_cam_min=3`. + +- **160906_band4** + +| Config | Actor 0 | Actor 1 | Actor 2 | Average | Download | +|:-------:|:--------:|:--------:|:--------:|:--------:|:--------:| +| [eval_keypoints3d.py](./panoptic_config/eval_keypoints3d.py) | 98.46 | 83.35 | 94.23 | 92.01 | [log](https://openxrlab-share.oss-cn-hongkong.aliyuncs.com/xrmocap/logs/MVPoseTracking/panoptic.zip) | + +- **160906_ian5** + +| Config | Actor 0 | Actor 1 | Average | Download | +|:-------:|:--------:|:--------:|:--------:|:--------:| +| [eval_keypoints3d.py](./panoptic_config/eval_keypoints3d.py) | 97.52 | 71.12 | 84.32 | [log](https://openxrlab-share.oss-cn-hongkong.aliyuncs.com/xrmocap/logs/MVPoseTracking/panoptic.zip) | + +- **160906_pizza1** + +| Config | Actor 0 | Actor 1 | Actor 2 | Actor 3 | Actor 4 | Actor 5 | Actor 6 | Average | Download | +|:-------:|:--------:|:--------:|:--------:|:--------:|:--------:|:--------:|:--------:|:--------:|:--------:| +| [eval_keypoints3d.py](./panoptic_config/eval_keypoints3d.py) | 98.66 | 85.58 | 68.65 | 95.05 | 93.05 | 67.09 | 94.88 | 86.14 | [log](https://openxrlab-share.oss-cn-hongkong.aliyuncs.com/xrmocap/logs/MVPoseTracking/panoptic.zip) | + +- **160422_haggling1** -| Config | AP25 | MPJPE(mm) | Download | -|:------:|:----:|:---------:|:--------:| -| - | - | - | [model]() | [log]() | +| Config | Actor 0 | Actor 1 | Actor 2 | Actor 3 | Actor 4 | Actor 5 | Actor 6 | Actor 7 | Actor 8 | Actor 9 | Actor 10 | Actor 11 | Actor 12 | Actor 13 | Actor 14 | Actor 15 | Actor 16 | Actor 17 | Average | Download | +|:-------:|:--------:|:--------:|:--------:|:--------:|:--------:|:-------:|:--------:|:--------:|:--------:|:--------:|:--------:|:-------:|:--------:|:--------:|:--------:|:--------:|:--------:|:--------:|:--------:|:--------:| +| [eval_keypoints3d.py](./panoptic_config/eval_keypoints3d.py) | 97.58 | 98.16 | 94.61 | 98.01 | 96.27 | 87.30 | 84.56 | 43.72 | 96.59 | 90.76 | 97.64 | 91.49 | 77.27 | 87.94 | 96.76 | 82.84 | 98.15 | 91.10 | 89.49 | [log](https://openxrlab-share.oss-cn-hongkong.aliyuncs.com/xrmocap/logs/MVPoseTracking/panoptic.zip) | diff --git a/configs/mvpose_tracking/campus_config/eval_keypoints3d.py b/configs/mvpose_tracking/campus_config/eval_keypoints3d.py index f036c47f..3764af77 100644 --- a/configs/mvpose_tracking/campus_config/eval_keypoints3d.py +++ b/configs/mvpose_tracking/campus_config/eval_keypoints3d.py @@ -1,11 +1,11 @@ type = 'TopDownAssociationEvaluation' __data_root__ = './xrmocap_data/CampusSeq1' -__meta_path__ = __data_root__ + '/xrmocap_meta_testset' +__meta_path__ = __data_root__ + '/xrmocap_meta_testset_fasterrcnn' __bbox_thr__ = 0.9 logger = None -output_dir = './output/campus' +output_dir = './output/mvpose_tracking/campus/fasterrcnn' pred_kps3d_convention = 'coco' eval_kps3d_convention = 'campus' selected_limbs_name = [ @@ -47,8 +47,8 @@ 'right_hip_extra' ]), checkpoint_path='./weight/resnet50_reid_camstyle.pth.tar', - best_distance=100, - interval=5, + best_distance=90, + interval=10, bbox_thr=__bbox_thr__, device='cuda', logger=logger, diff --git a/configs/mvpose_tracking/panoptic_config/eval_keypoints3d.py b/configs/mvpose_tracking/panoptic_config/eval_keypoints3d.py index 22b1a7b4..5c881fc3 100644 --- a/configs/mvpose_tracking/panoptic_config/eval_keypoints3d.py +++ b/configs/mvpose_tracking/panoptic_config/eval_keypoints3d.py @@ -2,10 +2,10 @@ __data_root__ = './xrmocap_data/CMU-panoptic' __meta_path__ = __data_root__ + '/xrmocap_meta_testset' -__bbox_thr__ = 0.99 +__bbox_thr__ = 0.9 logger = None -output_dir = './output/cmu-panoptic' +output_dir = './output/mvpose_tracking/cmu-panoptic' pred_kps3d_convention = 'coco' eval_kps3d_convention = 'campus' selected_limbs_name = [ diff --git a/configs/mvpose_tracking/shelf_config/eval_keypoints3d.py b/configs/mvpose_tracking/shelf_config/eval_keypoints3d.py index f61f57e8..ba40d85d 100644 --- a/configs/mvpose_tracking/shelf_config/eval_keypoints3d.py +++ b/configs/mvpose_tracking/shelf_config/eval_keypoints3d.py @@ -1,11 +1,11 @@ type = 'TopDownAssociationEvaluation' __data_root__ = './xrmocap_data/Shelf' -__meta_path__ = __data_root__ + '/xrmocap_meta_testset' +__meta_path__ = __data_root__ + '/xrmocap_meta_testset_fasterrcnn' __bbox_thr__ = 0.9 logger = None -output_dir = './output/shelf' +output_dir = './output/mvpose_tracking/shelf/fasterrcnn' pred_kps3d_convention = 'coco' eval_kps3d_convention = 'campus' selected_limbs_name = [ diff --git a/docs/en/apis.md b/docs/en/apis.md index f3897d4e..77932b14 100644 --- a/docs/en/apis.md +++ b/docs/en/apis.md @@ -1,13 +1,14 @@ # APIs -[TOC] +- [Multi-view single-person SMPL Estimator](#multi-view-single-person-smpl-estimator) +- [Multi-view multi-person SMPL Estimator](#multi-view-multi-person-smpl-estimator) -### Multi-view single-person SMPL Estimator +## Multi-view single-person SMPL Estimator -`MultiViewMultiPersonSMPLEstimator` is an API class for multi-view single-person scenario, taking multi-view videos and multi-view camera parameters as input, estimating SMPL parameters and some other important information. See the [estimator doc](./estimation/mview_sperson_smpl_estimator.md) for details. +`MultiViewSinglePersonSMPLEstimator` is an API class for multi-view single-person scenario, taking multi-view videos and multi-view camera parameters as input, estimating SMPL parameters and some other important information. See the [estimator doc](./estimation/mview_sperson_smpl_estimator.md) for details. -### Multi-view multi-person SMPL Estimator +## Multi-view multi-person SMPL Estimator -TODO: add brief for this estimator. +`MultiViewMultiPersonSMPLEstimator` is an API class for multi-view multi-person scenario, taking multi-person keypoints3d and multi-view camera parameters as input, estimating SMPL parameters and some other important information. See the [estimator doc](./tools/mview_mperson_smplify3d.md) for details. diff --git a/docs/en/benchmark.md b/docs/en/benchmark.md index 0782178e..9f71d66c 100644 --- a/docs/en/benchmark.md +++ b/docs/en/benchmark.md @@ -12,7 +12,7 @@ Please refer to [MVPose](../../configs/mvpose/README.md) for details. ### MVPose (Temporal tracking and filtering) -Please refer to [MVPose with tracking)](../../configs/mvpose_tracking/README.md) for details. +Please refer to [MVPose with tracking](../../configs/mvpose_tracking/README.md) for details. ### Shape-aware 3D Pose Optimization @@ -20,4 +20,4 @@ TBA ### MvP -Please refer to [MvP](../../configs/mvp/README.md) for details. +Please refer to [MvP benchmarks](../../configs/mvp/README.md) for details. diff --git a/docs/en/changelog.md b/docs/en/changelog.md index 40f5f1e6..5d97eac8 100644 --- a/docs/en/changelog.md +++ b/docs/en/changelog.md @@ -1,5 +1,5 @@ # Changelog -### v0.5.0 (22/08/2022/) +### v0.5.0 (01/09/2022/) write main features diff --git a/docs/en/data_structure/keypoints.md b/docs/en/data_structure/keypoints.md index 0d626eac..765eb644 100644 --- a/docs/en/data_structure/keypoints.md +++ b/docs/en/data_structure/keypoints.md @@ -1,8 +1,14 @@ # Keypoints -[TOC] - - +- [Overview](#overview) +- [Key/Value definition](#keyvalue-definition) +- [Attribute definition](#attribute-definition) +- [Name convention](#name-convention) +- [Create an instance](#create-an-instance) +- [Auto-completion](#auto-completion) +- [Convert between numpy and torch](#convert-between-numpy-and-torch) +- [File IO](#file-io) +- [Keypoints convention](#keypoints-convention) ### Overview diff --git a/docs/en/data_structure/limbs.md b/docs/en/data_structure/limbs.md index c0404e84..b7261274 100644 --- a/docs/en/data_structure/limbs.md +++ b/docs/en/data_structure/limbs.md @@ -1,6 +1,8 @@ # Limbs -[TOC] +- [Overview](#overview) +- [Attribute definition](#attribute-definition) +- [Create an instance](#create-an-instance) ### Overview diff --git a/docs/en/data_structure/smpl_data.md b/docs/en/data_structure/smpl_data.md index 13e1515e..ec86a751 100644 --- a/docs/en/data_structure/smpl_data.md +++ b/docs/en/data_structure/smpl_data.md @@ -1,8 +1,11 @@ # SMPLData -[TOC] - - +- [Overview](#overview) +- [Key/Value definition](#keyvalue-definition) +- [Attribute definition](#attribute-definition) +- [Create an instance](#create-an-instance) +- [Convert into body_model input](#convert-into-body_model-input) +- [File IO](#file-io) ### Overview diff --git a/docs/en/dataset_preparation.md b/docs/en/dataset_preparation.md index d2229be2..38e9a498 100644 --- a/docs/en/dataset_preparation.md +++ b/docs/en/dataset_preparation.md @@ -1,6 +1,10 @@ # Dataset preparation -[TOC] +- [Overview](#overview) +- [Supported datasets](#supported-datasets) +- [Download converted meta-data](#download-converted-meta-data) +- [Convert a dataset manually](#convert-a-dataset-manually) +- [Validate converted meta-data by visualization](#validate-converted-meta-data-by-visualization) ### Overview @@ -8,36 +12,41 @@ Our data pipeline converts original dataset to our unified meta-data, with data ### Supported datasets -| Dataset name | Dataset page | Download from public | Download from OpenXRLab | -| ------------ | ---------------------------------------------------------- | ------------------------------------------------------------ | ------------------------------------------------------------ | -| Campus | [Home page](https://campar.in.tum.de/Chair/MultiHumanPose) | [CampusSeq1.tar.bz2](https://www.campar.in.tum.de/public_datasets/2014_cvpr_belagiannis/CampusSeq1.tar.bz2) | [CampusSeq1.tar.bz2](http://10.4.11.59:18080/resources/XRlab/dataset/CampusSeq1.tar.bz2) | -| Shelf | [Home page](https://campar.in.tum.de/Chair/MultiHumanPose) | [Shelf.tar.bz2](https://www.campar.in.tum.de/public_datasets/2014_cvpr_belagiannis/Shelf.tar.bz2) | [Shelf.tar.bz2](http://10.4.11.59:18080/resources/XRlab/dataset/Shelf.tar.bz2) | -| CMU Panoptic | [Home page](http://domedb.perception.cs.cmu.edu/) | By [official script](https://github.com/CMU-Perceptual-Computing-Lab/panoptic-toolbox/blob/master/scripts/getData.sh) | N/A | - -### Convert a dataset manually - -Use our prepare_dataset tool to convert a dataset. See the [tool tutorial](./tool/prepare_dataset.md) for details. +| Dataset name | Dataset page | Download | +| ------------ | ---------------------------------------------------------- | ------------------------------------------------------------ | +| Campus | [Home page](https://campar.in.tum.de/Chair/MultiHumanPose) | [CampusSeq1.tar.bz2](https://www.campar.in.tum.de/public_datasets/2014_cvpr_belagiannis/CampusSeq1.tar.bz2) | +| Shelf | [Home page](https://campar.in.tum.de/Chair/MultiHumanPose) | [Shelf.tar.bz2](https://www.campar.in.tum.de/public_datasets/2014_cvpr_belagiannis/Shelf.tar.bz2) | +| CMU Panoptic | [Home page](http://domedb.perception.cs.cmu.edu/) | By [official script](https://github.com/CMU-Perceptual-Computing-Lab/panoptic-toolbox/blob/master/scripts/getData.sh) | ### Download converted meta-data Considering that it takes long to run a converter if perception2d is checked, we have done it for you. Our perception 2D is generated by mmtrack and mmpose, defined in coco_wholebody by default. You can download compressed zip file for converted meta-data below. +For where to put the downloaded meta-data, check [xrmocap dataset structure](tutorials/new_dataset.md#file-tree-of-our-unified-format) for details. + | Dataset name | meta name | Download link | Notes | | ------------ | ------------------ | ------------------------------------------------------------ | ------------------------------------------------------------ | -| Campus | testset | [download](http://10.4.11.59:18080/resources/XRlab/dataset/meta-data/CampusSeq1/xrmocap_meta_testset.zip) | | -| Campus | testset_mvpose2d | [download](http://10.4.11.59:18080/resources/XRlab/dataset/meta-data/CampusSeq1/xrmocap_meta_testset_mvpose2d.zip) | Perception 2D is generated by [MVPose](https://github.com/zju3dv/mvpose#accelerate-the-evaluation), defined in coco convention. | -| Campus | trainset | [download](http://10.4.11.59:18080/resources/XRlab/dataset/meta-data/CampusSeq1/xrmocap_meta_trainset.zip) | | -| Campus | trainset_pesudo_gt | [download](http://10.4.11.59:18080/resources/XRlab/dataset/meta-data/CampusSeq1/xrmocap_meta_trainset_pesudo_gt.zip) | Ground-truth keypoints3d is generated by [MvP](https://github.com/sail-sg/mvp#22-shelfcampus), defined in campus convention. | -| Shelf | testset | [download](http://10.4.11.59:18080/resources/XRlab/dataset/meta-data/Shelf/xrmocap_meta_testset.zip) | | -| Shelf | testset_mvpose2d | [download](http://10.4.11.59:18080/resources/XRlab/dataset/meta-data/Shelf/xrmocap_meta_testset_mvpose2d.zip) | Perception 2D is generated by [MVPose](https://github.com/zju3dv/mvpose#accelerate-the-evaluation), defined in coco. There's only data for the first three people in ground truth keypoints3d . | -| Shelf | trainset | [download](http://10.4.11.59:18080/resources/XRlab/dataset/meta-data/Shelf/xrmocap_meta_trainset.zip) | | -| Shelf | trainset_pesudo_gt | [download](http://10.4.11.59:18080/resources/XRlab/dataset/meta-data/Shelf/xrmocap_meta_trainset_pesudo_gt.zip) | Ground-truth keypoints3d is generated by [MvP](https://github.com/sail-sg/mvp#22-shelfcampus), defined in campus convention. | -| CMU Panoptic | 160906_band4 | [download](http://10.4.11.59:18080/resources/XRlab/dataset/meta-data/Panoptic/xrmocap_meta_band4.zip) | Only five views are selected: 03, 06, 12, 13, 23 | -| CMU Panoptic | 160906_ian5 | [download](http://10.4.11.59:18080/resources/XRlab/dataset/meta-data/Panoptic/xrmocap_meta_ian5.zip) | Only five views are selected: 03, 06, 12, 13, 23 | -| CMU Panoptic | 160906_pizza1 | [download](http://10.4.11.59:18080/resources/XRlab/dataset/meta-data/Panoptic/xrmocap_meta_pizza1.zip) | Only five views are selected: 03, 06, 12, 13, 23 | +| Campus | testset | [download](https://openxrlab-share.oss-cn-hongkong.aliyuncs.com/xrmocap/xrmocap_meta/Campus/xrmocap_meta_testset.zip) | | +| Campus | testset_fasterrcnn | [download](https://openxrlab-share.oss-cn-hongkong.aliyuncs.com/xrmocap/xrmocap_meta/Campus/xrmocap_meta_testset_fasterrcnn.zip) | Bbox 2D is generated by [mmdet Faster R-CNN](https://github.com/open-mmlab/mmdetection/tree/master/configs/faster_rcnn). | +| ..Campus | testset_mvpose2d | [download](https://openxrlab-share.oss-cn-hongkong.aliyuncs.com/xrmocap/xrmocap_meta/Campus/xrmocap_meta_testset_mvpose2d.zip) | Perception 2D is generated by [MVPose](https://github.com/zju3dv/mvpose#accelerate-the-evaluation), defined in coco convention. | +| Campus | trainset | [download](https://openxrlab-share.oss-cn-hongkong.aliyuncs.com/xrmocap/xrmocap_meta/Campus/xrmocap_meta_trainset.zip) | | +| Campus | trainset_pesudo_gt | [download](https://openxrlab-share.oss-cn-hongkong.aliyuncs.com/xrmocap/xrmocap_meta/Campus/xrmocap_meta_trainset_pesudo_gt.zip) | Ground-truth keypoints3d is generated by [MvP](https://github.com/sail-sg/mvp#22-shelfcampus), defined in campus convention. | +| Shelf | testset | [download](https://openxrlab-share.oss-cn-hongkong.aliyuncs.com/xrmocap/xrmocap_meta/Shelf/xrmocap_meta_testset.zip) | | +| Shelf | testset_fasterrcnn | [download](https://openxrlab-share.oss-cn-hongkong.aliyuncs.com/xrmocap/xrmocap_meta/Shelf/xrmocap_meta_testset_fasterrcnn.zip) | Bbox 2D is generated by [mmdet Faster R-CNN](https://github.com/open-mmlab/mmdetection/tree/master/configs/faster_rcnn). | +| Shelf | testset_mvpose2d | [download](https://openxrlab-share.oss-cn-hongkong.aliyuncs.com/xrmocap/xrmocap_meta/Shelf/xrmocap_meta_testset_mvpose2d.zip) | Perception 2D is generated by [MVPose](https://github.com/zju3dv/mvpose#accelerate-the-evaluation), defined in coco. There's only data for the first three people in ground truth keypoints3d . | +| Shelf | trainset | [download](https://openxrlab-share.oss-cn-hongkong.aliyuncs.com/xrmocap/xrmocap_meta/Shelf/xrmocap_meta_trainset.zip) | | +| Shelf | trainset_pesudo_gt | [download](https://openxrlab-share.oss-cn-hongkong.aliyuncs.com/xrmocap/xrmocap_meta/Shelf/xrmocap_meta_trainset_pesudo_gt.zip) | Ground-truth keypoints3d is generated by [MvP](https://github.com/sail-sg/mvp#22-shelfcampus), defined in campus convention. | +| CMU Panoptic | 160906_band4 | [download](https://openxrlab-share.oss-cn-hongkong.aliyuncs.com/xrmocap/xrmocap_meta/Panoptic/xrmocap_meta_band4.zip) | Only five views are selected: 03, 06, 12, 13, 23 | +| CMU Panoptic | 160422_haggling1 | [download](https://openxrlab-share.oss-cn-hongkong.aliyuncs.com/xrmocap/xrmocap_meta/Panoptic/xrmocap_meta_haggling1.zip) | Only five views are selected: 03, 06, 12, 13, 23 | +| CMU Panoptic | 160906_ian5 | [download](https://openxrlab-share.oss-cn-hongkong.aliyuncs.com/xrmocap/xrmocap_meta/Panoptic/xrmocap_meta_ian5.zip) | Only five views are selected: 03, 06, 12, 13, 23 | +| CMU Panoptic | 160906_pizza1 | [download](https://openxrlab-share.oss-cn-hongkong.aliyuncs.com/xrmocap/xrmocap_meta/Panoptic/xrmocap_meta_pizza1.zip) | Only five views are selected: 03, 06, 12, 13, 23 | For CMU panoptic meta-data, frames extracted from videos have been removed before uploading. One has to convert panoptic data locally with `bbox_detector = None` and `kps2d_estimator = None` first, and then copy download data into the converted meta-data directory. +### Convert a dataset manually + +Use our prepare_dataset tool to convert a dataset. See the [tool tutorial](./tool/prepare_dataset.md) for details. + ### Validate converted meta-data by visualization Use our visualize_dataset tool to visualize meta-data. See the [tool tutorial](./tool/visualize_dataset.md) for details. diff --git a/docs/en/estimation/mview_sperson_smpl_estimator.md b/docs/en/estimation/mview_sperson_smpl_estimator.md index b1401ef4..7fa86913 100644 --- a/docs/en/estimation/mview_sperson_smpl_estimator.md +++ b/docs/en/estimation/mview_sperson_smpl_estimator.md @@ -1,6 +1,12 @@ # Multi-view single-person SMPL Estimator -[TOC] +- [Overview](#overview) +- [Arguments](#arguments) +- [Run](#run) + - [Step0: estimate_keypoints2d](#step0-estimate_keypoints2d) + - [Step1: estimate_keypoints3d](#step1-estimate_keypoints3d) + - [Step2: estimate_smpl](#step2-estimate_smpl) +- [Example](#example) ### Overview diff --git a/docs/en/faq.md b/docs/en/faq.md index 314da88a..9dd006a3 100644 --- a/docs/en/faq.md +++ b/docs/en/faq.md @@ -6,13 +6,13 @@ We list some common troubles faced by many users and their corresponding solutio - 'ImportError: libpng16.so.16: cannot open shared object file: No such file or directory' - Please refer to [xrprimer faq](https://gitlab.bj.sensetime.com/openxrlab/xrprimer/-/blob/xrprimer_ee_dev/docs/en/faq.md). + Please refer to [xrprimer faq](https://github.com/openxrlab/xrprimer/blob/main/docs/en/faq.md). - 'ImportError: liblapack.so.3: cannot open shared object file: No such file or directory' - Please refer to [xrprimer faq](https://gitlab.bj.sensetime.com/openxrlab/xrprimer/-/blob/xrprimer_ee_dev/docs/en/faq.md). + Please refer to [xrprimer faq](https://github.com/openxrlab/xrprimer/blob/main/docs/en/faq.md). -- 'ModuleNotFoundError: No module named 'mmhuman3d.core.conventions.joints_mapping'' +- 'ModuleNotFoundError: No module named mmhuman3d.core.conventions.joints_mapping' Package `joints_mapping` actually exists in [github](https://github.com/open-mmlab/mmhuman3d/tree/main/mmhuman3d/core/conventions/joints_mapping), but it is not installed by pip for absence of `joints_mapping/__init__.py`. Install mmhuman3d from source will solve it: @@ -21,3 +21,11 @@ We list some common troubles faced by many users and their corresponding solutio git clone https://github.com/open-mmlab/mmhuman3d.git pip install -e ./mmhuman3d ``` + +- 'BrokenPipeError: ../../lib/python3.8/site-packages/xrprimer/utils/ffmpeg_utils.py:189: BrokenPipeError' + + You've installed a wrong version of ffmpeg. Try to install it by the following command, and do not specify any channel: + + ```bash + conda install ffmpeg + ``` diff --git a/docs/en/getting_started.md b/docs/en/getting_started.md index 71ecb0a4..3636e148 100644 --- a/docs/en/getting_started.md +++ b/docs/en/getting_started.md @@ -1,4 +1,12 @@ -# Get started +# Getting started + +- [Installation](#installation) +- [Data Preparation](#data-preparation) +- [Body Model Preparation (Optional)](#body-model-preparation-optional) +- [Inference](#inference) +- [Evaluation](#evaluation) +- [Training](#training) +- [More tutorials](#more-tutorials) ## Installation @@ -23,7 +31,7 @@ Download the above resources and arrange them in the following file structure: ```text xrmocap -├── xrmoccap +├── xrmocap ├── docs ├── tests ├── tools @@ -37,102 +45,172 @@ xrmocap └── SMPL_NEUTRAL.pkl ``` -## Inference / Demo +## Inference We provide a demo script to estimate SMPL parameters for single-person or multi-person from multi-view synchronized input images or videos. With this demo script, you only need to choose a method, we currently support two types of methods, namely, optimization-based approaches and end-to-end learning algorithms, specify a few arguments, and then you can get the estimated results. -We assume that the cameras have been calibrated. If you want to know more about camera calibration, refer to [XRPrimer]() for more details. +We assume that the cameras have been calibrated. If you want to know more about camera calibration, refer to [XRPrimer](https://github.com/openxrlab/xrprimer/blob/main/docs/en/tools/calibrate_multiple_cameras.md) for more details. + ### Perception Model - - **Prepare CamStyle models**: -You can find CamStyle model in `weight` file +Prepare perceptions models, including detection, 2d pose estimation, tracking and CamStyle models. + +``` +sh scripts/download_weight.sh +``` +You could find `resnet50_reid_camstyle.pth.tar` in `weight` file. ### Single Person Currently, we only provide optimization-based method for single person estimation. +1. Download a `.smc` file from [humman dataset](https://drive.google.com/drive/folders/17dinze70MWL5PmB9-Mw36zUjkrQvwb-J). +2. Extract the 7z file. + ```bash -xxx +7z x p000127_a000007.7z ``` -The above code is supposed to run successfully upon you finish the installation. +3. Run [process_smc](./tools/process_smc.md) tool. -### Multiple persons +```bash +mkdir xrmocap_data/humman +python tools/process_smc.py \ + --estimator_cofig configs/humman_mocap/mview_sperson_smpl_estimator.py \ + --smc_path p000127_a000007.smc \ + --output_dir xrmocap_data/humman/p000127_a000007_output \ + --visualize +``` + + +### Multiple People -For optimization-based approaches, it does not require any pretrained model. With downloaded datasets, it can be run as +A small test dataset for quick demo can be downloaded [here](https://openxrlab-share.oss-cn-hongkong.aliyuncs.com/xrmocap/example_resources/Shelf_50.zip). It contains 50 frames from the Shelf sequence, with 5 camera views calibrated and synchronized. + +#### Optimization-based methods + +For optimization-based approaches, it does not require any pretrained model. Taking [MVPose](../../configs/mvpose/) as an example, it can be run on Shelf_50 as ```bash -python tool/estimate_keypoints3d.py --config ./config/kps3d_estimation/shelf_config/estimate_kps3d.py +Coming soon! ``` Some useful configs are explained here: - If you want to use tracing on the input sequence, you can set `use_kalman_tracking` to True in config file. -For learning-based methods, we provide pretrained models in [model_zoo](), it can be downloaded and run the script as below. +#### Learning-based methods + +For learning-based methods, it resorts to an end-to-end learning scheme so as to require training before inference. +Taking [MvP](../../configs/mvp/) as an example, we can download [pretrained MvP model](https://openxrlab-share.oss-cn-hongkong.aliyuncs.com/xrmocap/weight/mvp/xrmocap_mvp_shelf-22d1b5ed_20220831.pth) and run it on Shelf_50 as: + +1. Install `Deformable` package + +Download the [`./ops`](https://github.com/sail-sg/mvp/tree/main/lib/models/ops) folder, rename and place the folder as `xrmocap/model/deformable`. Install `Deformable` by running: +``` +cd xrmocap/model/deformable/ +sh make.sh +``` + +2. Download data and run demo ```bash -sh script/eval_mvp.sh +# download data +mkdir -p xrmocap_data +wget https://openxrlab-share.oss-cn-hongkong.aliyuncs.com/xrmocap/example_resources/Shelf_50.zip -P xrmocap_dataa +cd xrmocap_data/ && unzip -q Shelf_50.zip && rm Shelf_50.zip && cd .. + +# download pretrained model +mkdir -p weight/mvp +wget https://openxrlab-share.oss-cn-hongkong.aliyuncs.com/xrmocap/weight/mvp/xrmocap_mvp_shelf-22d1b5ed_20220831.pth -P weight/mvp + +sh ./scripts/eval_mvp.sh 1 configs/mvp/shelf_config/mvp_shelf_50.py weight/mvp/xrmocap_mvp_shelf-22d1b5ed_20220831.pth ``` -## Evaluation +For detailed tutorials about dataset preparation, model weights and checkpoints download for learning-based methods, please refer to the [training tutorial](./tools/train_model.md) and [evaluation tutorial](./tools/eval_model.md). + -We provide pretrained models in the respective method folders in [config](config). +## Evaluation ### Evaluate with a single GPU / multiple GPUs -For optimization-based methods, +#### Optimization-based methods -```shell -# better to provide a script like beloew -# python tools/test.py ${CONFIG} --work-dir=${WORK_DIR} ${CHECKPOINT} --metrics=${METRICS} +- Evaluate on the Shelf dataset and run the tool without tracking. + +```bash +python tools/mview_mperson_evaluation.py \ + --enable_log_file \ + --evaluation_config configs/mvpose/shelf_config/eval_keypoints3d.py ``` -Evaluate on the Shelf/Campus/CMU Panoptic datasets +- Evaluate on the Shelf dataset and run the tool with tracking. -Example: -```shell -python xrmocap/core/evaluation/evaluate_keypoints3d.py --config ./config/kps3d_estimation/eval_kps3d_estimation.py +```bash +python tools/mview_mperson_evaluation.py \ + --enable_log_file \ + --evaluation_config configs/mvpose_tracking/shelf_config/eval_keypoints3d.py ``` -For learning-based methods, with the downloaded pretrained models from [model_zoo](): +#### Learning-based methods -```bash -sh script/slurm_eval_mvp.sh +For learning-based methods, more details about dataset preparation, model weights and checkpoints download can be found at [evaluation tutorial](./tools/eval_model.md). + +With the downloaded pretrained MvP models from [model_zoo](./benchmark.md): + +```shell +sh ./scripts/val_mvp.sh ${NUM_GPUS} ${CFG_FILE} ${MODEL_PATH} ``` +Example: +```shell +sh ./scripts/val_mvp.sh 8 configs/mvp/shelf_config/mvp_shelf.py weight/xrmocap_mvp_shelf.pth.tar +``` + + ### Evaluate with slurm -If you can run XRMoCap on a cluster managed with [slurm](https://slurm.schedmd.com/), you can use the script `slurm_test.sh`. +If you can run XRMoCap on a cluster managed with [slurm](https://slurm.schedmd.com/), you can use the script `scripts/slurm_eval_mvp.sh`. ```shell -./tools/slurm_test.sh ${PARTITION} ${JOB_NAME} ${CONFIG} ${WORK_DIR} ${CHECKPOINT} --metrics ${METRICS} +sh ./scripts/slurm_eval_mvp.sh ${PARTITION} ${NUM_GPUS} ${CFG_FILE} ${MODEL_PATH} ``` + Example: ```shell -./tools/slurm_test.sh my_partition test_hmr configs/hmr/resnet50_hmr_pw3d.py work_dirs/hmr work_dirs/hmr/latest.pth 8 --metrics pa-mpjpe mpjpe +sh ./scripts/slurm_eval_mvp.sh ${PARTITION} 8 configs/mvp/shelf_config/mvp_shelf.py weight/xrmocap_mvp_shelf.pth.tar ``` ## Training +Training is only applicable to learning-based methods. + ### Training with a single / multiple GPUs -```shell -python tools/train.py ${CONFIG_FILE} ${WORK_DIR} --no-validate +To train the learning-based model, such as a MvP model, follow the [training tutorial](./tools/train_model.md) to prepare the datasets and pre-trained weights: + ``` -Example: using 1 GPU to train HMR. -```shell -python tools/train.py ${CONFIG_FILE} ${WORK_DIR} --gpus 1 --no-validate +sh ./scripts/train_mvp.sh ${NUM_GPUS} ${CFG_FILE} +``` +Example: + +``` +sh ./scripts/train_mvp.sh 8 configs/mvp/campus_config/mvp_campus.py + ``` ### Training with Slurm -If you can run XRMoCap on a cluster managed with [slurm](https://slurm.schedmd.com/), you can use the script `slurm_train.sh`. +If you can run XRMoCap on a cluster managed with [slurm](https://slurm.schedmd.com/), you can use the script `scripts/slurm_train_mvp.sh`. ```shell -./tools/slurm_train.sh ${PARTITION} ${JOB_NAME} ${CONFIG_FILE} ${WORK_DIR} ${GPU_NUM} --no-validate +sh ./scripts/slurm_train_mvp.sh ${PARTITION} ${NUM_GPUS} ${CFG_FILE} +``` +Example: +```shell +sh ./scripts/slurm_train_mvp.sh ${PARTITION} 8 configs/mvp/shelf_config/mvp_shelf.py ``` @@ -141,3 +219,4 @@ If you can run XRMoCap on a cluster managed with [slurm](https://slurm.schedmd.c - [Introduction](./tutorials/introduction.md) - [Config](./tutorials/config.md) - [New dataset](./tutorials/new_dataset.md) +- [New module](./tutorials/new_module.md) diff --git a/docs/en/installation.md b/docs/en/installation.md index 78c592d3..53b9353c 100644 --- a/docs/en/installation.md +++ b/docs/en/installation.md @@ -1,14 +1,11 @@ # Installation - - -- [Installation](#installation) - - [Requirements](#requirements) - - [Prepare environment](#prepare-environment) - - [A from-scratch setup script](#a-from-scratch-setup-script) - - [Run with docker image](#run-with-docker-image) - - +- [Requirements](#requirements) +- [A from-scratch setup script](#a-from-scratch-setup-script) +- [Prepare environment](#prepare-environment) +- [Run with docker image](#run-with-docker-image) +- [Test environment](#test-environment) +- [Frequently Asked Questions](#frequently-asked-questions) ## Requirements @@ -18,7 +15,7 @@ - PyTorch 1.6.0, 1.7.0, 1.7.1, 1.8.0, 1.8.1, 1.9.0 or 1.9.1. - CUDA 9.2+ - GCC 5+ -- [XRPrimer](https://gitlab.bj.sensetime.com/openxrlab/xrprimer) +- [XRPrimer](https://github.com/openxrlab/xrprimer) - [MMHuman3D](https://github.com/open-mmlab/mmhuman3d) - [MMCV](https://github.com/open-mmlab/mmcv) @@ -31,29 +28,6 @@ Optional: | [MMTracking](https://github.com/open-mmlab/mmtracking) | Multiple object tracking. | Install `mmcv-full`, instead of `mmcv`. | | [Aniposelib](https://github.com/google/aistplusplus_api) | Triangulation. | Install from [github](https://github.com/liruilong940607/aniposelib), instead of pypi. | -## Prepare environment - -##### a. Create a conda virtual environment and activate it. - -```shell -conda create -n xrmocap python=3.8 -y -conda activate xrmocap -``` - -##### b. Install MMHuman3D following the [official instructions](https://github.com/open-mmlab/mmhuman3d/blob/main/docs/install.md). - -Important: Make sure that your compilation CUDA version and runtime CUDA version match. - -##### c. Install XRPrimer following the [official instructions](https://gitlab.bj.sensetime.com/openxrlab/xrprimer/-/blob/xrprimer_ee_dev/docs/python/install.md). - -##### d. Install XRMoCap to virtual environment, in editable mode. - -```shell -pip install -r requirements/build.txt -pip install -r requirements/runtime.txt -pip install -e . -``` - ## A from-scratch setup script ```shell @@ -75,7 +49,11 @@ conda install -y pytorch3d -c pytorch3d pip install mmcv-full==1.5.3 -f https://download.openmmlab.com/mmcv/dist/cu101/torch1.8.1/index.html # install xrprimer -pip install xrprimer -i https://repo.sensetime.com/repository/pypi/simple +pip install xrprimer + +# clone xrmocap +git clone https://github.com/openxrlab/xrmocap.git +cd xrmocap # install requirements for build pip install -r requirements/build.txt @@ -86,6 +64,64 @@ pip install -r requirements/runtime.txt rm -rf .eggs && pip install -e . ``` +## Prepare environment + +Here are advanced instructions for environment setup. If you have run [A from-scratch setup script](#a-from-scratch-setup-script) successfully, please skip this. + +#### a. Create a conda virtual environment and activate it. + +```shell +conda create -n xrmocap python=3.8 -y +conda activate xrmocap +``` + +#### b. Install MMHuman3D. + +Here we take `torch_version=1.8.1` and `cu_version=10.2` as example. For other versions, please follow the [official instructions](https://github.com/open-mmlab/mmhuman3d/blob/main/docs/install.md) + +```shell +# install ffmpeg from main channel +conda install ffmpeg +# install pytorch +conda install -y pytorch==1.8.1 torchvision==0.9.1 cudatoolkit=10.2 -c pytorch +# install pytorch3d +conda install -c fvcore -c iopath -c conda-forge fvcore iopath -y +conda install -c bottler nvidiacub -y +conda install pytorch3d -c pytorch3d +# install mmcv-full for human_perception +pip install mmcv-full==1.5.3 -f https://download.openmmlab.com/mmcv/dist/cu102/torch1.8.1/index.html +# install mmhuman3d +pip install git+https://github.com/open-mmlab/mmhuman3d.git +``` + +**Note1:** Make sure that your compilation CUDA version and runtime CUDA version match. + +**Note2:** The package `mmcv-full(gpu)` is essential if you are going to use `human_perception` modules. + +**Note3:** Do not install optional requirements of mmhuman3d in this step. + +#### c. Install XRPrimer. + +```shell +pip install xrprimer +``` + +If you want to edit xrprimer, please follow the [official instructions](https://github.com/openxrlab/xrprimer/) to install it from source. + +#### d. Install XRMoCap to virtual environment, in editable mode. + +```shell +git clone https://github.com/openxrlab/xrmocap.git +cd xrmocap +pip install -r requirements/build.txt +pip install -r requirements/runtime.txt +pip install -e . +``` + +#### e. Run unittests or demos + +If everything goes well, try to [run unittest](#test-environment) or go back to [run demos](./getting_started.md#inference) + ### Run with Docker Image We provide a [Dockerfile](../../Dockerfile) to build an image. Ensure that you are using [docker version](https://docs.docker.com/engine/install/) >=19.03 and `"default-runtime": "nvidia"` in daemon.json. @@ -100,3 +136,18 @@ Run it with ```shell docker run --gpus all --shm-size=8g -it -v {DATA_DIR}:/xrmocap/data xrmocap ``` + +Or pull a built image from docker hub. + +```shell +docker pull openxrlab/xrmocap_runtime +docker run --gpus all --shm-size=8g -it -v {DATA_DIR}:/xrmocap/data openxrlab/xrmocap_runtime +``` + +### Test environment + +To test whether the environment is well installed, please refer to [test doc](./test.md). + +### Frequently Asked Questions + +If your environment fails, check our [FAQ](./faq.md) first, it might be helpful to some typical questions. diff --git a/docs/en/model/smplify.md b/docs/en/model/smplify.md index 969cf8c0..5fd8c393 100644 --- a/docs/en/model/smplify.md +++ b/docs/en/model/smplify.md @@ -1,6 +1,11 @@ # SMPLify -[TOC] +- [Overview](https://github.com/open-mmlab/mmhuman3d/blob/main/docs/install.md#requirements) +- [Relationships between classes](#relationships-between-classes) +- [Build a registrant](#build-a-registrant) +- [Prepare the input and run](#prepare-the-input-and-run) +- [Develop a new loss](#develop-a-new-loss) +- [How to write a config file](#how-to-write-a-config-file) ### Overview @@ -15,7 +20,7 @@ - loss module: Sub-classes of `torch.nn.Module`. It has reduction, loss_weight and a forward method. -![SMPLify-classes.](http://assets.processon.com/chart_image/62a696db07912939b2288244.png) +![SMPLify-classes.](https://openxrlab-share.oss-cn-hongkong.aliyuncs.com/xrmocap/github_resources/SMPLify_classes.png) ### Build a registrant diff --git a/docs/en/ops/triangulation.md b/docs/en/ops/triangulation.md index 51474ed9..2ed64a01 100644 --- a/docs/en/ops/triangulation.md +++ b/docs/en/ops/triangulation.md @@ -1,15 +1,15 @@ # Triangulation - Triangulation - - [Prepare camera parameters](https://gitlab.bj.sensetime.com/openxrlab/xrprimer/-/blob/xrprimer_ee_dev/docs/en/python/triangulation.md#prepare-camera-parameters) - - [Build a triangulator](https://gitlab.bj.sensetime.com/openxrlab/xrprimer/-/blob/xrprimer_ee_dev/docs/en/python/triangulation.md#build-a-triangulator) + - [Prepare camera parameters](https://github.com/openxrlab/xrprimer/blob/main/docs/en/ops/triangulator.md#prepare-camera-parameters) + - [Build a triangulator](https://github.com/openxrlab/xrprimer/blob/main/docs/en/ops/triangulator.md#build-a-triangulator) - [Triangulate points from 2D to 3D](#triangulate-points-from-2d-to-3d) - - [Get reprojection error](https://gitlab.bj.sensetime.com/openxrlab/xrprimer/-/blob/xrprimer_ee_dev/docs/en/python/triangulation.md#get-reprojection-error) - - [Camera selection](https://gitlab.bj.sensetime.com/openxrlab/xrprimer/-/blob/xrprimer_ee_dev/docs/en/python/triangulation.md#camera-selection) + - [Get reprojection error](https://github.com/openxrlab/xrprimer/blob/main/docs/en/ops/triangulator.md#get-reprojection-error) + - [Camera selection](https://github.com/openxrlab/xrprimer/blob/main/docs/en/ops/triangulator.md#camera-selection) ### Overview -Triangulators in XRMoCap are sub-classes of XRPrimer triangulator. For basic usage of triangulators, please refer to [xrprimer doc](https://gitlab.bj.sensetime.com/openxrlab/xrprimer/-/blob/xrprimer_ee_dev/docs/en/python/triangulation.md#triangulate-points-from-2d-to-3d). +Triangulators in XRMoCap are sub-classes of XRPrimer triangulator. For basic usage of triangulators, please refer to [xrprimer doc](https://github.com/openxrlab/xrprimer/blob/main/docs/en/ops/triangulator.md#triangulate-points-from-2d-to-3d). ## Triangulate points from 2D to 3D diff --git a/docs/en/test.md b/docs/en/test.md index de51cd25..17a174b3 100644 --- a/docs/en/test.md +++ b/docs/en/test.md @@ -1,16 +1,21 @@ # Running Tests -- [Running Tests](#running-tests) - - [Data Preparation](#data-preparation) - - [Environment Preparation](#environment-preparation) - - [Running tests through pytest](#running-tests-through-pytest) +- [Data Preparation](#data-preparation) +- [Environment Preparation](#environment-preparation) +- [Running tests through pytest](#running-tests-through-pytest) ## Data Preparation Download data from the file server, and extract files to `tests/data`. ``` -sh script/download_test_data.sh +sh scripts/download_test_data.sh +``` + +Download weights from Internet, and extract files to `weight`. + +``` +sh scripts/download_weight.sh ``` ## Environment Preparation @@ -18,7 +23,7 @@ sh script/download_test_data.sh Install packages for test. ``` -pip install -r requirements/tests.txt +pip install -r requirements/test.txt ``` ## Running tests through pytest @@ -26,13 +31,20 @@ pip install -r requirements/tests.txt Running all the tests below `test/`. It is a good way to validate whether `XRMoCap` has been correctly installed: ``` -pytest test/ +pytest tests/ ``` -Generate a coverage for the test: +Or generate a coverage when testing: ``` -coverage run --source xrmocap -m pytest test/ +coverage run --source xrmocap -m pytest tests/ coverage xml coverage report -m ``` + +Or starts a CPU-only test on a GPU machine: + +``` +export CUDA_VISIBLE_DEVICES=-1 +pytest tests/ +``` diff --git a/docs/en/tool/train_model.md b/docs/en/tool/train_model.md deleted file mode 100644 index 10633380..00000000 --- a/docs/en/tool/train_model.md +++ /dev/null @@ -1,3 +0,0 @@ -# Learning-based model training - -@wanqi diff --git a/docs/en/tool/val_model.md b/docs/en/tool/val_model.md deleted file mode 100644 index 803287af..00000000 --- a/docs/en/tool/val_model.md +++ /dev/null @@ -1,3 +0,0 @@ -# Learning-based model validation - -@wanqi diff --git a/docs/en/tools/eval_model.md b/docs/en/tools/eval_model.md new file mode 100644 index 00000000..330a6daf --- /dev/null +++ b/docs/en/tools/eval_model.md @@ -0,0 +1,93 @@ +# Learning-based model evaluation + +- [Overview](#overview) +- [Preparation](#preparation) +- [Example](#example) + +## Overview + +This tool takes a config file and MvP model checkpoints and performs evaluation on Shelf, Campus or CMU Panoptic dataset. + +## Preparation + +1. Install `Deformable` package (Skip if you have done this step during model training) + +Download the [`./ops`](https://github.com/sail-sg/mvp/tree/main/lib/models/ops) folder, rename and place the folder as `ROOT/xrmocap/model/deformable`. Install `Deformable` by running: +``` +cd ROOT/xrmocap/model/deformable/ +sh make.sh +``` + +2. Prepare Datasets + +Follow the [dataset tool](./prepare_dataset.md) tutorial to prepare the train and test data. Some pre-processed datasets are available for download [here](../dataset_preparation.md). Place the `trainset_pesudo_gt` and `testset` data including meta data under `ROOT/xrmocap_data`. + + +3. Prepare pre-trained model weights and model checkpoints + +Download pre-trained backbone weights or MvP model checkpoints from [here](../../../configs/mvp/README.md). Place the model weights under `ROOT/weight`. + +4. Prepare config files + +Modify the config files in `ROOT/configs/mvp` if needed. Make sure the directories in config files match the directories and file names for your datasets and pre-traind model weights. + +The final file structure ready for evaluation would be like: + +```text +xrmocap +├── xrmocap +├── tools +├── configs +└── weight + ├── xrmocap_mvp_campus.pth.tar + ├── xrmocap_mvp_shelf.pth.tar + ├── xrmocap_mvp_panoptic_5view.pth.tar + ├── xrmocap_mvp_panoptic_3view_3_12_23.pth.tar + └── xrmocap_pose_resnet50_panoptic.pth.tar +└── xrmocap_data + └── meta + └── shelf + └── xrmocap_meta_testset + ├── campus + └── panoptic + ├── Shelf + ├── CampusSeq1 + └── panoptic + ├── 160906_band4 + ├── 160906_ian5 + ├── ... + └── 160906_pizza1 + +``` + +### Example + +Start evaluation with 8 GPUs with provided config file and pre-trained weights for Shelf dataset: + +```shell +python -m torch.distributed.launch \ + --nproc_per_node=8 \ + --use_env tools/eval_model.py \ + --cfg configs/mvp/shelf_config/mvp_shelf.py \ + --model_path weight/xrmocap_mvp_shelf.pth.tar +``` + +Alternatively, you can also run the script directly: + +```shell +sh ROOT/scripts/val_mvp.sh ${NUM_GPUS} ${CFG_FILE} ${MODEL_PATH} +``` + +Example: +```shell +sh ROOT/scripts/val_mvp.sh 8 configs/mvp/shelf_config/mvp_shelf.py weight/xrmocap_mvp_shelf.pth.tar +``` + +If you can run XRMoCap on a cluster managed with [slurm](https://slurm.schedmd.com/), you can use the script: +```shell +sh ROOT/scripts/slurm_eval_mvp.sh ${PARTITION} ${NUM_GPUS} ${CFG_FILE} ${MODEL_PATH} +``` +Example: +```shell +sh ROOT/scripts/slurm_eval_mvp.sh MyPartition 8 configs/mvp/shelf_config/mvp_shelf.py weight/xrmocap_mvp_shelf.pth.tar +``` diff --git a/docs/en/tool/mview_mperson_evaluation.md b/docs/en/tools/mview_mperson_evaluation.md similarity index 68% rename from docs/en/tool/mview_mperson_evaluation.md rename to docs/en/tools/mview_mperson_evaluation.md index ac7b2912..f6139d15 100644 --- a/docs/en/tool/mview_mperson_evaluation.md +++ b/docs/en/tools/mview_mperson_evaluation.md @@ -1,6 +1,8 @@ # Multi-view Multi-person Evaluation -[TOC] +- [Overview](#overview) +- [Argument](#argument) +- [Example](#example) ## Overview @@ -8,6 +10,9 @@ This tool takes calibrated camera parameters, RGB sequences, 2d perception data ## Argument +- **enable_log_file** +By default, enable_log_file is False and the tool will only print log to console. Add `--enable_log_file` makes it True and a log file named `{smc_file_name}_{time_str}.txt` will be written. + - **evaluation_config**: `evaluation_config` is the path to a `TopDownAssociationEvaluation` config file. For more details, see docs for `TopDownAssociationEvaluation` and the docstring in [code](../../../xrmocap/core/evaluation/top_down_association_evaluation.py). @@ -15,16 +20,18 @@ Also, you can find our prepared config files at `configs/mvpose/*/eval_keypoints ## Example -Run the tool without tracking. +Evaluate on the Shelf dataset and run the tool without tracking. ```bash python tools/mview_mperson_evaluation.py \ + --enable_log_file \ --evaluation_config configs/mvpose/shelf_config/eval_keypoints3d.py ``` -Run the tool with tracking. +Evaluate on the Shelf dataset and run the tool with tracking. ```bash python tools/mview_mperson_evaluation.py \ + --enable_log_file \ --evaluation_config configs/mvpose_tracking/shelf_config/eval_keypoints3d.py ``` diff --git a/docs/en/tool/mview_mperson_smplify3d.md b/docs/en/tools/mview_mperson_smplify3d.md similarity index 54% rename from docs/en/tool/mview_mperson_smplify3d.md rename to docs/en/tools/mview_mperson_smplify3d.md index ff7cc1e9..5b0f6881 100644 --- a/docs/en/tool/mview_mperson_smplify3d.md +++ b/docs/en/tools/mview_mperson_smplify3d.md @@ -1,6 +1,8 @@ # Multi-view Multi-person SMPLify3D -[TOC] +- [Overview](#overview) +- [Argument](#argument) +- [Example](#example) ## Overview @@ -8,8 +10,8 @@ This tool could generate multi-view multi-person SMPLData from keypoints3d. ## Argument -- **image_dir**: -`image_dir` is the path to the directory containing RGB sequence. If you have five views of the RGB sequences, there should be five folders in `image_dir`. +- **estimator_config**: +`estimator_config` is the path to a `MultiViewMultiPersonSMPLEstimator` config file. For more details, see docs for `MultiViewMultiPersonSMPLEstimator` and the docstring in [code](../../../xrmocap/core/estimation/mview_mperson_smpl_estimator.py). - **start_frame**: `start_frame` is the index of the start frame. @@ -23,39 +25,39 @@ This tool could generate multi-view multi-person SMPLData from keypoints3d. - **keypoints3d_path**: `keypoints3d_path` is the path to the keypoints3d file. -- **fisheye_param_dir**: -`fisheye_param_dir` is the path to the directory containing camera parameter. +- **matched_kps2d_idx**: +`matched_kps2d_idx` is the matched keypoints2d index from different views, where is generated by [code](../../../tools/mview_mperson_evaluation.py). + +- **image_and_camera_param**: +`image_and_camera_param` is a text file contains the image path and the corresponding camera parameters. Line 0 is the image path of the first view, and line 1 is the corresponding camera parameter path. Line 2 is the image path of the second view, and line 3 is the corresponding camera parameter path, and so on. - **perception2d_path**: `perception2d_path` is the path to the 2d perception data. -- **matched_list_path**: -`matched_list_path` is the path to the matched keypoints2d index from different views. - - **output_dir**: `output_dir` is the path to the directory saving all possible output files, including SMPLData and visualization videos. -- **estimator_config**: -`estimator_config` is the path to a `MultiViewMultiPersonSMPLEstimator` config file. For more details, see docs for `MultiViewMultiPersonSMPLEstimator` and the docstring in [code](../../../xrmocap/core/estimation/mview_mperson_smpl_estimator.py). - - **visualize**: By default, visualize is False. Add `--visualize` makes it True and the tool will visualize SMPLData with an orbit camera, overlay SMPL meshes on one view. +- **enable_log_file** +By default, enable_log_file is False and the tool will only print log to console. Add `--enable_log_file` makes it True and a log file named `{smc_file_name}_{time_str}.txt` will be written. + ## Example Run the tool with visualization. ```bash -python tool/mview_mperson_smplify3d.py \ - --image_dir 'xrmocap_data/Shelf' \ +python tools/mview_mperson_smplify3d.py \ + --estimator_config 'configs/modules/core/estimation/mview_mperson_smpl_estimator.py' \ --start_frame 300 \ --end_frame 600 \ - --keypoints3d_path 'output/Shelf/scene0_pred_keypoints3d.npz' \ - --fisheye_param_dir 'xrmocap_data/Shelf/xrmocap_meta_test/scene_0/camera_parameters' \ + --keypoints3d_path 'output/mvpose_tracking/shelf/scene0_pred_keypoints3d.npz' \ + --matched_kps2d_idx 'output/mvpose_tracking/shelf/scene0_matched_kps2d_idx.npy' \ + --image_and_camera_param 'xrmocap_data/Shelf/image_and_camera_param.txt' \ --perception2d_path 'xrmocap_data/Shelf/xrmocap_meta_test/scene_0/perception_2d.npz' \ - --matched_list_path 'output/Shelf/scene0_matched_kps2d_idx.npy' \ - --output_dir 'output/Shelf' \ - --estimator_config 'configs/modules/core/estimation/mview_mperson_smpl_estimator.py' \ - --visualize + --output_dir 'output/mvpose_tracking/shelf/smpl' \ + --visualize \ + --enable_log_file ``` diff --git a/docs/en/tool/prepare_dataset.md b/docs/en/tools/prepare_dataset.md similarity index 91% rename from docs/en/tool/prepare_dataset.md rename to docs/en/tools/prepare_dataset.md index 1a7cdf6b..e5aed8bf 100644 --- a/docs/en/tool/prepare_dataset.md +++ b/docs/en/tools/prepare_dataset.md @@ -1,6 +1,11 @@ # Tool prepare_dataset -[TOC] +- [Overview](#overview) +- [Argument: converter_config](#argument-converter_config) +- [Argument: overwrite](#argument-overwrite) +- [Argument: disable_log_file](#argument-disable_log_file) +- [Argument: paths](#argument-paths) +- [Example](#example) ### Overview diff --git a/docs/en/tool/process_smc.md b/docs/en/tools/process_smc.md similarity index 79% rename from docs/en/tool/process_smc.md rename to docs/en/tools/process_smc.md index 816f94d7..40263d28 100644 --- a/docs/en/tool/process_smc.md +++ b/docs/en/tools/process_smc.md @@ -1,6 +1,11 @@ # Tool process_smc -[TOC] +- [Overview](#overview) +- [Argument: estimator_config](#argument-estimator_config) +- [Argument: output_dir](#argument-output_dir) +- [Argument: disable_log_file](#argument-disable_log_file) +- [Argument: visualize](#argument-visualize) +- [Example](#example) ### Overview @@ -29,8 +34,8 @@ By default, visualize is False. Add `--visualize` makes it True and the tool wil Run the tool with visualization. ```bash -python tool/process_smc.py \ - --estimator_config config/estimation/mview_sperson_smpl_estimator.py \ +python tools/process_smc.py \ + --estimator_cofig configs/humman_mocap/mview_sperson_smpl_estimator.py \ --smc_path xrmocap_data/humman/raw_smc/p000105_a000195.smc \ --output_dir xrmocap_data/humman/p000105_a000195_output \ --visualize diff --git a/docs/en/tools/train_model.md b/docs/en/tools/train_model.md new file mode 100644 index 00000000..e8c8b0e4 --- /dev/null +++ b/docs/en/tools/train_model.md @@ -0,0 +1,92 @@ +# Learning-based model training + +- [Overview](#overview) +- [Preparation](#preparation) +- [Example](#example) + +## Overview + +This tool takes a config file and starts trainig MvP model with Shelf, Campus or CMU Panoptic dataset. + +## Preparation + +1. Install `Deformable` package (Skip if you have done this step during model evaluation) + +Download the [`./ops`](https://github.com/sail-sg/mvp/tree/main/lib/models/ops) folder, rename and place the folder as `ROOT/xrmocap/model/deformable`. Install `Deformable` by running: +``` +cd ROOT/xrmocap/model/deformable/ +sh make.sh +``` + +2. Prepare Datasets + +Follow the [dataset tool](./prepare_dataset.md) tutorial to prepare the train and test data. Some pre-processed datasets are available for download [here](../dataset_preparation.md). Place the `trainset_pesudo_gt` and `testset` data including meta data under `ROOT/xrmocap_data`. + + +3. Prepare pre-trained model weights and model checkpoints + +Download pre-trained backbone weights or MvP model checkpoints from [here](../../../configs/mvp/README.md). Place the model weights under `ROOT/weight`. + +4. Prepare config files + +Modify the config files in `ROOT/configs/mvp` if needed. Make sure the directories in config files match the directories and file names for your datasets and pre-traind weights. + +The final file structure ready for training would be like: + +```text +xrmocap +├── xrmocap +├── tools +├── configs +└── weight + ├── xrmocap_mvp_campus.pth.tar + ├── xrmocap_mvp_shelf.pth.tar + ├── xrmocap_mvp_panoptic_5view.pth.tar + ├── xrmocap_mvp_panoptic_3view_3_12_23.pth.tar + └── xrmocap_pose_resnet50_panoptic.pth.tar +└── xrmocap_data + └── meta + └── shelf + ├── xrmocap_meta_testset + └── xrmocap_meta_trainset_pesudo_gt + ├── campus + └── panoptic + ├── Shelf + ├── CampusSeq1 + └── panoptic + ├── 160906_band4 + ├── 160906_ian5 + ├── ... + └── 160906_pizza1 + +``` + +## Example + +Start training with 8 GPUs with provided config file for Campus dataset: + +```bash +python -m torch.distributed.launch \ + --nproc_per_node= 8 \ + --use_env tools/train_model.py \ + --cfg configs/mvp/campus_config/mvp_campus.py \ +``` + +Alternatively, you can also run the script directly: + +``` +sh ROOT/scripts/train_mvp.sh ${NUM_GPUS} ${CFG_FILE} +``` +Example: + +``` +sh ROOT/scripts/train_mvp.sh 8 configs/mvp/campus_config/mvp_campus.py +``` +If you can run XRMoCap on a cluster managed with [slurm](https://slurm.schedmd.com/), you can use the script: +```shell +sh ROOT/scripts/slurm_train_mvp.sh ${PARTITION} ${NUM_GPUS} ${CFG_FILE} +``` +Example: +```shell +sh ROOT/scripts/slurm_train_mvp.sh MyPartition 8 configs/mvp/shelf_config/mvp_shelf.py +``` diff --git a/docs/en/tool/visualize_dataset.md b/docs/en/tools/visualize_dataset.md similarity index 91% rename from docs/en/tool/visualize_dataset.md rename to docs/en/tools/visualize_dataset.md index 3ab4a6e3..81f8ad9d 100644 --- a/docs/en/tool/visualize_dataset.md +++ b/docs/en/tools/visualize_dataset.md @@ -1,6 +1,11 @@ # Tool visualize_dataset -[TOC] +- [Overview](#overview) +- [Argument: vis_config](#argument-vis_config) +- [Argument: overwrite](#argument-overwrite) +- [Argument: disable_log_file](#argument-disable_log_file) +- [Argument: paths](#argument-paths) +- [Example](#example) ### Overview diff --git a/docs/en/tutorials/config.md b/docs/en/tutorials/config.md index 704f9428..ab29f5f6 100644 --- a/docs/en/tutorials/config.md +++ b/docs/en/tutorials/config.md @@ -5,8 +5,17 @@ We incorporate modular and inheritance design into our config system, which is c ## Modify config through script arguments Take MVPose and MVPose tracking as an example + +If you want to use tracker, you need to create a variable of dictionary type containing `type='KalmanTracking'` and others needed in `__init__()`. Then you need to build it and will get a Kalman tracking module, otherwise you just need to set `kalman_tracking_config=None`. + +Example: ``` -# add config content +kalman_tracking_config=dict(type='KalmanTracking', n_cam_min=2, logger=logger) + +if isinstance(kalman_tracking_config, dict): + kalman_tracking = build_kalman_tracking(kalman_tracking_config) +else: + kalman_tracking = kalman_tracking_config ``` Using trackers diff --git a/docs/en/tutorials/new_dataset.md b/docs/en/tutorials/new_dataset.md index 48b4c614..3ef425f4 100644 --- a/docs/en/tutorials/new_dataset.md +++ b/docs/en/tutorials/new_dataset.md @@ -12,7 +12,7 @@ For online conversion, program does not write any file to disk. You have to defi For offline conversion, we convert the origin dataset into annotations in a unified format, save the annotations to disk, before training or evaluation starts. Such a conversion module is called `data_converter` in XRMoCap, and you can find examples in [xrmocap/data/data_converter](../../../xrmocap/data/data_converter). -##### File tree of our unified format +#### File tree of our unified format ``` Dataset_xxx @@ -37,19 +37,19 @@ Dataset_xxx └── ... ``` -##### Camera parameters of our unified format +#### Camera parameters of our unified format -Each scene has its independent multi-view camera parameters, and each json file is dumped by `class FisheyeCameraParameter` in [XRPrimer](https://gitlab.bj.sensetime.com/openxrlab/xrprimer/-/blob/xrprimer_ee_dev/python/xrprimer/data_structure/camera/fisheye_camera.py). +Each scene has its independent multi-view camera parameters, and each json file is dumped by `class FisheyeCameraParameter` in [XRPrimer](https://github.com/openxrlab/xrprimer/blob/main/docs/en/data_structure/camera.md#fisheye). -##### Image list of our unified format +#### Image list of our unified format In a scene whose frame length is `n_frame`, number of cameras is `n_view`, there are `n_view` image list files, and every file has `n_frame` lines inside, take the `frame_idx`-th line in file `image_list_view_{view_idx}.txt`, we get a path of image relative to dataset_root(Dataset_xxx). -##### Keypoints3d groundtruth of our unified format +#### Keypoints3d groundtruth of our unified format `keypoints3d_GT.npz` is a file dumped by `class Keypoints`, and it can be load by `keypoints3d = Keypoints.fromfile()`. In a scene whose frame length is `n_frame`, max number of objects is `n_person`, number of single person's keypoints is `n_kps`, `keypoints3d.get_keypoints()` returns an ndarray in shape [n_frame, n_person, n_kps, 4], and `keypoints3d.get_mask()` is an ndarray in shape [n_frame, n_person, n_kps] which indicates which person and which keypoint is valid at a certain frame. -##### Perception 2D of our unified format +#### Perception 2D of our unified format `perception_2d.npz` is an compressed npz file of a python dict, whose structure lies below: diff --git a/docs/en/tutorials/new_module.md b/docs/en/tutorials/new_module.md index e441e821..46e8efd3 100644 --- a/docs/en/tutorials/new_module.md +++ b/docs/en/tutorials/new_module.md @@ -1,10 +1,77 @@ # Add new module -If you want to add a new module for existing one, you can write a class and registry it. +If you want to add a new module, write a class and register it in builder. Here we take triangulator as example. -Take Triangulator as an example, +### Develop PytorchTriangulator class +1. Inherit from base class ---- +Inherit from `BaseTriangulator` and assign correct values for class attributes. -Add a new method, see whether existing modules are applicable (or write in a new md?) +```python +class PytorchTriangulator(BaseTriangulator): + CAMERA_CONVENTION = 'opencv' + CAMERA_WORLD2CAM = True +``` + +Complete `__init__` and do not forget to add arguments of super-class. + +```python + def __init__(self, + camera_parameters: List[FisheyeCameraParameter], + logger: Union[None, str, logging.Logger] = None) -> None: + self.logger = get_logger(logger) + super().__init__(camera_parameters=camera_parameters, logger=logger) + +``` + +2. Complete necessary methods defined by base class + +```python + def triangulate( + self, + points: Union[torch.Tensor, list, tuple], + points_mask: Union[torch.Tensor, list, tuple] = None) -> np.ndarray: + + def get_reprojection_error( + self, + points2d: torch.Tensor, + points3d: torch.Tensor, + points_mask: torch.Tensor = None, + reduction: Literal['mean', 'sum', 'none'] = 'none' + ) -> Union[torch.Tensor, float]: + + def get_projector(self) -> PytorchProjector: + +``` + +3. Add special methods of this class(Optional) + +```python + def get_device( + self) -> torch.device: + +``` + +4. Register the class in builder + +Insert the following lines into `xrmocap/ops/triangulation/builder.py`. + +```python +from .pytorch_triangulator import PytorchTriangulator + +TRIANGULATORS.register_module( + name='PytorchTriangulator', module=PytorchTriangulator) + +``` + +### Build and use + +Test whether the new module is OK to build. + +```python +from xrmocap.ops.triangulation.builder import build_triangulator + +triangulator = build_triangulator(dict(type='PytorchTriangulator')) + +``` diff --git a/scripts/download_test_data.sh b/scripts/download_test_data.sh index 1f79eaa8..335b1f3d 100644 --- a/scripts/download_test_data.sh +++ b/scripts/download_test_data.sh @@ -1,8 +1,8 @@ mkdir xrmocap_download cd xrmocap_download -wget -q http://10.4.11.59:18080/resources/XRlab/xrmocap/tests.tar.gz +wget -q https://openxrlab-share.oss-cn-hongkong.aliyuncs.com/xrmocap/tests.tar.gz tar -zxvf tests.tar.gz -wget -q http://10.4.11.59:18080/resources/XRlab/xrmocap/xrmocap_data.tar.gz +wget -q https://openxrlab-share.oss-cn-hongkong.aliyuncs.com/xrmocap/xrmocap_data.tar.gz tar -zxvf xrmocap_data.tar.gz cd .. cp -r xrmocap_download/tests ./ diff --git a/scripts/download_weight.sh b/scripts/download_weight.sh index 0a28dd31..a4bc4875 100644 --- a/scripts/download_weight.sh +++ b/scripts/download_weight.sh @@ -1,7 +1,7 @@ -mkdir xrmocap_download -cd xrmocap_download -wget -q http://10.4.11.59:18080/resources/XRlab/xrmocap/weight.tar.gz -tar -zxvf weight.tar.gz -cd .. -cp -r xrmocap_download/weight ./ -rm -rf xrmocap_download +mkdir weight +cd weight +wget https://download.openmmlab.com/mmdetection/v2.0/faster_rcnn/faster_rcnn_r50_fpn_1x_coco/faster_rcnn_r50_fpn_1x_coco_20200130-047c8118.pth +wget https://download.openmmlab.com/mmpose/top_down/hrnet/hrnet_w48_coco_wholebody_384x288_dark-f5726563_20200918.pth +wget https://download.openmmlab.com/mmtracking/mot/faster_rcnn/faster-rcnn_r50_fpn_4e_mot17-half-64ee2ed4.pth +wget https://download.openmmlab.com/mmtracking/mot/reid/tracktor_reid_r50_iter25245-a452f51f.pth +wget https://openxrlab-share.oss-cn-hongkong.aliyuncs.com/xrmocap/weight/resnet50_reid_camstyle.pth.tar diff --git a/scripts/eval_mvp.sh b/scripts/eval_mvp.sh index 2f29f147..740c2a30 100644 --- a/scripts/eval_mvp.sh +++ b/scripts/eval_mvp.sh @@ -2,30 +2,25 @@ set -x -CFG_FILE="configs/mvp/shelf_config/mvp_shelf.py" +# CFG_FILE="configs/mvp/shelf_config/mvp_shelf.py" # CFG_FILE="configs/mvp/campus_config/mvp_campus.py" # CFG_FILE="configs/mvp/panoptic_config/mvp_panoptic.py" # CFG_FILE="configs/mvp/panoptic_config/mvp_panoptic_3cam.py" -# Author provided/ trained with original github code -# MODEL_PATH="weight/convert_mvp_shelf.pth.tar" # 97.2 -# MODEL_PATH="weight/convert_mvp_shelf_self.pth.tar" # 97.1 -# MODEL_PATH="weight/convert_mvp_campus.pth.tar" #90.8 -# MODEL_PATH="weight/convert_model_best_5view.pth.tar" # ap25:92.3 - # Trained with xrmocap from scratch -MODEL_PATH="weight/convert_mvp_shelf_self_wo_band3_8gpu.pth.tar" #97.1 -# MODEL_PATH="weight/convert_mvp_campus_self_wo_band3_31223_2gpu.pth.tar" #96.79 -# MODEL_PATH="weight/convert_model_best_5view_self_wo_band3.pth.tar" #ap25:90.7 -# MODEL_PATH="weight/convert_model_best_31223.pth.tar" #ap25:53.54 +# MODEL_PATH="weight/xrmocap_mvp_shelf.pth.tar" +# MODEL_PATH="weight/xrmocap_mvp_campus.pth.tar" +# MODEL_PATH="weight/xrmocap_mvp_panoptic_5view.pth.tar" +# MODEL_PATH="weight/xrmocap_mvp_panoptic_3view_3_12_23.pth.tar" GPUS_PER_NODE=$1 +CFG_FILE=$2 +MODEL_PATH=$3 python -m torch.distributed.launch \ --nproc_per_node=${GPUS_PER_NODE} \ - --master_port 65524 \ - --use_env tool/val_model.py \ + --use_env tools/eval_model.py \ --cfg ${CFG_FILE} \ - --model_path ${MODEL_PATH} | tee ./output/log.txt + --model_path ${MODEL_PATH} diff --git a/scripts/slurm_eval_mvp.sh b/scripts/slurm_eval_mvp.sh index 33805791..c2179170 100644 --- a/scripts/slurm_eval_mvp.sh +++ b/scripts/slurm_eval_mvp.sh @@ -2,37 +2,23 @@ set -x -CFG_FILE="configs/mvp/shelf_config/mvp_shelf.py" +# CFG_FILE="configs/mvp/shelf_config/mvp_shelf.py" # CFG_FILE="configs/mvp/campus_config/mvp_campus.py" # CFG_FILE="configs/mvp/panoptic_config/mvp_panoptic.py" # CFG_FILE="configs/mvp/panoptic_config/mvp_panoptic_3cam.py" -# Author provided/ trained with original github code -# MODEL_PATH="weight/convert_mvp_shelf.pth.tar" # 97.2 -# MODEL_PATH="weight/convert_mvp_shelf_self.pth.tar" # 97.1 -# MODEL_PATH="weight/convert_mvp_campus.pth.tar" # 90.8 -# MODEL_PATH="weight/convert_model_best_5view.pth.tar" # ap25:92.3 - -# MODEL_PATH="weight/mvp_shelf.pth.tar" # 97.2 -# MODEL_PATH="weight/mvp_shelf_self.pth.tar" # 97.1 -# MODEL_PATH="weight/mvp_campus.pth.tar" # 90.8 -# MODEL_PATH="weight/model_best_5view.pth.tar" # ap25:92.3 - # Trained with xrmocap from scratch -MODEL_PATH="weight/convert_mvp_shelf_self_wo_band3_8gpu.pth.tar" # 97.1 -# MODEL_PATH="weight/convert_mvp_campus_self_wo_band3_31223_2gpu.pth.tar" # 96.79 -# MODEL_PATH="weight/convert_model_best_5view_self_wo_band3.pth.tar" # ap25:90.7 -# MODEL_PATH="weight/convert_model_best_31223.pth.tar" # ap25:53.54 - -# MODEL_PATH="weight/mvp_shelf_self_wo_band3_8gpu.pth.tar" # 97.1 -# MODEL_PATH="weight/mvp_campus_self_wo_band3_31223_2gpu.pth.tar" # 96.79 -# MODEL_PATH="weight/model_best_5view_self_wo_band3.pth.tar" # ap25:90.7 -# MODEL_PATH="weight/model_best_31223.pth.tar" # ap25:53.54 +# MODEL_PATH="weight/xrmocap_mvp_shelf.pth.tar" +# MODEL_PATH="weight/xrmocap_mvp_campus.pth.tar" +# MODEL_PATH="weight/xrmocap_mvp_panoptic_5view.pth.tar" +# MODEL_PATH="weight/xrmocap_mvp_panoptic_3view_3_12_23.pth.tar" PARTITION=$1 JOB_NAME=mvp_eval GPUS_PER_NODE=$2 +CFG_FILE=$3 +MODEL_PATH=$4 CPUS_PER_TASK=1 @@ -44,7 +30,6 @@ srun -p ${PARTITION} \ --kill-on-bad-exit=1 \ python -m torch.distributed.launch \ --nproc_per_node=${GPUS_PER_NODE} \ - --master_port 65523 \ - --use_env tool/val_model.py \ + --use_env tools/eval_model.py \ --cfg ${CFG_FILE} \ --model_path ${MODEL_PATH} diff --git a/scripts/slurm_train_mvp.sh b/scripts/slurm_train_mvp.sh index 3b918833..a79fe84f 100644 --- a/scripts/slurm_train_mvp.sh +++ b/scripts/slurm_train_mvp.sh @@ -2,7 +2,7 @@ set -x -CFG_FILE="configs/mvp/campus_config/mvp_campus.py" +# CFG_FILE="configs/mvp/campus_config/mvp_campus.py" # CFG_FILE="configs/mvp/shelf_config/mvp_shelf.py" # CFG_FILE="configs/mvp/panoptic_config/mvp_panoptic.py" # CFG_FILE="configs/mvp/panoptic_config/mvp_panoptic_3cam.py" @@ -10,6 +10,7 @@ CFG_FILE="configs/mvp/campus_config/mvp_campus.py" PARTITION=$1 JOB_NAME=mvp_train GPUS_PER_NODE=$2 +CFG_FILE=$3 CPUS_PER_TASK=1 srun -p ${PARTITION} \ @@ -20,6 +21,5 @@ srun -p ${PARTITION} \ --kill-on-bad-exit=1 \ python -m torch.distributed.launch \ --nproc_per_node=${GPUS_PER_NODE} \ - --master_port 65521 \ - --use_env tool/train_model.py \ + --use_env tools/train_model.py \ --cfg ${CFG_FILE} \ diff --git a/scripts/train_mvp.sh b/scripts/train_mvp.sh index a9a98c17..9b1ff6eb 100644 --- a/scripts/train_mvp.sh +++ b/scripts/train_mvp.sh @@ -2,15 +2,15 @@ set -x -CFG_FILE="configs/mvp/campus_config/mvp_campus.py" +# CFG_FILE="configs/mvp/campus_config/mvp_campus.py" # CFG_FILE="configs/mvp/shelf_config/mvp_shelf.py" # CFG_FILE="configs/mvp/panoptic_config/mvp_panoptic.py" # CFG_FILE="configs/mvp/panoptic_config/mvp_panoptic_3cam.py" GPUS_PER_NODE=$1 +CFG_FILE=$2 python -m torch.distributed.launch \ --nproc_per_node=${GPUS_PER_NODE} \ - --master_port 65520 \ - --use_env tool/train_model.py \ + --use_env tools/train_model.py \ --cfg ${CFG_FILE} \ diff --git a/setup.py b/setup.py index b0324bb2..bf51d116 100644 --- a/setup.py +++ b/setup.py @@ -99,7 +99,7 @@ def gen_packages_items(): author='', author_email='', keywords='', - url='https://gitlab.bj.sensetime.com/openxrlab/xrmocap', + url='https://github.com/openxrlab/xrmocap', packages=find_packages(exclude=('configs', 'tools', 'demo')), include_package_data=True, classifiers=[ diff --git a/tests/human_perception/test_bbox_detection.py b/tests/human_perception/test_bbox_detection.py index 2f999e8c..7eda458a 100644 --- a/tests/human_perception/test_bbox_detection.py +++ b/tests/human_perception/test_bbox_detection.py @@ -19,6 +19,7 @@ input_dir = 'tests/data/human_perception/test_bbox_detection' output_dir = 'tests/data/output/human_perception/test_bbox_detection' +device = 'cpu' if not torch.cuda.is_available() else 'cuda' @pytest.fixture(scope='module', autouse=True) @@ -29,7 +30,7 @@ def fixture(): smc_reader = SMCReader('tests/data/p000103_a000011_tiny.smc') single_image_array = smc_reader.get_kinect_color(kinect_id=0, frame_id=0) single_image_array = bgr2rgb(single_image_array) - image_array = single_image_array.repeat(4, axis=0) + image_array = single_image_array.repeat(2, axis=0) image_dir = os.path.join(output_dir, 'rgb_frames') array_to_images( image_array=image_array, output_folder=image_dir, disable_log=True) @@ -42,17 +43,16 @@ def test_mmdet_detector_build(): detector_config = dict( mmcv.Config.fromfile('configs/modules/human_perception/' + 'mmdet_faster_rcnn_detector.py')) + detector_config['mmdet_kwargs']['device'] = device # test init _ = build_detector(detector_config) -@pytest.mark.skipif( - not torch.cuda.is_available(), reason='No GPU device has been found.') def test_mmdet_detector_infer(): detector_config = dict( mmcv.Config.fromfile('configs/modules/human_perception/' + 'mmdet_faster_rcnn_detector.py')) - detector_config['mmdet_kwargs']['device'] = 'cuda:0' + detector_config['mmdet_kwargs']['device'] = device # test init mmdet_detector = build_detector(detector_config) # test infer frames @@ -114,17 +114,16 @@ def test_mmtrack_detector_build(): detector_config = dict( mmcv.Config.fromfile('configs/modules/human_perception/' + 'mmtrack_faster_rcnn_detector.py')) + detector_config['mmtrack_kwargs']['device'] = device # test init _ = build_detector(detector_config) -@pytest.mark.skipif( - not torch.cuda.is_available(), reason='No GPU device has been found.') def test_mmtrack_detector_infer(): detector_config = dict( mmcv.Config.fromfile('configs/modules/human_perception/' + 'mmtrack_faster_rcnn_detector.py')) - detector_config['mmtrack_kwargs']['device'] = 'cuda:0' + detector_config['mmtrack_kwargs']['device'] = device # test init mmtrack_detector = build_detector(detector_config) # test infer frames diff --git a/tests/human_perception/test_top_down_pose_estimation.py b/tests/human_perception/test_top_down_pose_estimation.py index e7272040..4f96017c 100644 --- a/tests/human_perception/test_top_down_pose_estimation.py +++ b/tests/human_perception/test_top_down_pose_estimation.py @@ -33,8 +33,6 @@ def fixture(): image_array=image_array, output_path=video_path, disable_log=True) -@pytest.mark.skipif( - not torch.cuda.is_available(), reason='No GPU device has been found.') def test_mmdet_detector(): empty_bbox = [0.0, 0.0, 0.0, 0.0, 0.0] single_person_bbox = np.load( @@ -44,7 +42,8 @@ def test_mmdet_detector(): estimator_config = dict( mmcv.Config.fromfile( 'configs/modules/human_perception/mmpose_hrnet_estimator.py')) - estimator_config['mmpose_kwargs']['device'] = 'cuda:0' + device = 'cpu' if not torch.cuda.is_available() else 'cuda' + estimator_config['mmpose_kwargs']['device'] = device # test init mmpose_estimator = build_detector(estimator_config) # test convention diff --git a/tests/model/registrant/test_smplify.py b/tests/model/registrant/test_smplify.py index a248665e..44b96779 100644 --- a/tests/model/registrant/test_smplify.py +++ b/tests/model/registrant/test_smplify.py @@ -21,6 +21,7 @@ # yapf: enable input_dir = 'tests/data/model/registrant' output_dir = 'tests/data/output/model/registrant/test_smplify' +device = 'cpu' if not torch.cuda.is_available() else 'cuda' @pytest.fixture(scope='module', autouse=True) @@ -35,6 +36,7 @@ def test_build(): smplify_config = dict( mmcv.Config.fromfile('configs/modules/model/' + 'registrant/smplify_test.py')) + smplify_config['device'] = device smplify = build_registrant(smplify_config) assert smplify is not None # build with body_model @@ -59,11 +61,7 @@ def test_build(): assert smplify is not None -@pytest.mark.skipif( - not torch.cuda.is_available(), reason='No GPU device has been found.') def test_smplify_keypoints3d(): - device = torch.device( - 'cuda') if torch.cuda.is_available() else torch.device('cpu') keypoints3d_path = os.path.join(input_dir, 'human_data_tri.npz') human_data = dict(np.load(keypoints3d_path, allow_pickle=True)) keypoints3d, keypoints3d_mask = convert_kps_mm( @@ -80,6 +78,7 @@ def test_smplify_keypoints3d(): smplify_config = dict( mmcv.Config.fromfile('configs/modules/model/' + 'registrant/smplify_test.py')) + smplify_config['device'] = device smplify = build_registrant(smplify_config) kp3d_mse_input = Keypoint3dMSEInput( keypoints3d=keypoints3d, diff --git a/tests/model/registrant/test_smplifyx.py b/tests/model/registrant/test_smplifyx.py index 1a5966cc..0f8b074a 100644 --- a/tests/model/registrant/test_smplifyx.py +++ b/tests/model/registrant/test_smplifyx.py @@ -21,6 +21,7 @@ # yapf: enable input_dir = 'tests/data/model/registrant' output_dir = 'tests/data/output/model/registrant/test_smplifyx' +device = 'cpu' if not torch.cuda.is_available() else 'cuda' @pytest.fixture(scope='module', autouse=True) @@ -35,6 +36,7 @@ def test_build(): smplifyx_config = dict( mmcv.Config.fromfile( 'configs/modules/model/registrant/smplifyx_test.py')) + smplifyx_config['device'] = device smplifyx = build_registrant(smplifyx_config) assert smplifyx is not None # build with body_model @@ -59,11 +61,7 @@ def test_build(): assert smplifyx is not None -@pytest.mark.skipif( - not torch.cuda.is_available(), reason='No GPU device has been found.') def test_smplifyx_kps3d(): - device = torch.device( - 'cuda') if torch.cuda.is_available() else torch.device('cpu') kps3d_path = os.path.join(input_dir, 'human_data_tri.npz') human_data = dict(np.load(kps3d_path, allow_pickle=True)) kps3d, kps3d_mask = convert_kps_mm( @@ -78,6 +76,7 @@ def test_smplifyx_kps3d(): smplifyx_config = dict( mmcv.Config.fromfile('configs/modules/model/' + 'registrant/smplifyx_test.py')) + smplifyx_config['device'] = device smplifyx = build_registrant(smplifyx_config) kp3d_mse_input = Keypoint3dMSEInput( keypoints3d=kps3d, diff --git a/tests/model/registrant/test_smplifyxd.py b/tests/model/registrant/test_smplifyxd.py index f92844dc..17b4a959 100644 --- a/tests/model/registrant/test_smplifyxd.py +++ b/tests/model/registrant/test_smplifyxd.py @@ -21,6 +21,7 @@ # yapf: enable input_dir = 'tests/data/model/registrant' output_dir = 'tests/data/output/model/registrant/test_smplifyxd' +device = 'cpu' if not torch.cuda.is_available() else 'cuda' @pytest.fixture(scope='module', autouse=True) @@ -35,6 +36,7 @@ def test_build(): smplifyxd_config = dict( mmcv.Config.fromfile( 'configs/modules/model/registrant/smplifyxd_test.py')) + smplifyxd_config['device'] = device smplifyxd = build_registrant(smplifyxd_config) assert smplifyxd is not None # build with body_model @@ -59,11 +61,7 @@ def test_build(): assert smplifyxd is not None -@pytest.mark.skipif( - not torch.cuda.is_available(), reason='No GPU device has been found.') def test_smplifyxd_keypoints3d(): - device = torch.device( - 'cuda') if torch.cuda.is_available() else torch.device('cpu') keypoints3d_path = os.path.join(input_dir, 'human_data_tri.npz') human_data = dict(np.load(keypoints3d_path, allow_pickle=True)) keypoints3d, keypoints3d_mask = convert_kps_mm( @@ -80,6 +78,7 @@ def test_smplifyxd_keypoints3d(): smplifyxd_config = dict( mmcv.Config.fromfile('configs/modules/model/' + 'registrant/smplifyxd_test.py')) + smplifyxd_config['device'] = device smplifyxd = build_registrant(smplifyxd_config) kp3d_mse_input = Keypoint3dMSEInput( keypoints3d=keypoints3d, diff --git a/tests/ops/test_top_down_association.py b/tests/ops/test_top_down_association.py index 0871dc68..3b84242b 100644 --- a/tests/ops/test_top_down_association.py +++ b/tests/ops/test_top_down_association.py @@ -16,6 +16,8 @@ def fixture(): os.makedirs(output_dir, exist_ok=False) +@pytest.mark.skipif( + not torch.cuda.is_available(), reason='No GPU device has been found.') def test_build_mvpose_associator(): associator_cfg = dict( mmcv.Config.fromfile('configs/modules/ops/' + 'top_down_association/' + diff --git a/tools/val_model.py b/tools/eval_model.py similarity index 100% rename from tools/val_model.py rename to tools/eval_model.py diff --git a/tools/mview_mperson_evaluation.py b/tools/mview_mperson_evaluation.py index 49aa8117..fa4d6992 100644 --- a/tools/mview_mperson_evaluation.py +++ b/tools/mview_mperson_evaluation.py @@ -1,7 +1,9 @@ # yapf: disable import argparse +import datetime import mmcv import os +from xrprimer.utils.log_utils import setup_logger from xrmocap.core.evaluation.builder import build_evaluation @@ -9,8 +11,16 @@ def main(args): + os.makedirs('logs', exist_ok=True) + if args.enable_log_file: + time_str = datetime.datetime.now().strftime('%Y-%m-%d_%H:%M:%S') + log_path = os.path.join('logs', f'evaluation_log_{time_str}.txt') + logger = setup_logger(logger_name=__name__, logger_path=log_path) + else: + logger = setup_logger(logger_name=__name__) evaluation_config = dict(mmcv.Config.fromfile(args.evaluation_config)) os.makedirs(evaluation_config['output_dir'], exist_ok=True) + evaluation_config['logger'] = logger evaluation = build_evaluation(evaluation_config) evaluation.run(overwrite=True) @@ -18,10 +28,14 @@ def main(args): def setup_parser(): parser = argparse.ArgumentParser( description='Evaluate Top-down keypoints3d estimator.') + parser.add_argument( + '--enable_log_file', + action='store_true', + help='If checked, log will be written as file.', + default=False) parser.add_argument( '--evaluation_config', - default='./config/evaluation/mview_mperson_keypoints3d/shelf_config' + - '/eval_keypoints3d.py') + default='configs/mvpose_tracking/campus_config/eval_keypoints3d.py') args = parser.parse_args() return args diff --git a/tools/mview_mperson_smplify3d.py b/tools/mview_mperson_smplify3d.py index d8f18d71..d80af9a3 100644 --- a/tools/mview_mperson_smplify3d.py +++ b/tools/mview_mperson_smplify3d.py @@ -1,6 +1,7 @@ # yapf: disable import argparse import cv2 +import datetime import glob import mmcv import numpy as np @@ -8,8 +9,10 @@ from mmhuman3d.core.visualization.visualize_smpl import ( visualize_smpl_calibration, ) +from mmhuman3d.utils.demo_utils import get_different_colors from typing import List from xrprimer.data_structure.camera import FisheyeCameraParameter +from xrprimer.utils.log_utils import setup_logger from xrmocap.core.estimation.builder import build_estimator from xrmocap.data_structure.keypoints import Keypoints @@ -19,15 +22,30 @@ def main(args): + os.makedirs('logs', exist_ok=True) + if args.enable_log_file: + time_str = datetime.datetime.now().strftime('%Y-%m-%d_%H:%M:%S') + log_path = os.path.join('logs', f'estimation_log_{time_str}.txt') + logger = setup_logger(logger_name=__name__, logger_path=log_path) + else: + logger = setup_logger(logger_name=__name__) keypoints3d = Keypoints.fromfile(npz_path=args.keypoints3d_path) - fisheye_param_paths = sorted( - glob.glob(os.path.join(args.fisheye_param_dir, 'fisheye_param_*'))) + image_dir = [] + fisheye_param_paths = [] + with open(args.image_and_camera_param, 'r') as f: + for i, line in enumerate(f.readlines()): + line = line.strip() + if i % 2 == 0: + image_dir.append(line) + else: + fisheye_param_paths.append(line) fisheye_params = load_camera_parameters(fisheye_param_paths) os.makedirs(args.output_dir, exist_ok=True) perception2d_dict = dict( np.load(args.perception2d_path, allow_pickle=True)) - matched_list = np.load(args.matched_list_path, allow_pickle=True) + matched_list = np.load(args.matched_kps2d_idx, allow_pickle=True) estimator_config = dict(mmcv.Config.fromfile(args.estimator_config)) + estimator_config['logger'] = logger smpl_estimator = build_estimator(estimator_config) keypoints2d_list = [] @@ -67,6 +85,10 @@ def main(args): if args.visualize: n_frame = args.end_frame - args.start_frame n_person = len(smpl_data_list) + colors = get_different_colors(n_person) + tmp = colors[:, 0].copy() + colors[:, 0] = colors[:, 2] + colors[:, 2] = tmp full_pose_list = [] transl_list = [] betas_list = [] @@ -87,20 +109,14 @@ def main(args): model_path='xrmocap_data/body_models', batch_size=1) # prepare camera - k_list = [] - r_list = [] - t_list = [] - for fisheye_param in fisheye_params: - k_list.append(np.array(fisheye_param.get_intrinsic(3))) - r_list.append(np.array(fisheye_param.get_extrinsic_r())) - t_list.append(np.array(fisheye_param.get_extrinsic_t())) - k_np = np.stack(k_list) - r_np = np.stack(r_list) - t_np = np.stack(t_list) - - for cam_name in sorted(os.listdir(args.image_dir)): + for idx, fisheye_param in enumerate(fisheye_params): + k_np = np.array(fisheye_param.get_intrinsic(3)) + r_np = np.array(fisheye_param.get_extrinsic_r()) + t_np = np.array(fisheye_param.get_extrinsic_t()) + cam_name = fisheye_param.name + view_name = cam_name.replace('fisheye_param_', '') frame_list = sorted( - glob.glob(os.path.join(args.image_dir, cam_name, '*.png'))) + glob.glob(os.path.join(image_dir[idx], '*.png'))) frame_list_start = int(frame_list[0][-10:-4]) frame_list = frame_list[args.start_frame - frame_list_start:args.end_frame - @@ -114,12 +130,13 @@ def main(args): poses=fullpose.reshape(n_frame, n_person, -1), betas=betas, transl=transl, + palette=colors, output_path=os.path.join(args.output_dir, - f'{cam_name}_smpl.mp4'), + f'{view_name}_smpl.mp4'), body_model_config=body_model_cfg, - K=k_np[int(cam_name[-1])], - R=r_np[int(cam_name[-1])], - T=t_np[int(cam_name[-1])], + K=k_np, + R=r_np, + T=t_np, image_array=image_array, resolution=(image_array.shape[1], image_array.shape[2]), overwrite=True) @@ -141,9 +158,11 @@ def setup_parser(): parser = argparse.ArgumentParser( description='Estimate smpl from keypoints3d') parser.add_argument( - '--image_dir', - help='Path to the directory containing image', - default='./data/shelf/img') + '--estimator_config', + help='Config file for MultiViewMultiPersonSMPLEstimator', + type=str, + default='configs/modules/core/estimation/' + 'mview_mperson_smpl_estimator.py') parser.add_argument('--start_frame', type=int, default=300) parser.add_argument('--end_frame', type=int, default=600) parser.add_argument('--bbox_thr', type=float, default=0.9) @@ -151,37 +170,36 @@ def setup_parser(): '--keypoints3d_path', type=str, help='Path to input keypoints3d file', - default='./output/shelf/init/scene0_pred_keypoints3d.npz') + default='./output/mvpose_tracking/shelf/scene0_pred_keypoints3d.npz') parser.add_argument( - '--fisheye_param_dir', + '--matched_kps2d_idx', type=str, - help='Path to camera parameter', - default='./xrmocap_data/Shelf/xrmocap_meta_testset/' - 'scene_0/camera_parameters') + default='./output/mvpose_tracking/shelf/scene0_matched_kps2d_idx.npy') + parser.add_argument( + '--image_and_camera_param', + help='A text file contains the image path and the corresponding' + 'camera parameters', + default='./xrmocap_data/Shelf/image_and_camera_param.txt') parser.add_argument( '--perception2d_path', type=str, default='./xrmocap_data/Shelf/xrmocap_meta_testset/' 'scene_0/perception_2d.npz') - parser.add_argument( - '--matched_list_path', - type=str, - default='./output/shelf/init/matched_kps2d_idx.npy') parser.add_argument( '--output_dir', type=str, help='Path to the directory saving', - default='./output/shelf/smpl') - parser.add_argument( - '--estimator_config', - help='Config file for MultiViewMultiPersonSMPLEstimator', - type=str, - default='configs/modules/estimation/mview_mperson_smpl_estimator.py') + default='./output/mvpose_tracking/shelf/smpl') parser.add_argument( '--visualize', action='store_true', help='If checked, visualize result.', default=False) + parser.add_argument( + '--enable_log_file', + action='store_true', + help='If checked, log will be written as file.', + default=False) args = parser.parse_args() return args diff --git a/tools/process_smc.py b/tools/process_smc.py index 12ef808c..a04c4e25 100644 --- a/tools/process_smc.py +++ b/tools/process_smc.py @@ -25,6 +25,12 @@ def main(args): + # check output path + exist_result = check_path_existence(args.output_dir, 'dir') + if exist_result == Existence.MissingParent: + raise FileNotFoundError + elif exist_result == Existence.DirectoryNotExist: + os.mkdir(args.output_dir) file_name = args.smc_path.rsplit('/', 1)[-1] if not args.disable_log_file: time_str = datetime.datetime.now().strftime('%Y-%m-%d_%H:%M:%S') @@ -36,12 +42,6 @@ def main(args): exist_result = check_path_existence(args.smc_path, 'file') if exist_result != Existence.FileExist: raise FileNotFoundError - # check output path - exist_result = check_path_existence(args.output_dir, 'dir') - if exist_result == Existence.MissingParent: - raise FileNotFoundError - elif exist_result == Existence.DirectoryNotExist: - os.mkdir(args.output_dir) # load smc file smc_reader = SMCReader(file_path=args.smc_path) # build estimator diff --git a/xrmocap/human_perception/bbox_detection/mmdet_detector.py b/xrmocap/human_perception/bbox_detection/mmdet_detector.py index 4aa5689d..238b43f1 100644 --- a/xrmocap/human_perception/bbox_detection/mmdet_detector.py +++ b/xrmocap/human_perception/bbox_detection/mmdet_detector.py @@ -50,9 +50,6 @@ def __init__(self, self.det_model = init_detector(**mmdet_kwargs) self.batch_size = batch_size - def __del__(self): - del self.det_model - def infer_array(self, image_array: Union[np.ndarray, list], disable_tqdm: bool = False, diff --git a/xrmocap/human_perception/bbox_detection/mmtrack_detector.py b/xrmocap/human_perception/bbox_detection/mmtrack_detector.py index c09621c4..fa0a566a 100644 --- a/xrmocap/human_perception/bbox_detection/mmtrack_detector.py +++ b/xrmocap/human_perception/bbox_detection/mmtrack_detector.py @@ -47,9 +47,6 @@ def __init__(self, # build the detector from a config file and a checkpoint file self.track_model = init_model(**mmtrack_kwargs) - def __del__(self): - del self.track_model - def infer_array(self, image_array: Union[np.ndarray, list], disable_tqdm: bool = False, diff --git a/xrmocap/ops/top_down_association/mvpose_associator.py b/xrmocap/ops/top_down_association/mvpose_associator.py index 1742fa73..9e136020 100644 --- a/xrmocap/ops/top_down_association/mvpose_associator.py +++ b/xrmocap/ops/top_down_association/mvpose_associator.py @@ -166,8 +166,7 @@ def associate_frame( mview_person_id.append(person_id) image_tensor, kps2d, dim_group, n_kps2d, bbox2d = self.process_data( mview_person_id, mview_img_arr, mview_bbox2d, mview_keypoints2d) - image_tensor = image_tensor.to(self.device) - if self.kalman_tracking is not None: + if len(kps2d) > 0 and self.kalman_tracking is not None: if self.counter == 0: kalman_tracking_requires_init = True else: @@ -187,6 +186,7 @@ def associate_frame( if len(kps2d) > 0: # detect the person kps2d_conf = kps2d[..., 2:3] kps2d = kps2d[..., :2] + image_tensor = image_tensor.to(self.device) matched_list, sub_imgid2cam = self.multi_way_matching( kps2d, image_tensor, self.affinity_estimator, self.fundamental_mat, affinity_type, n_kps2d, dim_group, @@ -309,8 +309,9 @@ def process_data(self, mview_person_id, mview_img_arr, mview_bbox2d, cnt += n_person this_dim.append(cnt) dim_group = torch.Tensor(this_dim).long() - cropped_img = torch.stack(cropped_img) - kps2d = np.concatenate(kps2d, axis=0) + if not (dim_group == 0).all(): + cropped_img = torch.stack(cropped_img) + kps2d = np.concatenate(kps2d, axis=0) n_kps2d = mview_keypoints2d[0].get_keypoints_number() ret_bbox2d = np.concatenate(ret_bbox2d, axis=0) diff --git a/xrmocap/version.py b/xrmocap/version.py index 98bc21c7..7caea7fc 100644 --- a/xrmocap/version.py +++ b/xrmocap/version.py @@ -1,6 +1,6 @@ # Copyright (c) -__version__ = '0.4.0' +__version__ = '0.5.0' def parse_version_info(version_str):