From 3f39b51b8a270d9e90a26ffe0168746dec6c148d Mon Sep 17 00:00:00 2001 From: Wei-Chen-hub <1259566226@qq.com> Date: Tue, 2 Jan 2024 11:07:26 +0800 Subject: [PATCH] update deco 240102 --- mmhuman3d/data/data_converters/__init__.py | 6 +- mmhuman3d/data/data_converters/deco.py | 247 +++++++++++++++++++++ tools/convert_datasets.py | 8 + tools/convert_datasets_commands.txt | 23 +- 4 files changed, 275 insertions(+), 9 deletions(-) create mode 100644 mmhuman3d/data/data_converters/deco.py diff --git a/mmhuman3d/data/data_converters/__init__.py b/mmhuman3d/data/data_converters/__init__.py index eb2e7541..38590654 100644 --- a/mmhuman3d/data/data_converters/__init__.py +++ b/mmhuman3d/data/data_converters/__init__.py @@ -11,6 +11,7 @@ from .coco_wholebody import CocoWholebodyConverter from .crowd3d import Crowd3dConverter from .crowdpose import CrowdposeConverter +from .deco import DecoConverter from .dynacam import DynacamConverter from .eft import EftConverter from .egobody import EgobodyConverter @@ -42,6 +43,7 @@ from .mscoco_neural_annot import MscocoNeuralConverter from .penn_action import PennActionConverter from .posetrack import PosetrackConverter +from .pw3d import Pw3dConverter from .pw3d_hybrik import Pw3dHybrIKConverter from .pw3d_neural_annot import Pw3dNeuralConverter from .renbody import RenbodyConverter @@ -78,6 +80,6 @@ 'HancoConverter', 'Sloper4dConverter', 'Cimi4dConverter', 'SgnifyConverter', 'DynacamConverter', 'BlurhandConverter', 'Hsc4dConverter', 'MotionXConverter', 'MscocoNeuralConverter', - 'TalkshowConverter', 'EmdbConverter', 'Flag3dConverter', 'Crowd3dConverter', - 'BedlamConverter' + 'TalkshowConverter', 'EmdbConverter', 'Flag3dConverter', + 'Crowd3dConverter', 'BedlamConverter','Pw3dConverter', 'DecoConverter' ] diff --git a/mmhuman3d/data/data_converters/deco.py b/mmhuman3d/data/data_converters/deco.py new file mode 100644 index 00000000..fcd88108 --- /dev/null +++ b/mmhuman3d/data/data_converters/deco.py @@ -0,0 +1,247 @@ +import glob +import os +import pdb +import random +import json +import pickle + +import cv2 +import numpy as np +import torch +from tqdm import tqdm + +from mmhuman3d.core.cameras import build_cameras +from mmhuman3d.core.conventions.keypoints_mapping import ( + convert_kps, + get_keypoint_idx, + get_keypoint_idxs_by_part, +) +from mmhuman3d.models.body_models.utils import transform_to_camera_frame +from mmhuman3d.data.data_structures.human_data import HumanData +from mmhuman3d.models.body_models.builder import build_body_model +from .base_converter import BaseModeConverter +from .builder import DATA_CONVERTERS + + +import pdb + + +@DATA_CONVERTERS.register_module() +class DecoConverter(BaseModeConverter): + + ACCEPTED_MODES = ['train', 'test'] + + def __init__(self, modes=[], *args, **kwargs): + + self.device = torch.device( + 'cuda') if torch.cuda.is_available() else torch.device('cpu') + + self.misc_config = dict( + bbox_source='keypoints2d_smpl', + smpl_source='original', + cam_param_type='prespective', + kps3d_root_aligned=False, + has_gender=True, + contact_label=['smpl_vertex', 'polygon_2d'], + ) + + self.smpl_shape = { + 'body_pose': (-1, 69), + 'betas': (-1, 10), + 'global_orient': (-1, 3), + 'transl': (-1, 3),} + + super(DecoConverter, self).__init__(modes) + + + def convert_by_mode(self, dataset_path: str, out_path: str, + mode: str) -> dict: + print('Converting Deco dataset...') + + # build smpl model + smpl_model = build_body_model( + dict( + type='SMPL', + keypoint_src='smpl_45', + keypoint_dst='smpl_45', + model_path='data/body_models/smpl', + gender='neutral', + num_betas=10, + use_pca=False, + batch_size=1)).to(self.device) + + # use HumanData to store data + human_data = HumanData() + + # initialize HumanData + smpl_ = {} + for key in self.smpl_shape.keys(): + smpl_[key] = [] + bboxs_ = {} + for key in ['bbox_xywh']: + bboxs_[key] = [] + image_path_ = [] + keypoints2d_smpl_, keypoints3d_smpl = [], [] + meta_ = {} + for meta_key in ['principal_point', 'focal_length', 'height', 'width']: + meta_[meta_key] = [] + contact_ = {} + for contact_key in self.misc_config['contact_label']: + contact_[contact_key] = [] + + # load annotations + if mode == 'test': + anno_path = os.path.join(dataset_path, 'hot_dca_test.npz') + else: + anno_path = os.path.join(dataset_path, 'hot_dca_trainval.npz') + anno = dict(np.load(anno_path, allow_pickle=True)) + + + seed, size = '231228', '9999' + random.seed(int(seed)) + + # build smpl and get kps3d + body_pose = np.array(anno['pose'])[:, 3:] + global_orient = np.array(anno['pose'])[:, :3] + betas = np.array(anno['shape']) + transl = np.array(anno['transl']) + + output = smpl_model( + body_pose=torch.tensor(body_pose).float().to(self.device), + global_orient=torch.tensor(global_orient).float().to(self.device), + betas=torch.tensor(betas).float().to(self.device), + transl=torch.tensor(transl).float().to(self.device), + ) + smpl_joints = output['joints'] + kps3d = smpl_joints.detach().cpu().numpy() + + # iterate over all images + for iid, imgn in enumerate(tqdm(anno['imgname'])): + + imgp = imgn.replace('datasets/', '') + + # get height and width + image_path = os.path.join(dataset_path, imgp) + image = cv2.imread(image_path) + height, width = image.shape[:2] + + # get cam params + intrinsics = np.array(anno['cam_k'][iid]) + focal_length = [intrinsics[0, 0], intrinsics[1, 1]] + principal_point = [intrinsics[0, 2], intrinsics[1, 2]] + + # build camera + camera = build_cameras( + dict( + type='PerspectiveCameras', + convention='opencv', + in_ndc=False, + focal_length=focal_length, + image_size=(width, height), + principal_point=principal_point)).to(self.device) + + kps2d = camera.transform_points_screen(smpl_joints[iid])[..., :2].detach().cpu().numpy() + + # test write kps2d + # img = cv2.imread(f'{image_path}') + # for kp in kps2d: + # if 0 < kp[0] < width and 0 < kp[1] < height: + # cv2.circle(img, (int(kp[0]), int(kp[1])), 3, (0,0,255), 1) + # pass + # # write image + # os.makedirs(f'{out_path}', exist_ok=True) + # cv2.imwrite(f'{out_path}/{iid}.jpg', img) + + bbox_xywh = self._keypoints_to_scaled_bbox(kps2d, scale=1.25) + + + # image path + image_path_.append(image_path) + + # bbox + bboxs_['bbox_xywh'].append(bbox_xywh) + + # keypoints + keypoints2d_smpl_.append(kps2d) + keypoints3d_smpl.append(kps3d[iid]) + + # smpl params + smpl_['body_pose'].append(body_pose[iid]) + smpl_['betas'].append(betas[iid]) + smpl_['global_orient'].append(global_orient[iid]) + smpl_['transl'].append(transl[iid]) + + # meta + meta_['principal_point'].append(principal_point) + meta_['focal_length'].append(focal_length) + meta_['height'].append(height) + meta_['width'].append(width) + + # contact + contact_['smpl_vertex'].append(anno['contact_label'][iid]) + contact_['polygon_2d'].append(anno['polygon_2d_contact'][iid]) + + size_i = min(int(size), len(image_path_)) + + # meta + human_data['meta'] = meta_ + + # image path + human_data['image_path'] = image_path_ + + # bbox + for bbox_name in bboxs_.keys(): + bbox_ = np.array(bboxs_[bbox_name]).reshape(-1, 4) + bbox_conf = np.ones((bbox_.shape[0], 1)) + bbox_ = np.concatenate([bbox_, bbox_conf], axis=1) + human_data[bbox_name] = bbox_ + + # smpl + for key in smpl_.keys(): + smpl_[key] = np.concatenate( + smpl_[key], axis=0).reshape(self.smpl_shape[key]) + human_data['smpl'] = smpl_ + + # contact + human_data['contact'] = contact_ + + # keypoints2d_smpl + keypoints2d_smpl = np.concatenate( + keypoints2d_smpl_, axis=0).reshape(-1, 45, 2) + keypoints2d_smpl_conf = np.ones([keypoints2d_smpl.shape[0], 45, 1]) + keypoints2d_smpl = np.concatenate( + [keypoints2d_smpl, keypoints2d_smpl_conf], axis=-1) + keypoints2d_smpl, keypoints2d_smpl_mask = \ + convert_kps(keypoints2d_smpl, src='smpl_45', dst='human_data') + human_data['keypoints2d_smpl'] = keypoints2d_smpl + human_data['keypoints2d_smpl_mask'] = keypoints2d_smpl_mask + + # keypoints3d_smpl + keypoints3d_smpl = np.concatenate( + keypoints3d_smpl, axis=0).reshape(-1, 45, 3) + keypoints3d_smpl_conf = np.ones([keypoints3d_smpl.shape[0], 45, 1]) + keypoints3d_smpl = np.concatenate( + [keypoints3d_smpl, keypoints3d_smpl_conf], axis=-1) + keypoints3d_smpl, keypoints3d_smpl_mask = \ + convert_kps(keypoints3d_smpl, src='smpl_45', dst='human_data') + human_data['keypoints3d_smpl'] = keypoints3d_smpl + human_data['keypoints3d_smpl_mask'] = keypoints3d_smpl_mask + + # misc + human_data['misc'] = self.misc_config + human_data['config'] = f'deco_{mode}' + + # save + human_data.compress_keypoints_by_mask() + os.makedirs(out_path, exist_ok=True) + out_file = os.path.join( + out_path, + f'deco_{mode}_{seed}_{"{:04d}".format(size_i)}.npz') + human_data.dump(out_file) + + + + + + + diff --git a/tools/convert_datasets.py b/tools/convert_datasets.py index 3a9d375b..7fde2dc8 100644 --- a/tools/convert_datasets.py +++ b/tools/convert_datasets.py @@ -93,6 +93,10 @@ prefix='gta_human2', modes=['single', 'multiple']), pw3d=dict( + type='Pw3dConverter', + prefix='pw3d', + modes=['train', 'test', 'val']), + pw3d_neural=dict( type='Pw3dNeuralConverter', prefix='pw3d', modes=['train', 'test', 'val']), @@ -117,6 +121,10 @@ type='Cimi4dConverter', # real prefix='cimi4d', modes=['train']), + deco=dict( + type='DecoConverter', # real + prefix='deco', + modes=['train', 'test']), ehf=dict( type='EhfConverter', # real prefix='ehf', diff --git a/tools/convert_datasets_commands.txt b/tools/convert_datasets_commands.txt index b607358c..72307bad 100644 --- a/tools/convert_datasets_commands.txt +++ b/tools/convert_datasets_commands.txt @@ -29,6 +29,13 @@ python tools/convert_datasets.py \ --output_path /mnt/d/datasets/crowd3d/output \ --modes train +# deco-converter +python tools/convert_datasets.py \ + --datasets deco \ + --root_path /mnt/d/datasets \ + --output_path /mnt/d/datasets/deco/output \ + --modes train + # dynacam-converter python tools/convert_datasets.py \ --datasets dynacam \ @@ -66,7 +73,7 @@ python tools/convert_datasets.py \ --datasets flag3d \ --root_path /mnt/d/datasets \ --output_path /mnt/d/datasets/flag3d/output \ - --modes subset + --modes train python tools/preprocess/flag3d_split.py @@ -80,8 +87,8 @@ python tools/convert_datasets.py \ # gta-converter python tools/convert_datasets.py \ --datasets gta_human2 \ - --root_path /mnt/e/ \ - --output_path /mnt/e/gta_human2/output \ + --root_path /mnt/e/datasets \ + --output_path /mnt/e/datasets/gta_human2/output \ --modes multiple # hanco-converter @@ -143,9 +150,9 @@ python tools/convert_datasets.py \ # mpii-converter python tools/convert_datasets.py \ --datasets mpii \ - --root_path /mnt/e \ - --output_path /mnt/e/mpii/output \ - --modes test + --root_path /mnt/e/datasets \ + --output_path /mnt/e/datasets/mpii/output \ + --modes train # pw3d-converter python tools/convert_datasets.py \ @@ -213,7 +220,7 @@ python tools/convert_datasets.py \ --datasets ubody \ --root_path /mnt/d/datasets \ --output_path /mnt/d/datasets/ubody/output \ - --modes inter intra + --modes intra inter # ------------------------------server-------------------------------- @@ -254,3 +261,5 @@ python tools/preprocess/synbody_preprocess_mp.py \ mirror -R -c -v moyo moyo --parallel=5 python tools/postprocess/humandata_sample.py + +