update

else/data_lake/car/Dockerfile

@@ -0,0 +1,31 @@
+# Base Images
+## 从天池基础镜像构建
+#FROM registry.cn-shanghai.aliyuncs.com/tcc-public/pytorch:1.4-cuda10.1-py3
+# ARG PYTORCH="1.7.1"
+# ARG CUDA="10.1"
+ARG PYTORCH="1.6.0"
+ARG CUDA="10.1"
+ARG CUDNN="7"
+FROM pytorch/pytorch:${PYTORCH}-cuda${CUDA}-cudnn${CUDNN}-devel
+#FROM 123696495963
+
+ENV TORCH_CUDA_ARCH_LIST="6.0 6.1 7.0+PTX"
+ENV TORCH_NVCC_FLAGS="-Xfatbin -compress-all"
+ENV CMAKE_PREFIX_PATH="$(dirname $(which conda))/../"
+
+RUN apt-get update && apt-get install -y libgl1-mesa-glx vim libglib2.0-dev ninja-build libglib2.0-0 libsm6 libxrender-dev libxext6 \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
+
+RUN conda clean --all
+
+# Install 
+RUN pip install -i http://pypi.douban.com/simple/ --trusted-host=pypi.douban.com/simple pandas opencv-python timm albumentations pillow scikit-learn
+
+## 把当前文件夹里的文件构建到镜像的根目录下
+ADD . /
+##
+WORKDIR /workspace
+
+#RUN pip install -r requirements.txt
+#RUN pip install --no-cache-dir -e .

else/data_lake/car/debug.py

@@ -0,0 +1,68 @@
+
+
+import cv2
+from PIL import Image
+import pandas as pd
+import numpy as np
+import random
+from torch.utils.data import Dataset,DataLoader
+import torch
+import albumentations as A
+
+from albumentations.pytorch import ToTensorV2
+
+def get_train_transforms(height_size,width_size):
+    return A.Compose([
+            A.Resize(height_size,width_size),#[height,width]
+            #A.HorizontalFlip(p=0.5),
+            A.GaussNoise(p=0.2),
+            A.ShiftScaleRotate(rotate_limit=10,p=0.5),
+            #A.HueSaturationValue(p=0.5),
+            A.RandomBrightnessContrast(p=0.5),
+            A.OneOf([
+                    # 模糊相关操作
+                    A.MotionBlur(p=.75),
+                    A.MedianBlur(blur_limit=3, p=0.5),
+                    A.Blur(blur_limit=3, p=0.75),
+                ], p=0.5),
+            A.OneOf(
+                [
+                A.CoarseDropout(max_holes=8,
+                            max_height=8,
+                            max_width=8,
+                            p=0.5),
+                A.Cutout(
+                    num_holes=8,
+                    max_h_size=8,
+                    max_w_size=8,
+                    p=0.5,)
+                    ],p=0.5),
+            A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),
+            ToTensorV2(p=1.0),
+        ], p=1.)
+
+
+def get_valid_transforms(height_size,width_size):
+    return A.Compose([
+            A.Resize(height_size,width_size),#
+            A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),
+            ToTensorV2(p=1.0),
+        ], p=1.)
+
+def get_img(path):
+    #im_bgr = cv2.imread(path)
+    #im_rgb = im_bgr[:, :, ::-1]
+    im_rgb=np.array(Image.open(path))
+    if len(im_rgb.shape)==2:
+        #im_rgb=np.stack([im_rgb,im_rgb,im_rgb],axis=2)
+        im_rgb=np.expand_dims(im_rgb,axis=-1)
+    return im_rgb
+
+
+if __name__=="__main__":
+    train_csv = pd.read_csv('../train_df.csv')
+    img=get_img('../../input/phase2_train/img_4.jpg')
+    cv2.imwrite('aug_before.jpg',img)
+    aug=A.Rotate(limit=[180,180],p=1.)
+    aug_img=aug(image=img)['image']
+    cv2.imwrite('aug_after.jpg',aug_img)

else/data_lake/car/docker_command.md

@@ -0,0 +1,14 @@
+sudo docker build -t tianma-qyl:1.0 . #当前目录为上下文构建镜像
+docker image ls #列出镜像
+docker ps -a ##列出容器
+docker image rm 12 #删除镜像12
+docker container rm 36 #删除容器36
+docker run -it --rm --gpus 1 e46eca105d3b nvidia-smi #以id为bb2d的镜像构建容器，执行nvidia-smi命令,不加命令则进入容器，完事删除容器
+docker cp ./data myContatiner:/data #拷贝文件到docker容器
+
+docker commit [OPTIONS] CONTAINER [REPOSITORY[:TAG]]#将容器打包为镜像
+docker save -o my_jenkins.tar jenkins:1.0 #打包镜像
+#登录阿里云
+docker login --username=趋于零lim0 registry.cn-hangzhou.aliyuncs.com
+docker tag e46eca105d3b registry.cn-hangzhou.aliyuncs.com/yaogan-swin/forgery:1.0
+docker push registry.cn-hangzhou.aliyuncs.com/yaogan-swin/forgery:1.0

else/data_lake/car/net.py

@@ -0,0 +1,35 @@
+from PIL.Image import Image
+import torch
+from torch import nn
+import timm
+from torch.cuda.amp import autocast, GradScaler
+class ImgClassifier(nn.Module):
+    def __init__(self, model_arch, n_class=14, in_channels=3,pretrained=False):
+        super().__init__()
+        self.model = timm.create_model(model_arch,in_chans=in_channels, pretrained=pretrained)
+        #print(self.model)
+        if model_arch[:2] == 'tf':
+            num_ftrs = self.model.classifier.in_features
+            self.model.classifier = nn.Linear(num_ftrs, n_class)
+        elif model_arch[:3] in ['vit','swi']:
+            num_ftrs = self.model.head.in_features
+            self.model.head = nn.Linear(num_ftrs, n_class)
+        elif model_arch[:3] in ['eca','con','rep','dm_']:
+            num_ftrs = self.model.head.fc.in_features
+            self.model.head.fc = nn.Linear(num_ftrs, n_class)
+        else:
+            num_ftrs = self.model.fc.in_features
+            self.model.fc = nn.Linear(num_ftrs, n_class)
+        # self.classifier = nn.Sequential(
+        #         nn.Dropout(0.5),
+        #         nn.Linear(num_ftrs, n_class)
+        #     )
+    @autocast()
+    def forward(self, x):
+        x = self.model(x)
+        return x
+#
+if __name__=="__main__":
+    net=ImgClassifier(model_arch='convnext_base_in22ft1k',n_class=2)
+    x=torch.randn([1,3,224,224])
+    net(x)

else/data_lake/car/predict.py

@@ -0,0 +1,138 @@
+#
+import torch
+import os
+os.environ['CUDA_VISIBLE_DEVICES'] = '0'
+import pandas as pd
+import numpy as np
+from tqdm import tqdm
+import torch.nn.functional as F
+from net import ImgClassifier
+from torch.utils.data import Dataset
+import albumentations as A
+from albumentations.pytorch import ToTensorV2
+from PIL import Image
+import csv
+
+def img_loader(img_path):
+    img = Image.open(img_path)
+    return img
+class carData(Dataset):
+    def __init__(self, data_path,
+                 transform=None):
+        super(Dataset, self).__init__()
+        self.samples = data_path
+        self.transform=transform
+
+    def __len__(self):
+        return len(self.samples)
+
+    def __getitem__(self, index):
+        img_path = self.samples[index]
+        img = img_loader(img_path)
+        if self.transform is not None:
+            img = self.transform(image=np.array(img))['image']
+        return img
+def get_inference_transforms(input_size):
+    return A.Compose([
+            A.Resize(input_size,input_size),#276, 344
+            #A.HorizontalFlip(p=0.5),
+            #A.RandomBrightnessContrast(p=0.5),
+            A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),
+            ToTensorV2(p=1.0),
+        ], p=1.)
+
+@torch.no_grad()
+def inference(model, data_loader, device):
+    model.eval()
+    image_preds_all = []
+    img_path_all=[]
+    pbar = tqdm(enumerate(data_loader), total=len(data_loader))
+    for step, (data) in pbar:
+        imgs = data
+        imgs=imgs.to(device).float()
+        image_preds = model(imgs)
+        #print(image_preds)
+        image_preds_all.append(image_preds)
+    #
+    image_preds_all=torch.cat(image_preds_all)
+    return image_preds_all
+if __name__=="__main__":
+    save_dir='./'
+    os.makedirs(save_dir,exist_ok=True)
+    df=pd.DataFrame(columns=['id','type','color','toward'])
+    test_dir='../data/phase2B_test/'
+    test_images = os.listdir(test_dir)
+    test_images.sort(key=lambda x: int(x.split('.')[0].split('_')[1]))
+    #print(test_images)
+    test_paths=[os.path.join(test_dir,i) for i in test_images]
+    preds_lst = []
+    device = torch.device('cuda')
+    for tta in range(1):
+        infer_ds = carData(test_paths,get_inference_transforms(224))
+        infer_loader = torch.utils.data.DataLoader(
+                infer_ds,
+                batch_size=8,
+                num_workers=4,
+                shuffle=False,
+                pin_memory=False,
+            )
+        # for fold in [0]:
+        #     print('Inference TTA:{} fold {} started'.format(tta,fold))
+        #     model = ImgClassifier('convnext_large_in22ft1k', 21, pretrained=False).to(device)
+        #     model.load_state_dict(torch.load('./ckpt_reproduct/convnext_xlarge_in22ft1k_fold_{}_best.pth'.format(fold)))
+        #     preds_lst.append(inference(model, infer_loader, device))
+        #     del model
+        #     torch.cuda.empty_cache()
+        for fold in [0]:
+            print('Inference TTA:{} fold {} started'.format(tta,fold))
+            model = ImgClassifier('swin_large_patch4_window7_224', 21, pretrained=False).to(device)
+            model.load_state_dict(torch.load('./ckpt/swin_large_patch4_window7_224/swin_large_patch4_window7_224_fold_{}_last.pth'.format(fold)))
+            preds_lst.append(inference(model, infer_loader, device))
+            del model
+            torch.cuda.empty_cache()
+        #
+        for fold in [0]:
+            print('Inference TTA:{} fold {} started'.format(tta,fold))
+            model = ImgClassifier('convnext_xlarge_in22ft1k', 21, pretrained=False).to(device)
+            model.load_state_dict(torch.load('./ckpt/convnext_xlarge_in22ft1k/convnext_xlarge_in22ft1k_fold_{}_last.pth'.format(fold)))
+            preds_lst.append(inference(model, infer_loader, device))
+            del model
+            torch.cuda.empty_cache()
+    #模型融合
+    print(len(preds_lst))
+    #ratio=[0.6/5]*5+[0.4/5]*5
+    ratio=[0.4,0.6]
+    assert abs(np.sum(ratio)-1)<1e-3
+    for i in range(len(preds_lst)):
+        if i==0:
+            preds=ratio[i]*preds_lst[i]
+        else:
+            preds+=ratio[i]*preds_lst[i]
+    #
+    dic={'type': {0: 'van', 1: 'truck', 2: 'car', 3: 'suv', 4: 'coach', 5: 'bus', 6: 'engineering_car'}, 
+    'color': {0: 'gray', 1: 'black', 2: 'indigo', 3: 'white', 4: 'red', 5: 'blue', 6: 'silvery', 7: 'brown', 8: 'gold', 9: 'yellow'},
+    'toward': {0: 'left', 1: 'right', 2: 'back', 3: 'front'}}
+    #
+    type_lst=[]
+    color_lst=[]
+    toward_lst=[]
+    name_lst=[]
+    cnt=0
+    for i in range(preds.shape[0]):
+        output=preds[i]
+        type= torch.argmax(output[:7]).item()
+        color= torch.argmax(output[7:17]).item()
+        toward= torch.argmax(output[17:]).item()
+        type_lst.append(dic['type'][type])
+        color_lst.append(dic['color'][color])
+        toward_lst.append(dic['toward'][toward])
+        #
+        name_lst.append(test_images[cnt])
+        cnt+=1
+    #
+    df['id']=name_lst
+    df['type']=type_lst
+    df['color']=color_lst
+    df['toward']=toward_lst
+    print(df)
+    df.to_csv(save_dir+'/result.csv',index=False)

else/data_lake/car/process.py

@@ -0,0 +1,45 @@
+
+import cv2
+import glob
+import os
+import pandas as pd
+#
+'''
+{'type': {'van': 0, 'truck': 1, 'car': 2, 'suv': 3, 'coach': 4, 'bus': 5, 'engineering_car': 6}, 
+ 'color': {'gray': 0, 'black': 1, 'indigo': 2, 'white': 3, 'red': 4, 'blue': 5, 'silvery': 6, 'brown': 7, 'gold': 8, 'yellow': 9},
+ 'toward': {'left': 0, 'right': 1, 'back': 2, 'front': 3}}
+'''
+#
+columns=['type','color','toward']
+# 
+if __name__=="__main__":
+    data_dir="../data/car/phase2_train"
+    df=pd.read_csv('../data/car/phase2_train_sorted.csv')
+    unique_dic={}
+    for col in columns:
+        tmp=df[col].unique().tolist()
+        dic={}
+        for i in range(len(tmp)):
+            dic[tmp[i]]=i
+        unique_dic[col]=dic
+    print(unique_dic)
+    df=df.fillna(0)
+    target=[]
+    for index,row in df.iterrows():
+        name,car_type,color,toward=row
+        #
+        car_type_code=[0]*len(unique_dic['type'])
+        car_type_code[unique_dic['type'][car_type]]=1
+        #
+        color_code=[0]*len(unique_dic['color'])
+        color_code[unique_dic['color'][color]]=1
+        #print(clothesStyles_code)
+        #
+        toward_code=[0]*len(unique_dic['toward'])
+        toward_code[unique_dic['toward'][toward]]=1
+        #print(hairStyles_code)
+        encode=car_type_code+color_code+toward_code
+        target.append(encode)
+    df['target']=target
+    df.to_csv('./train_df.csv',index=False)
+    print(df)