diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 0000000..00afb80 --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,22 @@ +xView2 +Copyright 2019 Carnegie Mellon University. +MIT (SEI) + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +xView2-update includes and/or can make use of certain third party software ("Third Party Software"). The Third Party Software that is used by xView2-update is dependent upon your system configuration, but typically includes the software identified below. By using xView2-update, You agree to comply with any and all relevant Third Party Software terms and conditions contained in any such Third Party Software or separate license file distributed with such Third Party Software. The parties who own the Third Party Software ("Third Party Licensors") are intended third party beneficiaries to this License with respect to the terms applicable to their Third Party Software. Third Party Software licenses only apply to the Third Party Software and not any other portion of xView2-update or xView2-update as a whole. + +This material is based upon work funded and supported by the Department of Defense under Contract No. FA8702-15-D-0002 with Carnegie Mellon University for the operation of the Software Engineering Institute, a federally funded research and development center. +The view, opinions, and/or findings contained in this material are those of the author(s) and should not be construed as an official Government position, policy, or decision, unless designated by other documentation. + +References herein to any specific commercial product, process, or service by trade name, trade mark, manufacturer, or otherwise, does not necessarily constitute or imply its endorsement, recommendation, or favoring by Carnegie Mellon University or its Software Engineering Institute. + +NO WARRANTY. THIS CARNEGIE MELLON UNIVERSITY AND SOFTWARE ENGINEERING INSTITUTE MATERIAL IS FURNISHED ON AN "AS-IS" BASIS. CARNEGIE MELLON UNIVERSITY MAKES NO WARRANTIES OF ANY KIND, EITHER EXPRESSED OR IMPLIED, AS TO ANY MATTER INCLUDING, BUT NOT LIMITED TO, WARRANTY OF FITNESS FOR PURPOSE OR MERCHANTABILITY, EXCLUSIVITY, OR RESULTS OBTAINED FROM USE OF THE MATERIAL. CARNEGIE MELLON UNIVERSITY DOES NOT MAKE ANY WARRANTY OF ANY KIND WITH RESPECT TO FREEDOM FROM PATENT, TRADEMARK, OR COPYRIGHT INFRINGEMENT. +[DISTRIBUTION STATEMENT A] This material has been approved for public release and unlimited distribution. Please see Copyright notice for non-US Government use and distribution. + +This Software includes and/or makes use of the following Third-Party Software subject to its own license: +1. SpaceNet (https://github.com/motokimura/spacenet_building_detection/blob/master/LICENSE) Copyright 2017 Motoki Kimura. + +DM19-0988 diff --git a/README.md b/README.md new file mode 100644 index 0000000..8c76119 --- /dev/null +++ b/README.md @@ -0,0 +1,323 @@ +# xView 2 Challenge + +xView2 + +Copyright 2019 Carnegie Mellon University. + +NO WARRANTY. THIS CARNEGIE MELLON UNIVERSITY AND SOFTWARE ENGINEERING INSTITUTE MATERIAL IS FURNISHED ON AN "AS-IS" BASIS. CARNEGIE MELLON UNIVERSITY MAKES NO WARRANTIES OF ANY KIND, EITHER EXPRESSED OR IMPLIED, AS TO ANY MATTER INCLUDING, BUT NOT LIMITED TO, WARRANTY OF FITNESS FOR PURPOSE OR MERCHANTABILITY, EXCLUSIVITY, OR RESULTS OBTAINED FROM USE OF THE MATERIAL. CARNEGIE MELLON UNIVERSITY DOES NOT MAKE ANY WARRANTY OF ANY KIND WITH RESPECT TO FREEDOM FROM PATENT, TRADEMARK, OR COPYRIGHT INFRINGEMENT. + +Released under a MIT (SEI)-style license, please see LICENSE.md or contact permission@sei.cmu.edu for full terms. + +[DISTRIBUTION STATEMENT A] This material has been approved for public release and unlimited distribution. Please see Copyright notice for non-US Government use and distribution. + +This Software includes and/or makes use of the following Third-Party Software subject to its own license: +1. SpaceNet (https://github.com/motokimura/spacenet_building_detection/blob/master/LICENSE) Copyright 2017 Motoki Kimura. +DM19-0988 + +## xView + +xView2 is a challenge set forth by [DIU](https://diu.mil) for assessing damaged buildings after a natural disaster. The challenge is open to all to participate and more information can be found at the [xview2](https://xview2.org) challenge site. + +## xBD + +xBD is the name of the dataset that was generated for this challenge. The dataset contains over 20,000KM2 of polygon labeled pre and post disaster imagery. The dataset provides the post-disaster imagery with transposed polygons from pre over the buildings, with damage classification labels. See the [xBD paper](http://openaccess.thecvf.com/content_CVPRW_2019/papers/cv4gc/Gupta_Creating_xBD_A_Dataset_for_Assessing_Building_Damage_from_Satellite_CVPRW_2019_paper.pdf) for more information. + +# CMU SEI Baseline Submission + +## About + +Our submission uses a combination of a localization polygon detection model and a classification model. + +## Environment Setup + +During development we used `Python`'s `miniconda` This is recommended as it can help control dependencies and the version of those dependencies are imported easily without breaking your existing repositories. You can also use whichever other virtual environment you want. You can get this through the `Python` package manager [pip](https://pip.pypa.io/en/stable/installing/). + +Our **minimum Python version is 3.6+**, you can get it from [here](https://www.python.org/downloads/release/python-369/). + + Before you get started be sure to have `Python`, `pip` (through conda), `conda` installed and running on Python 3.6+. + * We highly recommend using `conda` to manage your packages and versions. Initialize this by running `conda create -n $ENV_NAME` within your xView2 root directory. + * This will create an anaconda virtual environment ready to install dependencies. + * Finally install pip like `conda activate $ENV_NAME && conda install pip` + +Read more about `miniconda` [here](https://docs.conda.io/en/latest/miniconda.html). + +Once in your own virtual environment you can install the packages required to train and run the baseline model. + +Check you are using `conda`'s `pip` and `python` by doing `$ which pip` it should report something like: `/path/to/miniconda/envs/$ENV_NAME/bin/pip`, if not you may need to run `conda activate` or check your `.bashrc`. + +Before installing all dependencies run `$ pip install numpy tensorflow` for CPU-based machines or `$ pip install numpy tensorflow-gpu && conda install cupy` for GPU-based (CUDA) machines, as they are install-time dependencies for some other packages. + +Finally, use the provided [requirements.txt](./requirements.txt) file for the remainder of the Python dependencies like so, `$ pip install -r requirements.txt` (make sure you are in the same environment as before (`conda activate $ENV_NAME`), or else the packages will be scattered and not easily found). + +## Data Downloads + +The data during the challenge is available from [xView2](https://xview2.org) challenge website, please register or login to download the data! + +The current total space needed is: about **10GB** compressed and about **11GB** uncompressed. + +### Other Methods + +If you are using our pipeline the best way to format data is like so (the pipeline scripts `mask_polygons.py`, and `data_finalize.sh` provided in the repository expect this structure): + +``` +xBD + ├── disaster_name_1 + │ ├── images + │ │ └── .png + │ │ └── ... + │ ├── labels + │ │ └── .json + │ │ └── ... + ├── disaster_name_2 + │ ├── images + │ │ └── .png + │ │ └── ... + │ ├── labels + │ │ └── .json + │ │ └── ... + └── disaster_name_n +``` + +## Baseline + +We are using a fork of motokimura's [SpaceNet Building Detection](https://github.com/motokimura/spacenet_building_detection) for our localization in order to automatically pull our polygons to feed into our classifier. + +We have provided several resources for formatting the dataset for this submission in the [utils](./utils) folder. Below are our pipeline steps. + +### Installing Packages and Dependencies + +Before you get started be sure to have followed the steps mentioned in [Environment Setup](https://github.com/DIUx-xView/xView2#Environment-Setup) and have `conda activated $ENV_NAME` to have your dependencies loaded into your environment. + +### Training + +#### Localization Training + +Below we will walk through the steps we have used for the localization training. + +#### Localization Training Pipeline + +These are the pipeline steps are below for the instance segmentation training (these programs have been written and tested on Unix systems (Darwin, Fedora, Debian) only). + +First, we must create masks for the localization, and have the data in specific folders for the model to find and train itself. The steps we have built are described below: + +1. Run `mask_polygons.py` to generate a mask file for the chipped images. + * Sample call: `python mask_polygons.py --input /path/to/xBD --single-file --border 2` + * Here border refers to shrinking polygons by X number of pixels. This is to help the model separate buildings when there are a lot of "overlapping" or closely placed polygons + * Run `python mask_polygons.py --help` for the full description of the options. +2. Run `data_finalize.sh` to setup the image and labels directory hierarchy that the `spacenet` model expects (will also run a python `compute_mean.py` to create a mean image our model uses during training. + * sample call: `data_finalize.sh -i /path/to/xBD/ -x /path/to/xView2/repo/root/dir/ -s .75` + * -s is a crude train/val split, the decimal you give will be the amount of the total data to assign to training, the rest to validation + * You can find this later in /path/to/xBD/spacenet_gt/dataSplit in text files, and easily change them after the script has been run. + * Run `data_finalize.sh` for the full description of the options. + +After these steps have been ran you will be ready for the instance segmentation training. + +The directory structure will look like: + +``` +/path/to/xBD/ +├── guatemala-volcano +│   ├── images +│   ├── labels +│   └── masks +├── hurricane-florence +│   ├── images +│   ├── labels +│   └── masks +├── hurricane-harvey +│   ├── images +│   ├── labels +│   └── masks +├── hurricane-matthew +│   ├── images +│   ├── labels +│   └── masks +├── hurricane-michael +│   ├── images +│   ├── labels +│   └── masks +├── mexico-earthquake +│   ├── images +│   ├── labels +│   └── masks +├── midwest-flooding +│   ├── images +│   ├── labels +│   └── masks +├── nepal-flooding +│   ├── images +│   ├── labels +│   └── masks +├── palu-tsunami +│   ├── images +│   ├── labels +│   └── masks +├── santa-rosa-wildfire +│   ├── images +│   ├── labels +│   └── masks +├── socal-fire +│   ├── images +│   ├── labels +│   └── masks +└── spacenet_gt + ├── dataSet + ├── images + └── labels +``` + +The original images and labels are preserved in the `./xBD/org/$DISASTER/` directories, and just copies the images to the `spacenet_gt` directory. + +#### Training the SpaceNet Model + +Now you will be ready to start training a model (based off our provided [weights](put path to release weights here), or from a baseline). + +Using the `spacenet` model we forked, you can control all of the options via command line calls. + +In order for the model to find all of its required files, you will need to `$ cd /path/to/xView2/spacenet/src/models/` before running the training module. + +The main file is [`train_model.py`](./spacenet/src/models/train_model.py) and the options are below: + +``` +usage: train_model.py [-h] [--batchsize BATCHSIZE] + [--test-batchsize TEST_BATCHSIZE] [--epoch EPOCH] + [--frequency FREQUENCY] [--gpu GPU] [--out OUT] + [--resume RESUME] [--noplot] [--tcrop TCROP] + [--vcrop VCROP] + dataset images labels + +positional arguments: + dataset Path to directory containing train.txt, val.txt, and + mean.npy + images Root directory of input images + labels Root directory of label images + +optional arguments: + -h, --help show this help message and exit + --batchsize BATCHSIZE, -b BATCHSIZE + Number of images in each mini-batch + --test-batchsize TEST_BATCHSIZE, -B TEST_BATCHSIZE + Number of images in each test mini-batch + --epoch EPOCH, -e EPOCH + Number of sweeps over the dataset to train + --frequency FREQUENCY, -f FREQUENCY + Frequency of taking a snapshot + --gpu GPU, -g GPU GPU ID (negative value indicates CPU) + --out OUT, -o OUT Directory to output the result under "models" + directory + --resume RESUME, -r RESUME + Resume the training from snapshot + --noplot Disable PlotReport extension + --tcrop TCROP Crop size for train-set images + --vcrop VCROP Crop size for validation-set images +``` + +A sample call we used is below: + +`$ python train_model.py /path/to/xBD/spacenet_gt/dataSet/ /path/to/xBD/spacenet_gt/images/ /path/to/xBD/spacenet_gt/labels/ -e 100` + +**WARNING**: You must be in the `./spacenet/src/models/` directory to run the model due to relative paths for `spacenet` packages, as they have been edited for this instance and would be difficult to package. + +#### Damage Classification Training + +The damage classification training processing and training code can be found under `/path/to/xView2/model/` + +You will need to run the `process_data.py` python script to extract the polygon images used for training, testing, and holdout from the original satellite images and the polygon labels produced by SpaceNet. This will generate a csv file with polygon UUID and damage type +as well as extracting the actual polygons from the original satellite images. + +**Note** The process_data script only extracts polygons from post disaster images + +``` +usage: python process_data [-h] --input_dir --output_dir + +arguments: + +-h show help message and exit +--input_dir path to xBD data already split into train, test, and hold +--output_dir existing path to an output directory that has empty folders for train, test, and hold + +``` + +**Note** Please update the LOG_DIR to where you want your tensorboard logs to go before running damage_classification.py +If you want to update details like the BATCHSIZE, NUM_WORKERS, EPOCHS, or LEARNING_RATE you will need to open damage_classification.py and edit the configuration variables at the top of the script. + +``` +usage: python damage_classification.py [-h] --train_data TRAIN_DATA_PATH + --train_csv TRAIN_CSV + --test_data TEST_DATA_PATH + --test_csv TEST_CSV + [--model_in MODEL_WEIGHTS_PATH] + --model_out MODEL_WEIGHTS_OUT + +arguments: + + -h show help message and exit + --train_data TRAIN_DATA_PATH Path to training polygons + --train_csv TRAIN_CSV Path to train csv + --test_data TEST_DATA_PATH Path to test polygons + --test_csv TEST_CSV Path to test csv + --model_in MODEL_WEIGHTS_PATH Path to any input weights (optional) + --model_out MODEL_WEIGHTS_OUT Path to output weights (do not add file extention) +``` + +Sample command: `$ python damage_classification.py --train_data /path/to/XBD/polygons/train +--train_csv train.csv --test_data /path/to/XBD/polygons/test --test_csv test.csv --model_out path/to/xBD/baseline_trial --model_in /path/to/saved-model-01.hdf5` + +### Inference + +Since the inference is done over two models we have created a script [inference.sh](./utils/inference.sh) to script together the inference code. + +If you would like to run the inference steps individually the shell script will provide you with those steps. + +To run the inference code you will need: + +1. The `xView2` repository (this one, where you are reading this README) cloned +2. An input pre image +3. An input post image that matches the pre in the same directory +4. Weights for the localization model +5. Weights for the classification model + +You can find the weights we have trained in the releases section of this Github repository. + +As long as we can find the post image by replacing pre with post (`s/pre/post/g`) everything else should be run, this is used to dockerize the inference and run in parallel for each image individually based off the submission requirements of the challenge. + +Sample Call: `./utils/inference.sh -x /path/to/xView2/ -i /path/to/$DISASTER_$IMAGEID_pre_disaster.png -p /path/to/$DISASTER_$IMAGEID_post_disaster.png -l /path/to/localization_weights.h5 -c /path/to/classification_weights.hdf5 -o /path/to/output/image.png -y` + +The inference script writes log files to `/tmp/inference_log` and the intermediately steps to `/tmp/inference`, normally `/tmp/inference` is removed after each run, but can be kept (and you can see the final json for visual checking in `/tmp/inferences/inference.json`) by commenting out [this line](./utils/inference.sh#L172) in `./utils/inference.sh`. + +### Docker container + +The submission [Dockerfile](./submission/Dockerfile) wraps the above inference script and requires the user to mount a folder with the pre/post image pair, and a separate folder for the inference output image. + +To simplify the creation of the docker follow these steps: +1. Ensure Docker is installed and running (with the ability to have ~8GB of RAM) +2. Move a directory above this repository +3. Build by calling `$ docker build -t cmu-xview2-baseline -f /path/to/xView2/submission/Dockerfile .` + +This will build the image (based off Ubuntu Docker baseline) and add in the repository. + +**WARNING**: The docker image downloads some files (ResNet etc.) be sure to be off a VPN or give the Docker daemon access to the local network environment for network requests. + +Running the image will need a directory for the files, and an output directory to write to. Below is a sample call to run the inference docker image for submission (this will output two of the same output files for scoring (scoring is done at the localization and classification stages, but we output the file at the end and would use the same polygons found in the localization stage anyways). + + +`$ docker run -v /local/path/to/xBD/folder/with/images/:/submission/ -v /local/path/to/output/scoring/files/:/output/ cmu-xview2-baseline /submission/path/to/pre_image.png /submission/path/to/post_image.png /output/localization_output_name.png /output/classification_output_name.png` + +### Output + +The output will be a grey scale PNG with the following pixel values: + +``` +0 for no building +1 for building found and classified no-damaged +2 for building found and classified minor-damage +3 for building found and classified major-damage +4 for building found and classified destroyed +``` + +See the code we use to translate the json outputs to this submission image [inference_image_output.py](./utils/inference_image_output.py), also see the submission rules on the [xview2 website](https://xview2.org/challenge). + +## Contact + +See the [xView2 FAQ]() page first, if you would like to talk to us further reach out to the xView2 chat on [discord](https://xview2.org/contact). + diff --git a/model/damage_classification.py b/model/damage_classification.py new file mode 100644 index 0000000..cacc8b7 --- /dev/null +++ b/model/damage_classification.py @@ -0,0 +1,245 @@ +##################################################################################################################################################################### +# xView2 # +# Copyright 2019 Carnegie Mellon University. # +# NO WARRANTY. THIS CARNEGIE MELLON UNIVERSITY AND SOFTWARE ENGINEERING INSTITUTE MATERIAL IS FURNISHED ON AN "AS-IS" BASIS. CARNEGIE MELLON UNIVERSITY MAKES NO # +# WARRANTIES OF ANY KIND, EITHER EXPRESSED OR IMPLIED, AS TO ANY MATTER INCLUDING, BUT NOT LIMITED TO, WARRANTY OF FITNESS FOR PURPOSE OR MERCHANTABILITY, # +# EXCLUSIVITY, OR RESULTS OBTAINED FROM USE OF THE MATERIAL. CARNEGIE MELLON UNIVERSITY DOES NOT MAKE ANY WARRANTY OF ANY KIND WITH RESPECT TO FREEDOM FROM PATENT, # +# TRADEMARK, OR COPYRIGHT INFRINGEMENT. # +# Released under a MIT (SEI)-style license, please see LICENSE.md or contact permission@sei.cmu.edu for full terms. # +# [DISTRIBUTION STATEMENT A] This material has been approved for public release and unlimited distribution. Please see Copyright notice for non-US Government use # +# and distribution. # +# This Software includes and/or makes use of the following Third-Party Software subject to its own license: # +# 1. SpaceNet (https://github.com/motokimura/spacenet_building_detection/blob/master/LICENSE) Copyright 2017 Motoki Kimura. # +# DM19-0988 # +##################################################################################################################################################################### + +from PIL import Image +import time +import numpy as np +import pandas as pd +from tqdm import tqdm +import os +import math +import random +import argparse +import logging +import json +import cv2 +import datetime + +from sklearn.metrics import f1_score +from sklearn.utils.class_weight import compute_class_weight +import shapely.wkt +import shapely +from shapely.geometry import Polygon +from collections import defaultdict + +import tensorflow as tf +import keras +import ast +from keras import Sequential +from keras.layers import Conv2D, MaxPooling2D, Dense, Flatten, Add, Input, Concatenate +from keras.models import Model +from keras.applications.resnet50 import ResNet50 +from keras import backend as K + +from model import * + +logging.basicConfig(level=logging.INFO) + +# Configurations +NUM_WORKERS = 4 +NUM_CLASSES = 4 +BATCH_SIZE = 64 +NUM_EPOCHS = 100 +LEARNING_RATE = 0.0001 +RANDOM_SEED = 123 +LOG_STEP = 150 +LOG_DIR = '/path/to/logs' + datetime.datetime.now().strftime("%Y%m%d-%H%M%S") + +damage_intensity_encoding = dict() +damage_intensity_encoding[3] = '3' +damage_intensity_encoding[2] = '2' +damage_intensity_encoding[1] = '1' +damage_intensity_encoding[0] = '0' + + +### +# Function to compute unweighted f1 scores, just for reference +### +def f1(y_true, y_pred): + def recall(y_true, y_pred): + """Recall metric. + + Only computes a batch-wise average of recall. + + Computes the recall, a metric for multi-label classification of + how many relevant items are selected. + """ + true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1))) + possible_positives = K.sum(K.round(K.clip(y_true, 0, 1))) + recall = true_positives / (possible_positives + K.epsilon()) + return recall + + def precision(y_true, y_pred): + """Precision metric. + + Only computes a batch-wise average of precision. + + Computes the precision, a metric for multi-label classification of + how many selected items are relevant. + """ + true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1))) + predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1))) + precision = true_positives / (predicted_positives + K.epsilon()) + return precision + + + precision = precision(y_true, y_pred) + recall = recall(y_true, y_pred) + return 2*((precision*recall)/(precision+recall+K.epsilon())) + + +### +# Creates data generator for validation set +### +def validation_generator(test_csv, test_dir): + df = pd.read_csv(test_csv) + df = df.replace({"labels" : damage_intensity_encoding }) + + gen = keras.preprocessing.image.ImageDataGenerator( + rescale=1.4) + + + return gen.flow_from_dataframe(dataframe=df, + directory=test_dir, + x_col='uuid', + y_col='labels', + batch_size=BATCH_SIZE, + shuffle=False, + seed=RANDOM_SEED, + class_mode="categorical", + target_size=(128, 128)) + + +### +# Applies random transformations to training data +### +def augment_data(df, in_dir): + + df = df.replace({"labels" : damage_intensity_encoding }) + gen = keras.preprocessing.image.ImageDataGenerator(horizontal_flip=True, + vertical_flip=True, + width_shift_range=0.1, + height_shift_range=0.1, + rescale=1.4) + return gen.flow_from_dataframe(dataframe=df, + directory=in_dir, + x_col='uuid', + y_col='labels', + batch_size=BATCH_SIZE, + seed=RANDOM_SEED, + class_mode="categorical", + target_size=(128, 128)) + + +# Run training and evaluation based on existing or new model +def train_model(train_data, train_csv, test_data, test_csv, model_in, model_out): + + model = generate_xBD_baseline_model() + + # Add model weights if provided by user + if model_in is not None: + model.load_weights(model_in) + + df = pd.read_csv(train_csv) + class_weights = compute_class_weight('balanced', np.unique(df['labels'].to_list()), df['labels'].to_list()); + d_class_weights = dict(enumerate(class_weights)) + + samples = df['uuid'].count() + steps = np.ceil(samples/BATCH_SIZE) + + # Augments the training data + train_gen_flow = augment_data(df, train_data) + + #Set up tensorboard logging + tensorboard_callbacks = keras.callbacks.TensorBoard(log_dir=LOG_DIR, + batch_size=BATCH_SIZE) + + + #Filepath to save model weights + filepath = model_out + "-saved-model-{epoch:02d}-{acc:.2f}.hdf5" + checkpoints = keras.callbacks.ModelCheckpoint(filepath, + monitor=['loss', 'accuracy'], + verbose=1, + save_best_only=False, + mode='max') + + #Adds adam optimizer + adam = keras.optimizers.Adam(lr=LEARNING_RATE, + beta_1=0.9, + beta_2=0.999, + epsilon=None, + decay=0.0, + amsgrad=False) + + + model.compile(loss=ordinal_loss, optimizer=adam, metrics=['accuracy', f1]) + + #Training begins + model.fit_generator(generator=train_gen_flow, + steps_per_epoch=steps, + epochs=NUM_EPOCHS, + workers=NUM_WORKERS, + use_multiprocessing=True, + class_weight=d_class_weights, + callbacks=[tensorboard_callbacks, checkpoints], + verbose=1) + + + #Evalulate f1 weighted scores on validation set + validation_gen = validation_generator(test_csv, test_data) + predictions = model.predict(validation_gen) + + val_trues = validation_gen.classes + val_pred = np.argmax(predictions, axis=-1) + + f1_weighted = f1_score(val_trues, val_pred, average='weighted') + print(f1_weighted) + + +def main(): + + parser = argparse.ArgumentParser(description='Run Building Damage Classification Training & Evaluation') + parser.add_argument('--train_data', + required=True, + metavar="/path/to/xBD_train", + help="Full path to the train data directory") + parser.add_argument('--train_csv', + required=True, + metavar="/path/to/xBD_split", + help="Full path to the train csv") + parser.add_argument('--test_data', + required=True, + metavar="/path/to/xBD_test", + help="Full path to the test data directory") + parser.add_argument('--test_csv', + required=True, + metavar="/path/to/xBD_split", + help="Full path to the test csv") + parser.add_argument('--model_in', + default=None, + metavar='/path/to/input_model', + help="Path to save model") + parser.add_argument('--model_out', + required=True, + metavar='/path/to/save_model', + help="Path to save model") + + args = parser.parse_args() + + train_model(args.train_data, args.train_csv, args.test_data, args.test_csv, args.model_in, args.model_out) + + +if __name__ == '__main__': + main() diff --git a/model/damage_inference.py b/model/damage_inference.py new file mode 100644 index 0000000..4f10fa5 --- /dev/null +++ b/model/damage_inference.py @@ -0,0 +1,155 @@ +##################################################################################################################################################################### +# xView2 # +# Copyright 2019 Carnegie Mellon University. # +# NO WARRANTY. THIS CARNEGIE MELLON UNIVERSITY AND SOFTWARE ENGINEERING INSTITUTE MATERIAL IS FURNISHED ON AN "AS-IS" BASIS. CARNEGIE MELLON UNIVERSITY MAKES NO # +# WARRANTIES OF ANY KIND, EITHER EXPRESSED OR IMPLIED, AS TO ANY MATTER INCLUDING, BUT NOT LIMITED TO, WARRANTY OF FITNESS FOR PURPOSE OR MERCHANTABILITY, # +# EXCLUSIVITY, OR RESULTS OBTAINED FROM USE OF THE MATERIAL. CARNEGIE MELLON UNIVERSITY DOES NOT MAKE ANY WARRANTY OF ANY KIND WITH RESPECT TO FREEDOM FROM PATENT, # +# TRADEMARK, OR COPYRIGHT INFRINGEMENT. # +# Released under a MIT (SEI)-style license, please see LICENSE.md or contact permission@sei.cmu.edu for full terms. # +# [DISTRIBUTION STATEMENT A] This material has been approved for public release and unlimited distribution. Please see Copyright notice for non-US Government use # +# and distribution. # +# This Software includes and/or makes use of the following Third-Party Software subject to its own license: # +# 1. SpaceNet (https://github.com/motokimura/spacenet_building_detection/blob/master/LICENSE) Copyright 2017 Motoki Kimura. # +# DM19-0988 # +##################################################################################################################################################################### + + +from PIL import Image +import time +import numpy as np +import pandas as pd +from tqdm import tqdm +import os +import math +import random +import argparse +import logging +import json +from sys import exit +import cv2 +import datetime + +import shapely.wkt +import shapely +from shapely.geometry import Polygon +from collections import defaultdict + +import tensorflow as tf +import keras +from model import * + +from keras import Sequential +from keras.layers import Conv2D, MaxPooling2D, Dense +from model import * + +# Configurations +NUM_WORKERS = 4 +NUM_CLASSES = 4 +BATCH_SIZE = 64 +NUM_EPOCHS = 120 +LEARNING_RATE = 0.0001 +RANDOM_SEED = 123 +LOG_DIR = '/tmp/inference/classification_log_' + datetime.datetime.now().strftime("%Y%m%d-%H%M%S") + + +damage_intensity_encoding = dict() +damage_intensity_encoding[3] = 'destroyed' +damage_intensity_encoding[2] = 'major-damage' +damage_intensity_encoding[1] = 'minor-damage' +damage_intensity_encoding[0] = 'no-damage' + + +### +# Creates data generator for validation set +### +def create_generator(test_df, test_dir, output_json_path): + + gen = keras.preprocessing.image.ImageDataGenerator( + rescale=1.4) + + try: + gen_flow = gen.flow_from_dataframe(dataframe=test_df, + directory=test_dir, + x_col='uuid', + batch_size=BATCH_SIZE, + shuffle=False, + seed=RANDOM_SEED, + class_mode=None, + target_size=(128, 128)) + except: + # No polys detected so write out a blank json + blank = {} + with open(output_json_path , 'w') as outfile: + json.dump(blank, outfile) + exit(0) + + + return gen_flow + +# Runs inference on given test data and pretrained model +def run_inference(test_data, test_csv, model_weights, output_json_path): + + model = generate_xBD_baseline_model() + model.load_weights(model_weights) + + adam = keras.optimizers.Adam(lr=LEARNING_RATE, + beta_1=0.9, + beta_2=0.999, + epsilon=None, + decay=0.0, + amsgrad=False) + + + model.compile(loss=ordinal_loss, optimizer=adam, metrics=['accuracy']) + + df = pd.read_csv(test_csv) + + test_gen = create_generator(df, test_data, output_json_path) + test_gen.reset() + samples = df["uuid"].count() + + steps = np.ceil(samples/BATCH_SIZE) + + tensorboard_callbacks = keras.callbacks.TensorBoard(log_dir=LOG_DIR, histogram_freq=1) + + predictions = model.predict_generator(generator=test_gen, + callbacks=[tensorboard_callbacks], + verbose=1) + + predicted_indices = np.argmax(predictions, axis=1) + predictions_json = dict() + for i in range(samples): + filename_raw = test_gen.filenames[i] + filename = filename_raw.split(".")[0] + predictions_json[filename] = damage_intensity_encoding[predicted_indices[i]] + + with open(output_json_path , 'w') as outfile: + json.dump(predictions_json, outfile) + + +def main(): + + parser = argparse.ArgumentParser(description='Run Building Damage Classification Training & Evaluation') + parser.add_argument('--test_data', + required=True, + metavar="/path/to/xBD_test_dir", + help="Full path to the parent dataset directory") + parser.add_argument('--test_csv', + required=True, + metavar="/path/to/xBD_test_csv", + help="Full path to the parent dataset directory") + parser.add_argument('--model_weights', + default=None, + metavar='/path/to/input_model_weights', + help="Path to input weights") + parser.add_argument('--output_json', + required=True, + metavar="/path/to/output_json") + + args = parser.parse_args() + + run_inference(args.test_data, args.test_csv, args.model_weights, args.output_json) + + +if __name__ == '__main__': + main() diff --git a/model/model.py b/model/model.py new file mode 100644 index 0000000..d72f1f4 --- /dev/null +++ b/model/model.py @@ -0,0 +1,89 @@ +##################################################################################################################################################################### +# xView2 # +# Copyright 2019 Carnegie Mellon University. # +# NO WARRANTY. THIS CARNEGIE MELLON UNIVERSITY AND SOFTWARE ENGINEERING INSTITUTE MATERIAL IS FURNISHED ON AN "AS-IS" BASIS. CARNEGIE MELLON UNIVERSITY MAKES NO # +# WARRANTIES OF ANY KIND, EITHER EXPRESSED OR IMPLIED, AS TO ANY MATTER INCLUDING, BUT NOT LIMITED TO, WARRANTY OF FITNESS FOR PURPOSE OR MERCHANTABILITY, # +# EXCLUSIVITY, OR RESULTS OBTAINED FROM USE OF THE MATERIAL. CARNEGIE MELLON UNIVERSITY DOES NOT MAKE ANY WARRANTY OF ANY KIND WITH RESPECT TO FREEDOM FROM PATENT, # +# TRADEMARK, OR COPYRIGHT INFRINGEMENT. # +# Released under a MIT (SEI)-style license, please see LICENSE.md or contact permission@sei.cmu.edu for full terms. # +# [DISTRIBUTION STATEMENT A] This material has been approved for public release and unlimited distribution. Please see Copyright notice for non-US Government use # +# and distribution. # +# This Software includes and/or makes use of the following Third-Party Software subject to its own license: # +# 1. SpaceNet (https://github.com/motokimura/spacenet_building_detection/blob/master/LICENSE) Copyright 2017 Motoki Kimura. # +# DM19-0988 # +##################################################################################################################################################################### + +from PIL import Image +import time +import numpy as np +import pandas as pd +from tqdm import tqdm +import os +import math +import random +import argparse +import logging +import json +import cv2 +import datetime + +from sklearn.metrics import f1_score +from sklearn.utils.class_weight import compute_class_weight +import shapely.wkt +import shapely +from shapely.geometry import Polygon +from collections import defaultdict + +import tensorflow as tf +import keras +import ast +from keras import Sequential +from keras.layers import Conv2D, MaxPooling2D, Dense, Flatten, Add, Input, Concatenate +from keras.models import Model +from keras.applications.resnet50 import ResNet50 +from keras import backend as K + + +### +# Loss function for ordinal loss from https://github.com/JHart96/keras_ordinal_categorical_crossentropy +### +def ordinal_loss(y_true, y_pred): + weights = K.cast(K.abs(K.argmax(y_true, axis=1) - K.argmax(y_pred, axis=1))/(K.int_shape(y_pred)[1] - 1), dtype='float32') + return (1.0 + weights) * keras.losses.categorical_crossentropy(y_true, y_pred ) + + +### +# Generate a simple CNN +### +def generate_xBD_baseline_model(): + weights = 'imagenet' + inputs = Input(shape=(128, 128, 3)) + + base_model = ResNet50(include_top=False, weights=weights, input_shape=(128, 128, 3)) + + for layer in base_model.layers: + layer.trainable = False + + x = Conv2D(32, (5, 5), strides=(1, 1), padding='same', activation='relu', input_shape=(128, 128, 3))(inputs) + x = MaxPooling2D(pool_size=(2, 2), strides=None, padding='valid', data_format=None)(x) + + x = Conv2D(64, (3, 3), strides=(1, 1), padding='same', activation='relu')(x) + x = MaxPooling2D(pool_size=(2, 2), strides=None, padding='valid', data_format=None)(x) + + x = Conv2D(64, (3, 3), strides=(1, 1), padding='same', activation='relu')(x) + x = MaxPooling2D(pool_size=(2, 2), strides=None, padding='valid', data_format=None)(x) + + x = Flatten()(x) + + base_resnet = base_model(inputs) + base_resnet = Flatten()(base_resnet) + + concated_layers = Concatenate()([x, base_resnet]) + + concated_layers = Dense(2024, activation='relu')(concated_layers) + concated_layers = Dense(524, activation='relu')(concated_layers) + concated_layers = Dense(124, activation='relu')(concated_layers) + output = Dense(4, activation='relu')(concated_layers) + + model = Model(inputs=inputs, outputs=output) + return model diff --git a/model/process_data.py b/model/process_data.py new file mode 100644 index 0000000..fa737b3 --- /dev/null +++ b/model/process_data.py @@ -0,0 +1,178 @@ +##################################################################################################################################################################### +# xView2 # +# Copyright 2019 Carnegie Mellon University. # +# NO WARRANTY. THIS CARNEGIE MELLON UNIVERSITY AND SOFTWARE ENGINEERING INSTITUTE MATERIAL IS FURNISHED ON AN "AS-IS" BASIS. CARNEGIE MELLON UNIVERSITY MAKES NO # +# WARRANTIES OF ANY KIND, EITHER EXPRESSED OR IMPLIED, AS TO ANY MATTER INCLUDING, BUT NOT LIMITED TO, WARRANTY OF FITNESS FOR PURPOSE OR MERCHANTABILITY, # +# EXCLUSIVITY, OR RESULTS OBTAINED FROM USE OF THE MATERIAL. CARNEGIE MELLON UNIVERSITY DOES NOT MAKE ANY WARRANTY OF ANY KIND WITH RESPECT TO FREEDOM FROM PATENT, # +# TRADEMARK, OR COPYRIGHT INFRINGEMENT. # +# Released under a MIT (SEI)-style license, please see LICENSE.md or contact permission@sei.cmu.edu for full terms. # +# [DISTRIBUTION STATEMENT A] This material has been approved for public release and unlimited distribution. Please see Copyright notice for non-US Government use # +# and distribution. # +# This Software includes and/or makes use of the following Third-Party Software subject to its own license: # +# 1. SpaceNet (https://github.com/motokimura/spacenet_building_detection/blob/master/LICENSE) Copyright 2017 Motoki Kimura. # +# DM19-0988 # +##################################################################################################################################################################### + +from PIL import Image +import time +import numpy as np +import pandas as pd +from tqdm import tqdm +import os +import math +import random +import argparse +import logging +import json +import cv2 +import datetime + +import shapely.wkt +import shapely +from shapely.geometry import Polygon +from collections import defaultdict + +logging.basicConfig(level=logging.INFO) + +# Configurations +NUM_WORKERS = 4 +NUM_CLASSES = 4 +BATCH_SIZE = 64 +NUM_EPOCHS = 120 +LEARNING_RATE = 0.0001 +RANDOM_SEED = 123 +LOG_STEP = 150 + +damage_intensity_encoding = defaultdict(lambda: 0) +damage_intensity_encoding['destroyed'] = 3 +damage_intensity_encoding['major-damage'] = 2 +damage_intensity_encoding['minor-damage'] = 1 +damage_intensity_encoding['no-damage'] = 0 + + +def process_img(img_array, polygon_pts, scale_pct): + """Process Raw Data into + + Args: + img_array (numpy array): numpy representation of image. + polygon_pts (array): corners of the building polygon. + + Returns: + numpy array: extracted polygon image from img_array. + + """ + + height, width, _ = img_array.shape + + #Find the four corners of the polygon + xcoords = polygon_pts[:, 0] + ycoords = polygon_pts[:, 1] + xmin, xmax = np.min(xcoords), np.max(xcoords) + ymin, ymax = np.min(ycoords), np.max(ycoords) + + xdiff = xmax - xmin + ydiff = ymax - ymin + + #Extend image by scale percentage + xmin = max(int(xmin - (xdiff * scale_pct)), 0) + xmax = min(int(xmax + (xdiff * scale_pct)), width) + ymin = max(int(ymin - (ydiff * scale_pct)), 0) + ymax = min(int(ymax + (ydiff * scale_pct)), height) + + return img_array[ymin:ymax, xmin:xmax, :] + + +def process_data(input_path, output_path, data_type): + """Process Raw Data into + + Args: + dir_path (path): Path to the xBD dataset. + data_type (string): String to indicate whether to process + train, test, or holdout data. + + Returns: + x_data: A list of numpy arrays representing the images for training + y_data: A list of labels for damage represented in matrix form + + """ + x_data = [] + y_data = [] + + #Generate all the image paths + data_path = os.path.join(input_path, data_type) + disasters = [folder for folder in os.listdir(data_path) if not folder.startswith('.') and ('midwest') not in folder] + disaster_paths = ([data_path + "/" + d + "/images" for d in disasters]) + image_paths = [] + image_paths.extend([(disaster_path + "/" + pic) for pic in os.listdir(disaster_path)] for disaster_path in disaster_paths) + img_paths = np.concatenate(image_paths) + + #Process each image + for img_path in tqdm(img_paths): + + img_obj = Image.open(img_path) + + #Applies histogram equalization to image + img_array = np.array(img_obj) + + #Get corresponding label for the current image + label_path = img_path.replace('png', 'json').replace('images', 'labels') + label_file = open(label_path) + label_data = json.load(label_file) + + #Find all polygons in a given image + for feat in label_data['features']['xy']: + + # only images post-disaster will have damage type + try: + damage_type = feat['properties']['subtype'] + except: # pre-disaster damage is default no-damage, skip it + damage_type = "no-damage" + continue + + poly_uuid = feat['properties']['uid'] + ".png" + + y_data.append(damage_intensity_encoding[damage_type]) + + # Extract the polygon from the points given + polygon_geom = shapely.wkt.loads(feat['wkt']) + polygon_pts = np.array(list(polygon_geom.exterior.coords)) + poly_img = process_img(img_array, polygon_pts, 0.8) + + # Write out the polygon in its own image + output_data_path = os.path.join(output_path, data_type) + cv2.imwrite(output_data_path + "/" + poly_uuid, poly_img) + x_data.append(poly_uuid) + + data_array = {'uuid': x_data, 'labels': y_data} + df = pd.DataFrame(data = data_array) + df.to_csv(data_type + ".csv") + return df + +def main(): + + parser = argparse.ArgumentParser(description='Run Building Damage Classification Training & Evaluation') + parser.add_argument('--input_dir', + required=True, + metavar="/path/to/xBD_input", + help="Full path to the parent dataset directory") + parser.add_argument('--output_dir', + required=True, + metavar='/path/to/xBD_output', + help="Path to new directory to save images") + + args = parser.parse_args() + + logging.info("Started Processing for Train Data") + process_data(args.input_dir, args.output_dir, 'train') + logging.info("Finished Processing Train Data") + + logging.info("Started Processing Test Data") + process_data(args.input_dir, args.output_dir, 'test') + logging.info("Finished Processing Test Data") + + logging.info("Started Processing Holdout Data") + process_data(args.input_dir, args.output_dir, 'hold') + logging.info("Finished Processing Holdout Data") + +if __name__ == '__main__': + main() diff --git a/model/process_data_inference.py b/model/process_data_inference.py new file mode 100644 index 0000000..1359883 --- /dev/null +++ b/model/process_data_inference.py @@ -0,0 +1,125 @@ +##################################################################################################################################################################### +# xView2 # +# Copyright 2019 Carnegie Mellon University. # +# NO WARRANTY. THIS CARNEGIE MELLON UNIVERSITY AND SOFTWARE ENGINEERING INSTITUTE MATERIAL IS FURNISHED ON AN "AS-IS" BASIS. CARNEGIE MELLON UNIVERSITY MAKES NO # +# WARRANTIES OF ANY KIND, EITHER EXPRESSED OR IMPLIED, AS TO ANY MATTER INCLUDING, BUT NOT LIMITED TO, WARRANTY OF FITNESS FOR PURPOSE OR MERCHANTABILITY, # +# EXCLUSIVITY, OR RESULTS OBTAINED FROM USE OF THE MATERIAL. CARNEGIE MELLON UNIVERSITY DOES NOT MAKE ANY WARRANTY OF ANY KIND WITH RESPECT TO FREEDOM FROM PATENT, # +# TRADEMARK, OR COPYRIGHT INFRINGEMENT. # +# Released under a MIT (SEI)-style license, please see LICENSE.md or contact permission@sei.cmu.edu for full terms. # +# [DISTRIBUTION STATEMENT A] This material has been approved for public release and unlimited distribution. Please see Copyright notice for non-US Government use # +# and distribution. # +# This Software includes and/or makes use of the following Third-Party Software subject to its own license: # +# 1. SpaceNet (https://github.com/motokimura/spacenet_building_detection/blob/master/LICENSE) Copyright 2017 Motoki Kimura. # +# DM19-0988 # +##################################################################################################################################################################### + +from PIL import Image +import time +import numpy as np +import pandas as pd +from tqdm import tqdm +import os +import math +import random +import argparse +import logging +import json +import cv2 +import datetime + +import shapely.wkt +import shapely +from shapely.geometry import Polygon +from collections import defaultdict + + +def process_img(img_array, polygon_pts, scale_pct): + """Process Raw Data into + + Args: + img_array (numpy array): numpy representation of image. + polygon_pts (array): corners of the building polygon. + + Returns: + numpy array: extracted polygon image from img_array. + + """ + + height, width, _ = img_array.shape + + #Find the four corners of the polygon + xcoords = polygon_pts[:, 0] + ycoords = polygon_pts[:, 1] + xmin, xmax = np.min(xcoords), np.max(xcoords) + ymin, ymax = np.min(ycoords), np.max(ycoords) + + xdiff = xmax - xmin + ydiff = ymax - ymin + + #Extend image by scale percentage + xmin = max(int(xmin - (xdiff * scale_pct)), 0) + xmax = min(int(xmax + (xdiff * scale_pct)), width) + ymin = max(int(ymin - (ydiff * scale_pct)), 0) + ymax = min(int(ymax + (ydiff * scale_pct)), height) + + return img_array[ymin:ymax, xmin:xmax, :] + + +def process_img_poly(img_path, label_path, output_dir, output_csv): + x_data = [] + img_obj = Image.open(img_path) + + #Applies histogram equalization to image + img_array = np.array(img_obj) + #clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8, 8)) + #img_array = clahe.apply(img_array_pre) + + #Get corresponding label for the current image + label_file = open(label_path) + label_data = json.load(label_file) + + #Find all polygons in a given image + for feat in label_data['features']['xy']: + + poly_uuid = feat['properties']['uid'] + ".png" + + # Extract the polygon from the points given + polygon_geom = shapely.wkt.loads(feat['wkt']) + polygon_pts = np.array(list(polygon_geom.exterior.coords)) + poly_img = process_img(img_array, polygon_pts, 0.8) + + # Write out the polygon in its own image + cv2.imwrite(output_dir + "/" + poly_uuid, poly_img) + x_data.append(poly_uuid) + + data_array = {'uuid': x_data} + df = pd.DataFrame(data = data_array) + df.to_csv(output_csv) + +def main(): + + parser = argparse.ArgumentParser(description='Run Building Damage Classification Training & Evaluation') + parser.add_argument('--input_img', + required=True, + metavar="/path/to/xBD_input", + help="Full path to the parent dataset directory") + parser.add_argument('--label_path', + required=True, + metavar="/path/to/xBD_input", + help="Full path to the parent dataset directory") + parser.add_argument('--output_dir', + required=True, + metavar='/path/to/xBD_output', + help="Path to new directory to save images") + parser.add_argument('--output_csv', + required=True, + metavar='/path/to/xBD_output', + help="Path to save the csv file") + + args = parser.parse_args() + + process_img_poly(args.input_img, args.label_path, args.output_dir, args.output_csv) + + +if __name__ == '__main__': + main() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..e02aa04 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,32 @@ +##################################################################################################################################################################### +# xView2 # +# Copyright 2019 Carnegie Mellon University. # +# NO WARRANTY. THIS CARNEGIE MELLON UNIVERSITY AND SOFTWARE ENGINEERING INSTITUTE MATERIAL IS FURNISHED ON AN "AS-IS" BASIS. CARNEGIE MELLON UNIVERSITY MAKES NO # +# WARRANTIES OF ANY KIND, EITHER EXPRESSED OR IMPLIED, AS TO ANY MATTER INCLUDING, BUT NOT LIMITED TO, WARRANTY OF FITNESS FOR PURPOSE OR MERCHANTABILITY, # +# EXCLUSIVITY, OR RESULTS OBTAINED FROM USE OF THE MATERIAL. CARNEGIE MELLON UNIVERSITY DOES NOT MAKE ANY WARRANTY OF ANY KIND WITH RESPECT TO FREEDOM FROM PATENT, # +# TRADEMARK, OR COPYRIGHT INFRINGEMENT. # +# Released under a MIT (SEI)-style license, please see LICENSE.md or contact permission@sei.cmu.edu for full terms. # +# [DISTRIBUTION STATEMENT A] This material has been approved for public release and unlimited distribution. Please see Copyright notice for non-US Government use # +# and distribution. # +# This Software includes and/or makes use of the following Third-Party Software subject to its own license: # +# 1. SpaceNet (https://github.com/motokimura/spacenet_building_detection/blob/master/LICENSE) Copyright 2017 Motoki Kimura. # +# DM19-0988 # +##################################################################################################################################################################### +numpy +matplotlib +tqdm +libtiff +scipy +Pillow +scikit-image +opencv-python +imgaug +IPython +geopandas +keras +imantics +simplification +scikit-learn +chainer +tensorboard +tensorboardX diff --git a/spacenet/inference/inference.py b/spacenet/inference/inference.py new file mode 100644 index 0000000..99866b0 --- /dev/null +++ b/spacenet/inference/inference.py @@ -0,0 +1,161 @@ +##################################################################################################################################################################### +# xView2 # +# Copyright 2019 Carnegie Mellon University. # +# NO WARRANTY. THIS CARNEGIE MELLON UNIVERSITY AND SOFTWARE ENGINEERING INSTITUTE MATERIAL IS FURNISHED ON AN "AS-IS" BASIS. CARNEGIE MELLON UNIVERSITY MAKES NO # +# WARRANTIES OF ANY KIND, EITHER EXPRESSED OR IMPLIED, AS TO ANY MATTER INCLUDING, BUT NOT LIMITED TO, WARRANTY OF FITNESS FOR PURPOSE OR MERCHANTABILITY, # +# EXCLUSIVITY, OR RESULTS OBTAINED FROM USE OF THE MATERIAL. CARNEGIE MELLON UNIVERSITY DOES NOT MAKE ANY WARRANTY OF ANY KIND WITH RESPECT TO FREEDOM FROM PATENT, # +# TRADEMARK, OR COPYRIGHT INFRINGEMENT. # +# Released under a MIT (SEI)-style license, please see LICENSE.md or contact permission@sei.cmu.edu for full terms. # +# [DISTRIBUTION STATEMENT A] This material has been approved for public release and unlimited distribution. Please see Copyright notice for non-US Government use # +# and distribution. # +# This Software includes and/or makes use of the following Third-Party Software subject to its own license: # +# 1. SpaceNet (https://github.com/motokimura/spacenet_building_detection/blob/master/LICENSE) Copyright 2017 Motoki Kimura. # +# DM19-0988 # +##################################################################################################################################################################### + +import resource +import sys +sys.path.append('../src/models') + +from segmentation_cpu import SegmentationModel as Model +from os import path +from PIL import Image +import numpy as np +from uuid import uuid4 +import json +from imantics import Polygons, Mask +from simplification.cutil import simplify_coords_vwp + +def create_wkt(polygon): + """ + :param polygon: a single polygon in the format [(x1,y1), (x2,y2), ...] + :returns: a wkt formatted string ready to be put into the json + """ + wkt = 'POLYGON ((' + + for coords in polygon: + wkt += "{} {},".format(coords[0], coords[1]) + + wkt = wkt[:-1] + '))' + + return wkt + + +def create_json(adjusted_polygons): + """ + :param polygons: list of polygons in the format [(x1,y1), (x2,y2), ...] + :returns: json with found and adjusted polygon pixel x,y values in WKT format + """ + # Create a blank json that matched the labeler provided jsons with null or default values + output_json = { + "features": { + "lng_lat": [], + "xy": [] + }, + "metadata": { + "sensor": "", + "provider_asset_type": "", + "gsd": 0, + "capture_date": "", + "off_nadir_angle": 0, + "pan_resolution": 0, + "sun_azimuth": 0, + "sun_elevation": 0, + "target_azimuth": 0, + "disaster": "", + "disaster_type": "", + "catalog_id": "", + "original_width": 0, + "original_height": 0, + "width": 0, + "height": 0, + "id": "", + "img_name": "" + } + } + + # Using a lambda function to place the WKT string in the list of polygons + polygon_template = lambda poly, uuid: { + 'properties': { + 'feature_type': 'building', + 'uid': uuid + }, + 'wkt': poly + } + + # For each adjusted polygon add the wkt for the polygon points + for polygon in adjusted_polygons: + wkt = create_wkt(polygon) + uuid = gen_uuid() + poly = polygon_template(wkt, uuid) + output_json['features']['xy'].append(poly) + + return output_json + +def gen_uuid(): + return str(uuid4()) + +def inference(image, score, output_file): + building_score = score[1] + + building_mask_pred = (np.argmax(score, axis=0) == 1) + polygons = Mask(building_mask_pred).polygons() + + new_predictions = [] + + for poly in polygons: + if len(poly) >= 3: + f = poly.reshape(-1, 2) + simplified_vw = simplify_coords_vwp(f, .3) + if len(simplified_vw) > 2: + mpoly = [] + # Rebuilding the polygon in the way that PIL expects the values [(x1,y1),(x2,y2)] + for i in simplified_vw: + mpoly.append((i[0], i[1])) + # Adding the first point to the last to close the polygon + mpoly.append((simplified_vw[0][0], simplified_vw[0][1])) + new_predictions.append(mpoly) + + # Creating the json with the predicted and then adjusted polygons + output_json = create_json(new_predictions) + + with open(output_file, 'w') as out_file: + json.dump(output_json, out_file) + +if __name__ == "__main__": + import argparse + + # Parse command line arguments + parser = argparse.ArgumentParser( + description= + """inference.py: takes an image and creates inferred polygons json off the VW algorithm and the unet model predictions""" + ) + parser.add_argument('--input', + required=True, + metavar='/path/to/input/image.png') + parser.add_argument( + '--weights', + required=True, + metavar='/full/path/to/mode_iter_XXXX', + help="Must be the output to a unet model weights trained for xView2" + ) + parser.add_argument( + '--mean', + required=True, + metavar='/full/path/to/mean.npy', + help="a numpy data structure file that is the mean of the training images (found by running ./src/features/compute_mean.py)" + ) + parser.add_argument('--output', + required=True, + metavar="/path/to/output/file.json") + args = parser.parse_args() + + # Load trained model + # Modify the paths based on your trained model location if needed. + mean = np.load(args.mean) + model = Model(args.weights, mean) + + image = np.array(Image.open(args.input)) + score = model.apply_segmentation(image) + inference(image, score, args.output) + diff --git a/spacenet/src/features/build_labels.py b/spacenet/src/features/build_labels.py new file mode 100644 index 0000000..48051cc --- /dev/null +++ b/spacenet/src/features/build_labels.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +##################################################################################################################################################################### +# xView2 # +# Copyright 2019 Carnegie Mellon University. # +# NO WARRANTY. THIS CARNEGIE MELLON UNIVERSITY AND SOFTWARE ENGINEERING INSTITUTE MATERIAL IS FURNISHED ON AN "AS-IS" BASIS. CARNEGIE MELLON UNIVERSITY MAKES NO # +# WARRANTIES OF ANY KIND, EITHER EXPRESSED OR IMPLIED, AS TO ANY MATTER INCLUDING, BUT NOT LIMITED TO, WARRANTY OF FITNESS FOR PURPOSE OR MERCHANTABILITY, # +# EXCLUSIVITY, OR RESULTS OBTAINED FROM USE OF THE MATERIAL. CARNEGIE MELLON UNIVERSITY DOES NOT MAKE ANY WARRANTY OF ANY KIND WITH RESPECT TO FREEDOM FROM PATENT, # +# TRADEMARK, OR COPYRIGHT INFRINGEMENT. # +# Released under a MIT (SEI)-style license, please see LICENSE.md or contact permission@sei.cmu.edu for full terms. # +# [DISTRIBUTION STATEMENT A] This material has been approved for public release and unlimited distribution. Please see Copyright notice for non-US Government use # +# and distribution. # +# This Software includes and/or makes use of the following Third-Party Software subject to its own license: # +# 1. SpaceNet (https://github.com/motokimura/spacenet_building_detection/blob/master/LICENSE) Copyright 2017 Motoki Kimura. # +# DM19-0988 # +##################################################################################################################################################################### + +import argparse +import os +from tqdm import tqdm + +import sys +sys.path.append("spacenet_lib") + +from create_poly_mask import create_poly_mask + + +def build_labels(src_raster_dir, src_vector_dir, dst_dir): + + os.makedirs(dst_dir, exist_ok=True) + + file_count = len([f for f in os.walk(src_vector_dir).__next__()[2] if f[-8:] == ".geojson"]) + + print("[INFO] Found {} geojson files. Preparing building mask images...".format(file_count)) + + for idx in tqdm(range(1, file_count + 1)): + + src_raster_filename = "3band_AOI_1_RIO_img{}.tif".format(idx) + src_vector_filename = "Geo_AOI_1_RIO_img{}.geojson".format(idx) + + src_raster_path = os.path.join(src_raster_dir, src_raster_filename) + src_vector_path = os.path.join(src_vector_dir, src_vector_filename) + dst_path = os.path.join(dst_dir, src_raster_filename) + + create_poly_mask( + src_raster_path, src_vector_path, npDistFileName=dst_path, + noDataValue=0, burn_values=255 + ) + + +if __name__ == "__main__": + + parser = argparse.ArgumentParser() + + parser.add_argument('src_raster_dir', help='Root directory for raster files (.tif)') + parser.add_argument('src_vector_dir', help='Root directory for vector files (.geojson)') + parser.add_argument('dst_dir', help='Output directory') + + args = parser.parse_args() + + build_labels(args.src_raster_dir, args.src_vector_dir, args.dst_dir) diff --git a/spacenet/src/features/compute_mean.py b/spacenet/src/features/compute_mean.py new file mode 100644 index 0000000..ba741d0 --- /dev/null +++ b/spacenet/src/features/compute_mean.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python + +##################################################################################################################################################################### +# xView2 # +# Copyright 2019 Carnegie Mellon University. # +# NO WARRANTY. THIS CARNEGIE MELLON UNIVERSITY AND SOFTWARE ENGINEERING INSTITUTE MATERIAL IS FURNISHED ON AN "AS-IS" BASIS. CARNEGIE MELLON UNIVERSITY MAKES NO # +# WARRANTIES OF ANY KIND, EITHER EXPRESSED OR IMPLIED, AS TO ANY MATTER INCLUDING, BUT NOT LIMITED TO, WARRANTY OF FITNESS FOR PURPOSE OR MERCHANTABILITY, # +# EXCLUSIVITY, OR RESULTS OBTAINED FROM USE OF THE MATERIAL. CARNEGIE MELLON UNIVERSITY DOES NOT MAKE ANY WARRANTY OF ANY KIND WITH RESPECT TO FREEDOM FROM PATENT, # +# TRADEMARK, OR COPYRIGHT INFRINGEMENT. # +# Released under a MIT (SEI)-style license, please see LICENSE.md or contact permission@sei.cmu.edu for full terms. # +# [DISTRIBUTION STATEMENT A] This material has been approved for public release and unlimited distribution. Please see Copyright notice for non-US Government use # +# and distribution. # +# This Software includes and/or makes use of the following Third-Party Software subject to its own license: # +# 1. SpaceNet (https://github.com/motokimura/spacenet_building_detection/blob/master/LICENSE) Copyright 2017 Motoki Kimura. # +# DM19-0988 # +##################################################################################################################################################################### + +import argparse +import sys +import os + +import six +import numpy as np + +try: + from PIL import Image + available = True +except ImportError as e: + available = False + _import_error = e + +from chainer.dataset import dataset_mixin + + +def _check_pillow_availability(): + if not available: + raise ImportError('PIL cannot be loaded. Install Pillow!\n' + 'The actual import error is as follows:\n' + + str(_import_error)) + + +def _read_image_as_array(path, dtype): + f = Image.open(path) + try: + image = np.asarray(f, dtype=dtype) + finally: + # Only pillow >= 3.0 has 'close' method + if hasattr(f, 'close'): + f.close() + return image + + +class ImageDataset(dataset_mixin.DatasetMixin): + + def __init__(self, paths, root='.', dtype=np.float32): + _check_pillow_availability() + if isinstance(paths, six.string_types): + with open(paths) as paths_file: + paths = [path.rstrip() for path in paths_file] + self._paths = paths + self._root = root + self._dtype = dtype + + def __len__(self): + return len(self._paths) + + def get_example(self, i): + path = os.path.join(self._root, self._paths[i]) + image = _read_image_as_array(path, self._dtype) + + if image.ndim == 2: + # image is greyscale + image = image[:, :, np.newaxis] + return image.transpose(2, 0, 1) + + +def compute_mean(dataset): + print('compute mean image') + sum_color = 0 + N = len(dataset) + for i, image in enumerate(dataset): + sum_color += image.mean(axis=2, keepdims=False).mean(axis=1, keepdims=False) + sys.stderr.write('{} / {}\r'.format(i, N)) + sys.stderr.flush() + sys.stderr.write('\n') + return sum_color / N + + +if __name__ == '__main__': + + parser = argparse.ArgumentParser(description='Compute images mean array') + + parser.add_argument('dataset', + help='Path to training image-label list file') + parser.add_argument('--root', '-R', default='.', + help='Root directory path of image files') + parser.add_argument('--output', '-o', default='mean.npy', + help='path to output mean array') + args = parser.parse_args() + + dataset = ImageDataset(args.dataset, args.root) + mean = compute_mean(dataset) + + np.save(args.output, mean) + diff --git a/spacenet/src/features/split_dataset.py b/spacenet/src/features/split_dataset.py new file mode 100644 index 0000000..9babff0 --- /dev/null +++ b/spacenet/src/features/split_dataset.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +##################################################################################################################################################################### +# xView2 # +# Copyright 2019 Carnegie Mellon University. # +# NO WARRANTY. THIS CARNEGIE MELLON UNIVERSITY AND SOFTWARE ENGINEERING INSTITUTE MATERIAL IS FURNISHED ON AN "AS-IS" BASIS. CARNEGIE MELLON UNIVERSITY MAKES NO # +# WARRANTIES OF ANY KIND, EITHER EXPRESSED OR IMPLIED, AS TO ANY MATTER INCLUDING, BUT NOT LIMITED TO, WARRANTY OF FITNESS FOR PURPOSE OR MERCHANTABILITY, # +# EXCLUSIVITY, OR RESULTS OBTAINED FROM USE OF THE MATERIAL. CARNEGIE MELLON UNIVERSITY DOES NOT MAKE ANY WARRANTY OF ANY KIND WITH RESPECT TO FREEDOM FROM PATENT, # +# TRADEMARK, OR COPYRIGHT INFRINGEMENT. # +# Released under a MIT (SEI)-style license, please see LICENSE.md or contact permission@sei.cmu.edu for full terms. # +# [DISTRIBUTION STATEMENT A] This material has been approved for public release and unlimited distribution. Please see Copyright notice for non-US Government use # +# and distribution. # +# This Software includes and/or makes use of the following Third-Party Software subject to its own license: # +# 1. SpaceNet (https://github.com/motokimura/spacenet_building_detection/blob/master/LICENSE) Copyright 2017 Motoki Kimura. # +# DM19-0988 # +##################################################################################################################################################################### + +import argparse +import os +import random +from tqdm import tqdm + + +def dump_filenames(filenames, dst_path): + + with open(dst_path, 'w') as f: + + for i, filename in enumerate(filenames): + if i != 0: + f.write("\n") + + f.write(filename) + + +def split_dataset(img_dir, dst_dir, ratio, seed=0): + + filenames = os.listdir(img_dir) + + random.seed(seed) + random.shuffle(filenames) + + file_count = len(filenames) + + train_ratio, val_ratio, test_ratio = ratio + total = train_ratio + val_ratio + test_ratio + + train_count= int(float(file_count * train_ratio) / float(total)) + val_count = int(float(file_count * val_ratio) / float(total)) + + train_files = filenames[:train_count] + val_files = filenames[train_count:train_count + val_count] + test_files = filenames[train_count + val_count:] + + dump_filenames(train_files, os.path.join(dst_dir, "train.txt")) + dump_filenames(val_files, os.path.join(dst_dir, "val.txt")) + dump_filenames(test_files, os.path.join(dst_dir, "test.txt")) + + +if __name__ == "__main__": + + parser = argparse.ArgumentParser() + + parser.add_argument('img_dir', help='Root directory for building mask images (.tif)') + parser.add_argument('dst_dir', help='Root directory to output train.txt, val.txt, and test.txt') + parser.add_argument('--ratio', help='Split ratio for train/val/test set', + type=int, nargs=3, default=[7, 1, 2]) + parser.add_argument('--seed', help='random seed', + type=int, default=0) + + args = parser.parse_args() + + split_dataset(args.img_dir, args.dst_dir, args.ratio, args.seed) diff --git a/spacenet/src/models/dataset.py b/spacenet/src/models/dataset.py new file mode 100644 index 0000000..bdb3db6 --- /dev/null +++ b/spacenet/src/models/dataset.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python + +##################################################################################################################################################################### +# xView2 # +# Copyright 2019 Carnegie Mellon University. # +# NO WARRANTY. THIS CARNEGIE MELLON UNIVERSITY AND SOFTWARE ENGINEERING INSTITUTE MATERIAL IS FURNISHED ON AN "AS-IS" BASIS. CARNEGIE MELLON UNIVERSITY MAKES NO # +# WARRANTIES OF ANY KIND, EITHER EXPRESSED OR IMPLIED, AS TO ANY MATTER INCLUDING, BUT NOT LIMITED TO, WARRANTY OF FITNESS FOR PURPOSE OR MERCHANTABILITY, # +# EXCLUSIVITY, OR RESULTS OBTAINED FROM USE OF THE MATERIAL. CARNEGIE MELLON UNIVERSITY DOES NOT MAKE ANY WARRANTY OF ANY KIND WITH RESPECT TO FREEDOM FROM PATENT, # +# TRADEMARK, OR COPYRIGHT INFRINGEMENT. # +# Released under a MIT (SEI)-style license, please see LICENSE.md or contact permission@sei.cmu.edu for full terms. # +# [DISTRIBUTION STATEMENT A] This material has been approved for public release and unlimited distribution. Please see Copyright notice for non-US Government use # +# and distribution. # +# This Software includes and/or makes use of the following Third-Party Software subject to its own license: # +# 1. SpaceNet (https://github.com/motokimura/spacenet_building_detection/blob/master/LICENSE) Copyright 2017 Motoki Kimura. # +# DM19-0988 # +##################################################################################################################################################################### + +import os +import numpy as np +import random + +try: + from PIL import Image + available = True +except ImportError as e: + available = False + _import_error = e +import six + +from chainer.dataset import dataset_mixin + +from transforms import random_color_distort + + +def _check_pillow_availability(): + if not available: + raise ImportError('PIL cannot be loaded. Install Pillow!\n' + 'The actual import error is as follows:\n' + + str(_import_error)) + + +def _read_label_image_as_array(path, dtype): + f = Image.open(path) + f = f.convert('1') + try: + image = np.asarray(f, dtype=dtype) + finally: + # Only pillow >= 3.0 has 'close' method + if hasattr(f, 'close'): + f.close() + return image + + +def _read_image_as_array(path, dtype): + f = Image.open(path) + try: + image = np.asarray(f, dtype=dtype) + finally: + # Only pillow >= 3.0 has 'close' method + if hasattr(f, 'close'): + f.close() + return image + +class LabeledImageDataset(dataset_mixin.DatasetMixin): + def __init__(self, dataset, root, label_root, dtype=np.float32, + label_dtype=np.int32, mean=0, crop_size=256, test=False, + distort=False): + _check_pillow_availability() + if isinstance(dataset, six.string_types): + dataset_path = dataset + with open(dataset_path) as f: + pairs = [] + for i, line in enumerate(f): + line = line.rstrip('\n') + image_filename = line + label_filename = line + pairs.append((image_filename, label_filename)) + self._pairs = pairs + self._root = root + self._label_root = label_root + self._dtype = dtype + self._label_dtype = label_dtype + self._mean = mean[np.newaxis, np.newaxis, :] + self._crop_size = crop_size + self._test = test + self._distort = distort + + def __len__(self): + return len(self._pairs) + + def get_example(self, i): + image_filename, label_filename = self._pairs[i] + + image_path = os.path.join(self._root, image_filename) + image = _read_image_as_array(image_path, self._dtype) + if self._distort: + image = random_color_distort(image) + image = np.asarray(image, dtype=self._dtype) + + image = (image - self._mean) / 255.0 + + label_path = os.path.join(self._label_root, label_filename) + label_image = _read_label_image_as_array(label_path, self._label_dtype) + + h, w, _ = image.shape + + label = np.zeros(shape=[h, w], dtype=np.int32) # 0: background + label[label_image > 0] = 1 # 1: "building" + + # Padding + if (h < self._crop_size) or (w < self._crop_size): + H, W = max(h, self._crop_size), max(w, self._crop_size) + + pad_y1, pad_x1 = (H - h) // 2, (W - w) // 2 + pad_y2, pad_x2 = (H - h - pad_y1), (W - w - pad_x1) + image = np.pad(image, ((pad_y1, pad_y2), (pad_x1, pad_x2), (0, 0)), 'symmetric') + + if self._test: + # Pad with ignore_value for test set + label = np.pad(label, ((pad_y1, pad_y2), (pad_x1, pad_x2)), 'constant', constant_values=255) + else: + # Pad with original label for train set + label = np.pad(label, ((pad_y1, pad_y2), (pad_x1, pad_x2)), 'symmetric') + + h, w = H, W + + # Randomly flip and crop the image/label for train-set + if not self._test: + + # Horizontal flip + if random.randint(0, 1): + image = image[:, ::-1, :] + label = label[:, ::-1] + + # Vertical flip + if random.randint(0, 1): + image = image[::-1, :, :] + label = label[::-1, :] + + # Random crop + top = random.randint(0, h - self._crop_size) + left = random.randint(0, w - self._crop_size) + + # Crop the center for test-set + else: + top = (h - self._crop_size) // 2 + left = (w - self._crop_size) // 2 + + bottom = top + self._crop_size + right = left + self._crop_size + + image = image[top:bottom, left:right] + label = label[top:bottom, left:right] + + return image.transpose(2, 0, 1), label diff --git a/spacenet/src/models/evaluate_model.py b/spacenet/src/models/evaluate_model.py new file mode 100644 index 0000000..48741bc --- /dev/null +++ b/spacenet/src/models/evaluate_model.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python + +##################################################################################################################################################################### +# xView2 # +# Copyright 2019 Carnegie Mellon University. # +# NO WARRANTY. THIS CARNEGIE MELLON UNIVERSITY AND SOFTWARE ENGINEERING INSTITUTE MATERIAL IS FURNISHED ON AN "AS-IS" BASIS. CARNEGIE MELLON UNIVERSITY MAKES NO # +# WARRANTIES OF ANY KIND, EITHER EXPRESSED OR IMPLIED, AS TO ANY MATTER INCLUDING, BUT NOT LIMITED TO, WARRANTY OF FITNESS FOR PURPOSE OR MERCHANTABILITY, # +# EXCLUSIVITY, OR RESULTS OBTAINED FROM USE OF THE MATERIAL. CARNEGIE MELLON UNIVERSITY DOES NOT MAKE ANY WARRANTY OF ANY KIND WITH RESPECT TO FREEDOM FROM PATENT, # +# TRADEMARK, OR COPYRIGHT INFRINGEMENT. # +# Released under a MIT (SEI)-style license, please see LICENSE.md or contact permission@sei.cmu.edu for full terms. # +# [DISTRIBUTION STATEMENT A] This material has been approved for public release and unlimited distribution. Please see Copyright notice for non-US Government use # +# and distribution. # +# This Software includes and/or makes use of the following Third-Party Software subject to its own license: # +# 1. SpaceNet (https://github.com/motokimura/spacenet_building_detection/blob/master/LICENSE) Copyright 2017 Motoki Kimura. # +# DM19-0988 # +##################################################################################################################################################################### + +# from https://github.com/chainer/chainercv/blob/master/chainercv/evaluations/eval_semantic_segmentation.py + +from __future__ import division + +import numpy as np +import six + + +def calc_semantic_segmentation_confusion(pred_labels, gt_labels): + """Collect a confusion matrix. + The number of classes :math:`n\_class` is + :math:`max(pred\_labels, gt\_labels) + 1`, which is + the maximum class id of the inputs added by one. + Args: + pred_labels (iterable of numpy.ndarray): A collection of predicted + labels. The shape of a label array + is :math:`(H, W)`. :math:`H` and :math:`W` + are height and width of the label. + gt_labels (iterable of numpy.ndarray): A collection of ground + truth labels. The shape of a ground truth label array is + :math:`(H, W)`, and its corresponding prediction label should + have the same shape. + A pixel with value :obj:`-1` will be ignored during evaluation. + Returns: + numpy.ndarray: + A confusion matrix. Its shape is :math:`(n\_class, n\_class)`. + The :math:`(i, j)` th element corresponds to the number of pixels + that are labeled as class :math:`i` by the ground truth and + class :math:`j` by the prediction. + """ + pred_labels = iter(pred_labels) + gt_labels = iter(gt_labels) + + n_class = 0 + confusion = np.zeros((n_class, n_class), dtype=np.int64) + for pred_label, gt_label in six.moves.zip(pred_labels, gt_labels): + if pred_label.ndim != 2 or gt_label.ndim != 2: + raise ValueError('ndim of labels should be two.') + if pred_label.shape != gt_label.shape: + raise ValueError('Shape of ground truth and prediction should' + ' be same.') + pred_label = pred_label.flatten() + gt_label = gt_label.flatten() + + # Dynamically expand the confusion matrix if necessary. + lb_max = np.max((pred_label, gt_label)) + if lb_max >= n_class: + expanded_confusion = np.zeros( + (lb_max + 1, lb_max + 1), dtype=np.int64) + expanded_confusion[0:n_class, 0:n_class] = confusion + + n_class = lb_max + 1 + confusion = expanded_confusion + + # Count statistics from valid pixels. + mask = gt_label >= 0 + confusion += np.bincount( + n_class * gt_label[mask].astype(int) + + pred_label[mask], minlength=n_class**2).reshape((n_class, n_class)) + + for iter_ in (pred_labels, gt_labels): + # This code assumes any iterator does not contain None as its items. + if next(iter_, None) is not None: + raise ValueError('Length of input iterables need to be same') + return confusion + + +def calc_semantic_segmentation_iou(confusion): + """Calculate Intersection over Union with a given confusion matrix. + The definition of Intersection over Union (IoU) is as follows, + where :math:`N_{ij}` is the number of pixels + that are labeled as class :math:`i` by the ground truth and + class :math:`j` by the prediction. + * :math:`\\text{IoU of the i-th class} = \ + \\frac{N_{ii}}{\\sum_{j=1}^k N_{ij} + \\sum_{j=1}^k N_{ji} - N_{ii}}` + Args: + confusion (numpy.ndarray): A confusion matrix. Its shape is + :math:`(n\_class, n\_class)`. + The :math:`(i, j)` th element corresponds to the number of pixels + that are labeled as class :math:`i` by the ground truth and + class :math:`j` by the prediction. + Returns: + numpy.ndarray: + An array of IoUs for the :math:`n\_class` classes. Its shape is + :math:`(n\_class,)`. + """ + iou_denominator = (confusion.sum(axis=1) + confusion.sum(axis=0) - + np.diag(confusion)) + iou = np.diag(confusion) / iou_denominator + return iou + + +def eval_semantic_segmentation(pred_labels, gt_labels): + """Evaluate metrics used in Semantic Segmentation. + This function calculates Intersection over Union (IoU), Pixel Accuracy + and Class Accuracy for the task of semantic segmentation. + The definition of metrics calculated by this function is as follows, + where :math:`N_{ij}` is the number of pixels + that are labeled as class :math:`i` by the ground truth and + class :math:`j` by the prediction. + * :math:`\\text{IoU of the i-th class} = \ + \\frac{N_{ii}}{\\sum_{j=1}^k N_{ij} + \\sum_{j=1}^k N_{ji} - N_{ii}}` + * :math:`\\text{mIoU} = \\frac{1}{k} \ + \\sum_{i=1}^k \ + \\frac{N_{ii}}{\\sum_{j=1}^k N_{ij} + \\sum_{j=1}^k N_{ji} - N_{ii}}` + * :math:`\\text{Pixel Accuracy} = \ + \\frac \ + {\\sum_{i=1}^k N_{ii}} \ + {\\sum_{i=1}^k \\sum_{j=1}^k N_{ij}}` + * :math:`\\text{Class Accuracy} = \ + \\frac{N_{ii}}{\\sum_{j=1}^k N_{ij}}` + * :math:`\\text{Mean Class Accuracy} = \\frac{1}{k} \ + \\sum_{i=1}^k \ + \\frac{N_{ii}}{\\sum_{j=1}^k N_{ij}}` + The more detailed description of the above metrics can be found in a + review on semantic segmentation [#]_. + The number of classes :math:`n\_class` is + :math:`max(pred\_labels, gt\_labels) + 1`, which is + the maximum class id of the inputs added by one. + .. [#] Alberto Garcia-Garcia, Sergio Orts-Escolano, Sergiu Oprea, \ + Victor Villena-Martinez, Jose Garcia-Rodriguez. \ + `A Review on Deep Learning Techniques Applied to Semantic Segmentation \ + `_. arXiv 2017. + Args: + pred_labels (iterable of numpy.ndarray): A collection of predicted + labels. The shape of a label array + is :math:`(H, W)`. :math:`H` and :math:`W` + are height and width of the label. + For example, this is a list of labels + :obj:`[label_0, label_1, ...]`, where + :obj:`label_i.shape = (H_i, W_i)`. + gt_labels (iterable of numpy.ndarray): A collection of ground + truth labels. The shape of a ground truth label array is + :math:`(H, W)`, and its corresponding prediction label should + have the same shape. + A pixel with value :obj:`-1` will be ignored during evaluation. + Returns: + dict: + The keys, value-types and the description of the values are listed + below. + * **iou** (*numpy.ndarray*): An array of IoUs for the \ + :math:`n\_class` classes. Its shape is :math:`(n\_class,)`. + * **miou** (*float*): The average of IoUs over classes. + * **pixel_accuracy** (*float*): The computed pixel accuracy. + * **class_accuracy** (*numpy.ndarray*): An array of class accuracies \ + for the :math:`n\_class` classes. \ + Its shape is :math:`(n\_class,)`. + * **mean_class_accuracy** (*float*): The average of class accuracies. + """ + # Evaluation code is based on + # https://github.com/shelhamer/fcn.berkeleyvision.org/blob/master/ + # score.py#L37 + confusion = calc_semantic_segmentation_confusion( + pred_labels, gt_labels) + iou = calc_semantic_segmentation_iou(confusion) + pixel_accuracy = np.diag(confusion).sum() / confusion.sum() + class_accuracy = np.diag(confusion) / np.sum(confusion, axis=1) + + return {'iou': iou, 'miou': np.nanmean(iou), + 'pixel_accuracy': pixel_accuracy, + 'class_accuracy': class_accuracy, + 'mean_class_accuracy': np.nanmean(class_accuracy)} diff --git a/spacenet/src/models/segmentation.py b/spacenet/src/models/segmentation.py new file mode 100644 index 0000000..650af99 --- /dev/null +++ b/spacenet/src/models/segmentation.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python + +##################################################################################################################################################################### +# xView2 # +# Copyright 2019 Carnegie Mellon University. # +# NO WARRANTY. THIS CARNEGIE MELLON UNIVERSITY AND SOFTWARE ENGINEERING INSTITUTE MATERIAL IS FURNISHED ON AN "AS-IS" BASIS. CARNEGIE MELLON UNIVERSITY MAKES NO # +# WARRANTIES OF ANY KIND, EITHER EXPRESSED OR IMPLIED, AS TO ANY MATTER INCLUDING, BUT NOT LIMITED TO, WARRANTY OF FITNESS FOR PURPOSE OR MERCHANTABILITY, # +# EXCLUSIVITY, OR RESULTS OBTAINED FROM USE OF THE MATERIAL. CARNEGIE MELLON UNIVERSITY DOES NOT MAKE ANY WARRANTY OF ANY KIND WITH RESPECT TO FREEDOM FROM PATENT, # +# TRADEMARK, OR COPYRIGHT INFRINGEMENT. # +# Released under a MIT (SEI)-style license, please see LICENSE.md or contact permission@sei.cmu.edu for full terms. # +# [DISTRIBUTION STATEMENT A] This material has been approved for public release and unlimited distribution. Please see Copyright notice for non-US Government use # +# and distribution. # +# This Software includes and/or makes use of the following Third-Party Software subject to its own license: # +# 1. SpaceNet (https://github.com/motokimura/spacenet_building_detection/blob/master/LICENSE) Copyright 2017 Motoki Kimura. # +# DM19-0988 # +##################################################################################################################################################################### + +import numpy as np +import cv2 +import math + +import chainer +import chainer.functions as F +from chainer import cuda, serializers, Variable + +from unet import UNet + + +class SegmentationModel: + + def __init__(self, model_path, mean, gpu=0): + + # Load model + self.__model = UNet() + serializers.load_npz(model_path, self.__model) + + chainer.cuda.get_device(gpu).use() + self.__model.to_gpu(gpu) + + # Add height and width dimensions to mean + self.__mean = mean[np.newaxis, np.newaxis, :] + + + def apply_segmentation(self, image): + + image_in, crop = self.__preprocess(image) + + with chainer.using_config('train', False): + score = self.__model.forward(image_in) + + score = F.softmax(score) + score = cuda.to_cpu(score.data)[0] + + top, left, bottom, right = crop + score = score[:, top:bottom, left:right] + + return score + + + def apply_segmentation_to_mosaic(self, mosaic, grid_px=800, tile_overlap_px=200): + + h, w, _ = mosaic.shape + + assert ((grid_px + tile_overlap_px * 2) % 16 == 0), "(grid_px + tile_overlap_px * 2) must be divisible by 16" + + pad_y1 = tile_overlap_px + pad_x1 = tile_overlap_px + + n_y = int(float(h) / float(grid_px)) + n_x = int(float(w) / float(grid_px)) + pad_y2 = n_y * grid_px + 2 * tile_overlap_px - h - pad_y1 + pad_x2 = n_x * grid_px + 2 * tile_overlap_px - h - pad_x1 + + mosaic_padded = np.pad(mosaic, ((pad_y1, pad_y2), (pad_x1, pad_x2), (0, 0)), 'symmetric') + + H, W, _ = mosaic_padded.shape + score_padded = np.zeros(shape=[self.__model.class_num, H, W], dtype=np.float32) + + for yi in range(n_y): + for xi in range(n_x): + + top = yi * grid_px + left = xi * grid_px + bottom = top + grid_px + 2 * tile_overlap_px + right = left + grid_px + 2 * tile_overlap_px + + tile = mosaic_padded[top:bottom, left:right] + + score_tile = self.apply_segmentation(tile) + + score_padded[:, top:bottom, left:right] = score_tile + + score = score_padded[:, pad_y1:-pad_y2, pad_x1:-pad_x2] + + return score + + + def __preprocess(self, image): + + h, w, _ = image.shape + h_padded = int(math.ceil(float(h) / 16.0) * 16) + w_padded = int(math.ceil(float(w) / 16.0) * 16) + + pad_y1 = (h_padded - h) // 2 + pad_x1 = (w_padded - w) // 2 + pad_y2 = h_padded - h - pad_y1 + pad_x2 = w_padded - w - pad_x1 + + image_padded = np.pad(image, ((pad_y1, pad_y2), (pad_x1, pad_x2), (0, 0)), 'symmetric') + image_in = (image_padded - self.__mean) / 255.0 + image_in = image_in.transpose(2, 0, 1) + image_in = image_in[np.newaxis, :, :, :] + image_in = Variable(cuda.cupy.asarray(image_in, dtype=cuda.cupy.float32)) + + top, left = pad_y1, pad_x1 + bottom, right = top + h, left + w + + return image_in, (top, left, bottom, right) diff --git a/spacenet/src/models/segmentation_cpu.py b/spacenet/src/models/segmentation_cpu.py new file mode 100644 index 0000000..e7b1061 --- /dev/null +++ b/spacenet/src/models/segmentation_cpu.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python + +##################################################################################################################################################################### +# xView2 # +# Copyright 2019 Carnegie Mellon University. # +# NO WARRANTY. THIS CARNEGIE MELLON UNIVERSITY AND SOFTWARE ENGINEERING INSTITUTE MATERIAL IS FURNISHED ON AN "AS-IS" BASIS. CARNEGIE MELLON UNIVERSITY MAKES NO # +# WARRANTIES OF ANY KIND, EITHER EXPRESSED OR IMPLIED, AS TO ANY MATTER INCLUDING, BUT NOT LIMITED TO, WARRANTY OF FITNESS FOR PURPOSE OR MERCHANTABILITY, # +# EXCLUSIVITY, OR RESULTS OBTAINED FROM USE OF THE MATERIAL. CARNEGIE MELLON UNIVERSITY DOES NOT MAKE ANY WARRANTY OF ANY KIND WITH RESPECT TO FREEDOM FROM PATENT, # +# TRADEMARK, OR COPYRIGHT INFRINGEMENT. # +# Released under a MIT (SEI)-style license, please see LICENSE.md or contact permission@sei.cmu.edu for full terms. # +# [DISTRIBUTION STATEMENT A] This material has been approved for public release and unlimited distribution. Please see Copyright notice for non-US Government use # +# and distribution. # +# This Software includes and/or makes use of the following Third-Party Software subject to its own license: # +# 1. SpaceNet (https://github.com/motokimura/spacenet_building_detection/blob/master/LICENSE) Copyright 2017 Motoki Kimura. # +# DM19-0988 # +##################################################################################################################################################################### + +import numpy as np +import cv2 +import math + +import chainer +import chainer.functions as F +from chainer import cuda, serializers, Variable + +from unet import UNet + +class SegmentationModel: + + def __init__(self, model_path, mean): + + # Load model + self.__model = UNet() + serializers.load_npz(model_path, self.__model) + + # Add height and width dimensions to mean + self.__mean = mean[np.newaxis, np.newaxis, :] + + def apply_segmentation(self, image): + + image_in, crop = self.__preprocess(image) + + chainer.using_config('device', -1) + + with chainer.using_config('train', False): + score = self.__model.forward(image_in) + + + score = F.softmax(score) + score = score.data[0] + + top, left, bottom, right = crop + score = score[:, top:bottom, left:right] + + return score + + def apply_segmentation_to_mosaic(self, mosaic, grid_px=800, tile_overlap_px=200): + + h, w, _ = mosaic.shape + + assert ((grid_px + tile_overlap_px * 2) % 16 == 0), "(grid_px + tile_overlap_px * 2) must be divisible by 16" + + pad_y1 = tile_overlap_px + pad_x1 = tile_overlap_px + + n_y = int(float(h) / float(grid_px)) + n_x = int(float(w) / float(grid_px)) + pad_y2 = n_y * grid_px + 2 * tile_overlap_px - h - pad_y1 + pad_x2 = n_x * grid_px + 2 * tile_overlap_px - h - pad_x1 + + mosaic_padded = np.pad(mosaic, ((pad_y1, pad_y2), (pad_x1, pad_x2), (0, 0)), 'symmetric') + + H, W, _ = mosaic_padded.shape + score_padded = np.zeros(shape=[self.__model.class_num, H, W], dtype=np.float32) + + for yi in range(n_y): + for xi in range(n_x): + + top = yi * grid_px + left = xi * grid_px + bottom = top + grid_px + 2 * tile_overlap_px + right = left + grid_px + 2 * tile_overlap_px + + tile = mosaic_padded[top:bottom, left:right] + + score_tile = self.apply_segmentation(tile) + + score_padded[:, top:bottom, left:right] = score_tile + + score = score_padded[:, pad_y1:-pad_y2, pad_x1:-pad_x2] + + return score + + def __preprocess(self, image): + + h, w, _ = image.shape + h_padded = int(math.ceil(float(h) / 16.0) * 16) + w_padded = int(math.ceil(float(w) / 16.0) * 16) + + pad_y1 = (h_padded - h) // 2 + pad_x1 = (w_padded - w) // 2 + pad_y2 = h_padded - h - pad_y1 + pad_x2 = w_padded - w - pad_x1 + + image_padded = np.pad(image, ((pad_y1, pad_y2), (pad_x1, pad_x2), (0, 0)), 'symmetric') + image_in = (image_padded - self.__mean) / 255.0 + image_in = image_in.transpose(2, 0, 1) + image_in = image_in[np.newaxis, :, :, :] + image_in = Variable(np.asarray(image_in, dtype=np.float32)) + + top, left = pad_y1, pad_x1 + bottom, right = top + h, left + w + + return image_in, (top, left, bottom, right) diff --git a/spacenet/src/models/tboard_logger.py b/spacenet/src/models/tboard_logger.py new file mode 100644 index 0000000..2aace7c --- /dev/null +++ b/spacenet/src/models/tboard_logger.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python + +##################################################################################################################################################################### +# xView2 # +# Copyright 2019 Carnegie Mellon University. # +# NO WARRANTY. THIS CARNEGIE MELLON UNIVERSITY AND SOFTWARE ENGINEERING INSTITUTE MATERIAL IS FURNISHED ON AN "AS-IS" BASIS. CARNEGIE MELLON UNIVERSITY MAKES NO # +# WARRANTIES OF ANY KIND, EITHER EXPRESSED OR IMPLIED, AS TO ANY MATTER INCLUDING, BUT NOT LIMITED TO, WARRANTY OF FITNESS FOR PURPOSE OR MERCHANTABILITY, # +# EXCLUSIVITY, OR RESULTS OBTAINED FROM USE OF THE MATERIAL. CARNEGIE MELLON UNIVERSITY DOES NOT MAKE ANY WARRANTY OF ANY KIND WITH RESPECT TO FREEDOM FROM PATENT, # +# TRADEMARK, OR COPYRIGHT INFRINGEMENT. # +# Released under a MIT (SEI)-style license, please see LICENSE.md or contact permission@sei.cmu.edu for full terms. # +# [DISTRIBUTION STATEMENT A] This material has been approved for public release and unlimited distribution. Please see Copyright notice for non-US Government use # +# and distribution. # +# This Software includes and/or makes use of the following Third-Party Software subject to its own license: # +# 1. SpaceNet (https://github.com/motokimura/spacenet_building_detection/blob/master/LICENSE) Copyright 2017 Motoki Kimura. # +# DM19-0988 # +##################################################################################################################################################################### + +from chainer.training import extension +from chainer import Variable +import cupy + +class TensorboardLogger(extension.Extension): + + def __init__(self, logger, entries=None): + + self._entries = entries + self._logger = logger + + return + + def __call__(self, trainer): + + observation = trainer.observation + for k, v in observation.items(): + if (self._entries is not None) and (k not in self._entries): + continue + + if isinstance(v, cupy.core.core.ndarray): + v = Variable(v) + + self._logger.add_scalar(k, v, trainer.updater.iteration) + + return diff --git a/spacenet/src/models/tboard_logger_cpu.py b/spacenet/src/models/tboard_logger_cpu.py new file mode 100644 index 0000000..19216c2 --- /dev/null +++ b/spacenet/src/models/tboard_logger_cpu.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python + +##################################################################################################################################################################### +# xView2 # +# Copyright 2019 Carnegie Mellon University. # +# NO WARRANTY. THIS CARNEGIE MELLON UNIVERSITY AND SOFTWARE ENGINEERING INSTITUTE MATERIAL IS FURNISHED ON AN "AS-IS" BASIS. CARNEGIE MELLON UNIVERSITY MAKES NO # +# WARRANTIES OF ANY KIND, EITHER EXPRESSED OR IMPLIED, AS TO ANY MATTER INCLUDING, BUT NOT LIMITED TO, WARRANTY OF FITNESS FOR PURPOSE OR MERCHANTABILITY, # +# EXCLUSIVITY, OR RESULTS OBTAINED FROM USE OF THE MATERIAL. CARNEGIE MELLON UNIVERSITY DOES NOT MAKE ANY WARRANTY OF ANY KIND WITH RESPECT TO FREEDOM FROM PATENT, # +# TRADEMARK, OR COPYRIGHT INFRINGEMENT. # +# Released under a MIT (SEI)-style license, please see LICENSE.md or contact permission@sei.cmu.edu for full terms. # +# [DISTRIBUTION STATEMENT A] This material has been approved for public release and unlimited distribution. Please see Copyright notice for non-US Government use # +# and distribution. # +# This Software includes and/or makes use of the following Third-Party Software subject to its own license: # +# 1. SpaceNet (https://github.com/motokimura/spacenet_building_detection/blob/master/LICENSE) Copyright 2017 Motoki Kimura. # +# DM19-0988 # +##################################################################################################################################################################### + +from chainer.training import extension +from chainer import Variable +import numpy as np + +class TensorboardLogger(extension.Extension): + + def __init__(self, logger, entries=None): + + self._entries = entries + self._logger = logger + + return + + def __call__(self, trainer): + + observation = trainer.observation + for k, v in observation.items(): + if (self._entries is not None) and (k not in self._entries): + continue + + if isinstance(v, np.core.ndarray): + v = Variable(v) + + self._logger.add_scalar(k, v, trainer.updater.iteration) + + return diff --git a/spacenet/src/models/train_model.py b/spacenet/src/models/train_model.py new file mode 100644 index 0000000..87b1288 --- /dev/null +++ b/spacenet/src/models/train_model.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python + +##################################################################################################################################################################### +# xView2 # +# Copyright 2019 Carnegie Mellon University. # +# NO WARRANTY. THIS CARNEGIE MELLON UNIVERSITY AND SOFTWARE ENGINEERING INSTITUTE MATERIAL IS FURNISHED ON AN "AS-IS" BASIS. CARNEGIE MELLON UNIVERSITY MAKES NO # +# WARRANTIES OF ANY KIND, EITHER EXPRESSED OR IMPLIED, AS TO ANY MATTER INCLUDING, BUT NOT LIMITED TO, WARRANTY OF FITNESS FOR PURPOSE OR MERCHANTABILITY, # +# EXCLUSIVITY, OR RESULTS OBTAINED FROM USE OF THE MATERIAL. CARNEGIE MELLON UNIVERSITY DOES NOT MAKE ANY WARRANTY OF ANY KIND WITH RESPECT TO FREEDOM FROM PATENT, # +# TRADEMARK, OR COPYRIGHT INFRINGEMENT. # +# Released under a MIT (SEI)-style license, please see LICENSE.md or contact permission@sei.cmu.edu for full terms. # +# [DISTRIBUTION STATEMENT A] This material has been approved for public release and unlimited distribution. Please see Copyright notice for non-US Government use # +# and distribution. # +# This Software includes and/or makes use of the following Third-Party Software subject to its own license: # +# 1. SpaceNet (https://github.com/motokimura/spacenet_building_detection/blob/master/LICENSE) Copyright 2017 Motoki Kimura. # +# DM19-0988 # +##################################################################################################################################################################### + +from __future__ import print_function + +import argparse +import numpy as np + +import chainer +import chainer.functions as F +import chainer.links as L +from chainer import training +from chainer.training import extensions + +from unet import UNet +from dataset import LabeledImageDataset + +from tensorboardX import SummaryWriter + +import os + + +def train_model(): + parser = argparse.ArgumentParser() + + parser.add_argument('dataset', help='Path to directory containing train.txt, val.txt, and mean.npy') + parser.add_argument('images', help='Root directory of input images') + parser.add_argument('labels', help='Root directory of label images') + + parser.add_argument('--batchsize', '-b', type=int, default=16, + help='Number of images in each mini-batch') + parser.add_argument('--test-batchsize', '-B', type=int, default=4, + help='Number of images in each test mini-batch') + parser.add_argument('--epoch', '-e', type=int, default=50, + help='Number of sweeps over the dataset to train') + parser.add_argument('--frequency', '-f', type=int, default=1, + help='Frequency of taking a snapshot') + parser.add_argument('--gpu', '-g', type=int, default=0, + help='GPU ID (negative value indicates CPU)') + parser.add_argument('--out', '-o', default='logs', + help='Directory to output the result under "models" directory') + parser.add_argument('--resume', '-r', default='', + help='Resume the training from snapshot') + parser.add_argument('--noplot', dest='plot', action='store_false', + help='Disable PlotReport extension') + + parser.add_argument('--tcrop', type=int, default=400, + help='Crop size for train-set images') + parser.add_argument('--vcrop', type=int, default=480, + help='Crop size for validation-set images') + + args = parser.parse_args() + + assert (args.tcrop % 16 == 0) and (args.vcrop % 16 == 0), "tcrop and vcrop must be divisible by 16." + + if args.gpu < 0: + from tboard_logger_cpu import TensorboardLogger + else: + from tboard_logger import TensorboardLogger + + print('GPU: {}'.format(args.gpu)) + print('# Minibatch-size: {}'.format(args.batchsize)) + print('# Crop-size: {}'.format(args.tcrop)) + print('# epoch: {}'.format(args.epoch)) + print('') + + this_dir = os.path.dirname(os.path.abspath(__file__)) + models_dir = os.path.normpath(os.path.join(this_dir, "../../models")) + log_dir = os.path.join(models_dir, args.out) + writer = SummaryWriter(log_dir=log_dir) + + # Set up a neural network to train + # Classifier reports softmax cross entropy loss and accuracy at every + # iteration, which will be used by the PrintReport extension below. + model = UNet() + if args.gpu >= 0: + # Make a specified GPU current + chainer.cuda.get_device_from_id(args.gpu).use() + model.to_gpu() # Copy the model to the GPU + + # Setup an optimizer + optimizer = chainer.optimizers.Adam() + optimizer.setup(model) + + # Load mean image + mean = np.load(os.path.join(args.dataset, "mean.npy")) + + # Load the MNIST dataset + train = LabeledImageDataset(os.path.join(args.dataset, "train.txt"), args.images, args.labels, + mean=mean, crop_size=args.tcrop, test=False, distort=False) + + test = LabeledImageDataset (os.path.join(args.dataset, "val.txt"), args.images, args.labels, + mean=mean, crop_size=args.vcrop, test=True, distort=False) + + train_iter = chainer.iterators.SerialIterator(train, args.batchsize) + test_iter = chainer.iterators.SerialIterator(test, args.test_batchsize, repeat=False, shuffle=False) + + # Set up a trainer + updater = training.StandardUpdater( + train_iter, optimizer, device=args.gpu) + trainer = training.Trainer(updater, (args.epoch, 'epoch'), out=log_dir) + + # Evaluate the model with the test dataset for each epoch + trainer.extend(extensions.Evaluator(test_iter, model, device=args.gpu)) + + # Dump a computational graph from 'loss' variable at the first iteration + # The "main" refers to the target link of the "main" optimizer. + trainer.extend(extensions.dump_graph('main/loss')) + + # Take a snapshot for each specified epoch + frequency = args.epoch if args.frequency == -1 else max(1, args.frequency) + trainer.extend(extensions.snapshot(), trigger=(frequency, 'epoch')) + + # Save trained model for each specific epoch + trainer.extend(extensions.snapshot_object( + model, 'model_iter_{.updater.iteration}'), trigger=(frequency, 'epoch')) + + # Write a log of evaluation statistics for each epoch + trainer.extend(extensions.LogReport()) + + # Save two plot images to the result dir + if args.plot and extensions.PlotReport.available(): + trainer.extend( + extensions.PlotReport(['main/loss', 'validation/main/loss'], + 'epoch', file_name='loss.png')) + trainer.extend( + extensions.PlotReport( + ['main/accuracy', 'validation/main/accuracy'], + 'epoch', file_name='accuracy.png')) + + # Print selected entries of the log to stdout + # Here "main" refers to the target link of the "main" optimizer again, and + # "validation" refers to the default name of the Evaluator extension. + # Entries other than 'epoch' are reported by the Classifier link, called by + # either the updater or the evaluator. + trainer.extend(extensions.PrintReport( + ['epoch', 'main/loss', 'validation/main/loss', + 'main/accuracy', 'validation/main/accuracy', 'elapsed_time'])) + + # Print a progress bar to stdout + trainer.extend(extensions.ProgressBar()) + + # Write training log to TensorBoard log file + trainer.extend(TensorboardLogger(writer, + ['main/loss', 'validation/main/loss', + 'main/accuracy', 'validation/main/accuracy'])) + + if args.resume: + # Resume from a snapshot + chainer.serializers.load_npz(args.resume, trainer) + + # Run the training + trainer.run() + + +if __name__ == '__main__': + train_model() diff --git a/spacenet/src/models/transforms.py b/spacenet/src/models/transforms.py new file mode 100644 index 0000000..2c682e6 --- /dev/null +++ b/spacenet/src/models/transforms.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python + +##################################################################################################################################################################### +# xView2 # +# Copyright 2019 Carnegie Mellon University. # +# NO WARRANTY. THIS CARNEGIE MELLON UNIVERSITY AND SOFTWARE ENGINEERING INSTITUTE MATERIAL IS FURNISHED ON AN "AS-IS" BASIS. CARNEGIE MELLON UNIVERSITY MAKES NO # +# WARRANTIES OF ANY KIND, EITHER EXPRESSED OR IMPLIED, AS TO ANY MATTER INCLUDING, BUT NOT LIMITED TO, WARRANTY OF FITNESS FOR PURPOSE OR MERCHANTABILITY, # +# EXCLUSIVITY, OR RESULTS OBTAINED FROM USE OF THE MATERIAL. CARNEGIE MELLON UNIVERSITY DOES NOT MAKE ANY WARRANTY OF ANY KIND WITH RESPECT TO FREEDOM FROM PATENT, # +# TRADEMARK, OR COPYRIGHT INFRINGEMENT. # +# Released under a MIT (SEI)-style license, please see LICENSE.md or contact permission@sei.cmu.edu for full terms. # +# [DISTRIBUTION STATEMENT A] This material has been approved for public release and unlimited distribution. Please see Copyright notice for non-US Government use # +# and distribution. # +# This Software includes and/or makes use of the following Third-Party Software subject to its own license: # +# 1. SpaceNet (https://github.com/motokimura/spacenet_building_detection/blob/master/LICENSE) Copyright 2017 Motoki Kimura. # +# DM19-0988 # +##################################################################################################################################################################### + +import numpy as np +import random + +import cv2 + + +def random_color_distort( + img, + brightness_delta=32, + contrast_low=0.5, contrast_high=1.5, + saturation_low=0.5, saturation_high=1.5, + hue_delta=18): + """A color related data augmentation used in SSD. + This function is a combination of four augmentation methods: + brightness, contrast, saturation and hue. + * brightness: Adding a random offset to the intensity of the image. + * contrast: Multiplying the intensity of the image by a random scale. + * saturation: Multiplying the saturation of the image by a random scale. + * hue: Adding a random offset to the hue of the image randomly. + This data augmentation is used in training of + Single Shot Multibox Detector [#]_. + Note that this function requires :mod:`cv2`. + .. [#] Wei Liu, Dragomir Anguelov, Dumitru Erhan, Christian Szegedy, + Scott Reed, Cheng-Yang Fu, Alexander C. Berg. + SSD: Single Shot MultiBox Detector. ECCV 2016. + Args: + img (~numpy.ndarray): An image array to be augmented. This is in + CHW and RGB format. + brightness_delta (float): The offset for saturation will be + drawn from :math:`[-brightness\_delta, brightness\_delta]`. + The default value is :obj:`32`. + contrast_low (float): The scale for contrast will be + drawn from :math:`[contrast\_low, contrast\_high]`. + The default value is :obj:`0.5`. + contrast_high (float): See :obj:`contrast_low`. + The default value is :obj:`1.5`. + saturation_low (float): The scale for saturation will be + drawn from :math:`[saturation\_low, saturation\_high]`. + The default value is :obj:`0.5`. + saturation_high (float): See :obj:`saturation_low`. + The default value is :obj:`1.5`. + hue_delta (float): The offset for hue will be + drawn from :math:`[-hue\_delta, hue\_delta]`. + The default value is :obj:`18`. + Returns: + An image in CHW and RGB format. + """ + + cv_img = img[::-1].astype(np.uint8) # RGB to BGR + + def convert(img, alpha=1, beta=0): + img = img.astype(float) * alpha + beta + img[img < 0] = 0 + img[img > 255] = 255 + return img.astype(np.uint8) + + def brightness(cv_img, delta): + if random.randrange(2): + return convert( + cv_img, + beta=random.uniform(-delta, delta)) + else: + return cv_img + + def contrast(cv_img, low, high): + if random.randrange(2): + return convert( + cv_img, + alpha=random.uniform(low, high)) + else: + return cv_img + + def saturation(cv_img, low, high): + if random.randrange(2): + cv_img = cv2.cvtColor(cv_img, cv2.COLOR_BGR2HSV) + cv_img[:, :, 1] = convert( + cv_img[:, :, 1], + alpha=random.uniform(low, high)) + return cv2.cvtColor(cv_img, cv2.COLOR_HSV2BGR) + else: + return cv_img + + def hue(cv_img, delta): + if random.randrange(2): + cv_img = cv2.cvtColor(cv_img, cv2.COLOR_BGR2HSV) + cv_img[:, :, 0] = ( + cv_img[:, :, 0].astype(int) + + random.randint(-delta, delta)) % 180 + return cv2.cvtColor(cv_img, cv2.COLOR_HSV2BGR) + else: + return cv_img + + cv_img = brightness(cv_img, brightness_delta) + + if random.randrange(2): + cv_img = contrast(cv_img, contrast_low, contrast_high) + cv_img = saturation(cv_img, saturation_low, saturation_high) + cv_img = hue(cv_img, hue_delta) + else: + cv_img = saturation(cv_img, saturation_low, saturation_high) + cv_img = hue(cv_img, hue_delta) + cv_img = contrast(cv_img, contrast_low, contrast_high) + + return cv_img[::-1] # RGB to BGR diff --git a/spacenet/src/models/unet.py b/spacenet/src/models/unet.py new file mode 100644 index 0000000..3ef9883 --- /dev/null +++ b/spacenet/src/models/unet.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python + +##################################################################################################################################################################### +# xView2 # +# Copyright 2019 Carnegie Mellon University. # +# NO WARRANTY. THIS CARNEGIE MELLON UNIVERSITY AND SOFTWARE ENGINEERING INSTITUTE MATERIAL IS FURNISHED ON AN "AS-IS" BASIS. CARNEGIE MELLON UNIVERSITY MAKES NO # +# WARRANTIES OF ANY KIND, EITHER EXPRESSED OR IMPLIED, AS TO ANY MATTER INCLUDING, BUT NOT LIMITED TO, WARRANTY OF FITNESS FOR PURPOSE OR MERCHANTABILITY, # +# EXCLUSIVITY, OR RESULTS OBTAINED FROM USE OF THE MATERIAL. CARNEGIE MELLON UNIVERSITY DOES NOT MAKE ANY WARRANTY OF ANY KIND WITH RESPECT TO FREEDOM FROM PATENT, # +# TRADEMARK, OR COPYRIGHT INFRINGEMENT. # +# Released under a MIT (SEI)-style license, please see LICENSE.md or contact permission@sei.cmu.edu for full terms. # +# [DISTRIBUTION STATEMENT A] This material has been approved for public release and unlimited distribution. Please see Copyright notice for non-US Government use # +# and distribution. # +# This Software includes and/or makes use of the following Third-Party Software subject to its own license: # +# 1. SpaceNet (https://github.com/motokimura/spacenet_building_detection/blob/master/LICENSE) Copyright 2017 Motoki Kimura. # +# DM19-0988 # +##################################################################################################################################################################### + +import chainer +import chainer.functions as F +import chainer.links as L + + +class UNet(chainer.Chain): + + def __init__(self, class_num=2, ignore_label=255): + + self.__class_num = class_num + self.__ignore_label = ignore_label + + super(UNet, self).__init__( + c0=L.Convolution2D(3, 32, 3, 1, 1), + c1=L.Convolution2D(32, 64, 4, 2, 1), + c2=L.Convolution2D(64, 64, 3, 1, 1), + c3=L.Convolution2D(64, 128, 4, 2, 1), + c4=L.Convolution2D(128, 128, 3, 1, 1), + c5=L.Convolution2D(128, 256, 4, 2, 1), + c6=L.Convolution2D(256, 256, 3, 1, 1), + c7=L.Convolution2D(256, 512, 4, 2, 1), + c8=L.Convolution2D(512, 512, 3, 1, 1), + + dc8=L.Deconvolution2D(1024, 512, 4, 2, 1), + dc7=L.Convolution2D(512, 256, 3, 1, 1), + dc6=L.Deconvolution2D(512, 256, 4, 2, 1), + dc5=L.Convolution2D(256, 128, 3, 1, 1), + dc4=L.Deconvolution2D(256, 128, 4, 2, 1), + dc3=L.Convolution2D(128, 64, 3, 1, 1), + dc2=L.Deconvolution2D(128, 64, 4, 2, 1), + dc1=L.Convolution2D(64, 32, 3, 1, 1), + dc0=L.Convolution2D(64, class_num, 3, 1, 1), + + bnc0=L.BatchNormalization(32), + bnc1=L.BatchNormalization(64), + bnc2=L.BatchNormalization(64), + bnc3=L.BatchNormalization(128), + bnc4=L.BatchNormalization(128), + bnc5=L.BatchNormalization(256), + bnc6=L.BatchNormalization(256), + bnc7=L.BatchNormalization(512), + bnc8=L.BatchNormalization(512), + + bnd8=L.BatchNormalization(512), + bnd7=L.BatchNormalization(256), + bnd6=L.BatchNormalization(256), + bnd5=L.BatchNormalization(128), + bnd4=L.BatchNormalization(128), + bnd3=L.BatchNormalization(64), + bnd2=L.BatchNormalization(64), + bnd1=L.BatchNormalization(32) + ) + + + def forward(self, x): + + e0 = F.relu(self.bnc0(self.c0(x))) + e1 = F.relu(self.bnc1(self.c1(e0))) + e2 = F.relu(self.bnc2(self.c2(e1))) + del e1 + e3 = F.relu(self.bnc3(self.c3(e2))) + e4 = F.relu(self.bnc4(self.c4(e3))) + del e3 + e5 = F.relu(self.bnc5(self.c5(e4))) + e6 = F.relu(self.bnc6(self.c6(e5))) + del e5 + e7 = F.relu(self.bnc7(self.c7(e6))) + e8 = F.relu(self.bnc8(self.c8(e7))) + + d8 = F.relu(self.bnd8(self.dc8(F.concat([e7, e8])))) + del e7, e8 + d7 = F.relu(self.bnd7(self.dc7(d8))) + del d8 + d6 = F.relu(self.bnd6(self.dc6(F.concat([e6, d7])))) + del d7, e6 + d5 = F.relu(self.bnd5(self.dc5(d6))) + del d6 + d4 = F.relu(self.bnd4(self.dc4(F.concat([e4, d5])))) + del d5, e4 + d3 = F.relu(self.bnd3(self.dc3(d4))) + del d4 + d2 = F.relu(self.bnd2(self.dc2(F.concat([e2, d3])))) + del d3, e2 + d1 = F.relu(self.bnd1(self.dc1(d2))) + del d2 + d0 = self.dc0(F.concat([e0, d1])) + + return d0 + + + def __call__(self, x, t): + + h = self.forward(x) + + loss = F.softmax_cross_entropy(h, t, ignore_label=self.__ignore_label) + accuracy = F.accuracy(h, t, ignore_label=self.__ignore_label) + + chainer.report({'loss': loss, 'accuracy': accuracy}, self) + + return loss + + + @property + def class_num(self): + return self.__class_num + diff --git a/submission/Dockerfile b/submission/Dockerfile new file mode 100644 index 0000000..e340a21 --- /dev/null +++ b/submission/Dockerfile @@ -0,0 +1,54 @@ +##################################################################################################################################################################### +# xView2 # +# Copyright 2019 Carnegie Mellon University. # +# NO WARRANTY. THIS CARNEGIE MELLON UNIVERSITY AND SOFTWARE ENGINEERING INSTITUTE MATERIAL IS FURNISHED ON AN "AS-IS" BASIS. CARNEGIE MELLON UNIVERSITY MAKES NO # +# WARRANTIES OF ANY KIND, EITHER EXPRESSED OR IMPLIED, AS TO ANY MATTER INCLUDING, BUT NOT LIMITED TO, WARRANTY OF FITNESS FOR PURPOSE OR MERCHANTABILITY, # +# EXCLUSIVITY, OR RESULTS OBTAINED FROM USE OF THE MATERIAL. CARNEGIE MELLON UNIVERSITY DOES NOT MAKE ANY WARRANTY OF ANY KIND WITH RESPECT TO FREEDOM FROM PATENT, # +# TRADEMARK, OR COPYRIGHT INFRINGEMENT. # +# Released under a MIT (SEI)-style license, please see LICENSE.md or contact permission@sei.cmu.edu for full terms. # +# [DISTRIBUTION STATEMENT A] This material has been approved for public release and unlimited distribution. Please see Copyright notice for non-US Government use # +# and distribution. # +# This Software includes and/or makes use of the following Third-Party Software subject to its own license: # +# 1. SpaceNet (https://github.com/motokimura/spacenet_building_detection/blob/master/LICENSE) Copyright 2017 Motoki Kimura. # +# DM19-0988 # +##################################################################################################################################################################### + +FROM ubuntu + +SHELL ["/bin/bash", "-c"] + +# Making the appropate directories for the inference code, inputs, and outputs +RUN mkdir -p /code/xview-2/ +RUN mkdir -p /submission/ +RUN mkdir -p /output/ + +ADD xview-2 /code/xview-2/ + +# Updating the image +RUN apt update +RUN apt upgrade -y + +# Installing all dependencies +RUN apt install -y python3 python3-pip curl +RUN pip3 install numpy +RUN pip3 install -U "tensorflow==1.*" +RUN pip3 install matplotlib tqdm libtiff scipy Pillow scikit-image opencv-python imgaug IPython geopandas keras imantics simplification scikit-learn chainer tensorboard tensorboardX + +# Grabbing weights for our models +# UNCOMMENT THIS WHEN REPO/WEIGHTS ARE PUBLIC +#RUN curl -L https://github.com/DIUx-xView/xView2/releases/download/v1.0/localization.h5 -o /code/xview-2/weights/localization.h5 +#RUN curl -L https://github.com/DIUx-xView/xView2/releases/download/v1.0/classification.hdf5 -o /code/xview-2/weights/classification.hdf5 + +# Downloading ResNet and including it in the image +RUN mkdir -p /root/.keras/models/ + +ADD https://github.com/fchollet/deep-learning-models/releases/download/v0.2/resnet50_weights_tf_dim_ordering_tf_kernels.h5 /root/.keras/models/resnet50_weights_tf_dim_ordering_tf_kernels.h5 +ADD https://github.com/fchollet/deep-learning-models/releases/download/v0.2/resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5 /root/.keras/models/resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5 + + +# Making python3 the default Python +RUN ln -s /usr/bin/python3 /usr/bin/python + +# Finally running the container with the run.sh wrapper script to pass the CLI arguments to inference.sh +RUN chmod +x /code/xview-2/submission/run.sh +ENTRYPOINT ["/code/xview-2/submission/run.sh"] diff --git a/submission/build b/submission/build new file mode 100755 index 0000000..835a366 --- /dev/null +++ b/submission/build @@ -0,0 +1,28 @@ +#!/bin/bash -e + +##################################################################################################################################################################### +# xView2 # +# Copyright 2019 Carnegie Mellon University. # +# NO WARRANTY. THIS CARNEGIE MELLON UNIVERSITY AND SOFTWARE ENGINEERING INSTITUTE MATERIAL IS FURNISHED ON AN "AS-IS" BASIS. CARNEGIE MELLON UNIVERSITY MAKES NO # +# WARRANTIES OF ANY KIND, EITHER EXPRESSED OR IMPLIED, AS TO ANY MATTER INCLUDING, BUT NOT LIMITED TO, WARRANTY OF FITNESS FOR PURPOSE OR MERCHANTABILITY, # +# EXCLUSIVITY, OR RESULTS OBTAINED FROM USE OF THE MATERIAL. CARNEGIE MELLON UNIVERSITY DOES NOT MAKE ANY WARRANTY OF ANY KIND WITH RESPECT TO FREEDOM FROM PATENT, # +# TRADEMARK, OR COPYRIGHT INFRINGEMENT. # +# Released under a MIT (SEI)-style license, please see LICENSE.md or contact permission@sei.cmu.edu for full terms. # +# [DISTRIBUTION STATEMENT A] This material has been approved for public release and unlimited distribution. Please see Copyright notice for non-US Government use # +# and distribution. # +# This Software includes and/or makes use of the following Third-Party Software subject to its own license: # +# 1. SpaceNet (https://github.com/motokimura/spacenet_building_detection/blob/master/LICENSE) Copyright 2017 Motoki Kimura. # +# DM19-0988 # +##################################################################################################################################################################### + +# WARNING: You may have to run this with sudo please read what this script does! +# Building the docker image from the current directory or the directory provided + +if [ $# == 1 ]; then + DOCKER_LOCATION=${1} + docker build -t cmu-xview2-baseline -f ${DOCKER_LOCATION} . +else + echo "Error: expect only 1 arugment: Dockerfile location (go up 1 level from ./xview-2 to include xview-2 in the image (REQUIRED))" + exit 1 +fi + diff --git a/submission/run.sh b/submission/run.sh new file mode 100755 index 0000000..9a9a29a --- /dev/null +++ b/submission/run.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +##################################################################################################################################################################### +# xView2 # +# Copyright 2019 Carnegie Mellon University. # +# NO WARRANTY. THIS CARNEGIE MELLON UNIVERSITY AND SOFTWARE ENGINEERING INSTITUTE MATERIAL IS FURNISHED ON AN "AS-IS" BASIS. CARNEGIE MELLON UNIVERSITY MAKES NO # +# WARRANTIES OF ANY KIND, EITHER EXPRESSED OR IMPLIED, AS TO ANY MATTER INCLUDING, BUT NOT LIMITED TO, WARRANTY OF FITNESS FOR PURPOSE OR MERCHANTABILITY, # +# EXCLUSIVITY, OR RESULTS OBTAINED FROM USE OF THE MATERIAL. CARNEGIE MELLON UNIVERSITY DOES NOT MAKE ANY WARRANTY OF ANY KIND WITH RESPECT TO FREEDOM FROM PATENT, # +# TRADEMARK, OR COPYRIGHT INFRINGEMENT. # +# Released under a MIT (SEI)-style license, please see LICENSE.md or contact permission@sei.cmu.edu for full terms. # +# [DISTRIBUTION STATEMENT A] This material has been approved for public release and unlimited distribution. Please see Copyright notice for non-US Government use # +# and distribution. # +# This Software includes and/or makes use of the following Third-Party Software subject to its own license: # +# 1. SpaceNet (https://github.com/motokimura/spacenet_building_detection/blob/master/LICENSE) Copyright 2017 Motoki Kimura. # +# DM19-0988 # +##################################################################################################################################################################### + +# Running inference using CLI arugments passed in +# 1) input pre image +# 2) input post image +# 3) output localization image +# 4) output localization+classifcation image + +/code/xview-2/utils/inference.sh -x /code/xview-2/ -i $1 -p $2 -o $3 -l /code/xview-2/weights/localization.h5 -c /code/xview-2/weights/classification.hdf5 -y + +# The two images we will use for scoring will be identical so just copying the output localization path to the localization+classification path +cp $3 $4 diff --git a/utils/combine_jsons.py b/utils/combine_jsons.py new file mode 100644 index 0000000..336d582 --- /dev/null +++ b/utils/combine_jsons.py @@ -0,0 +1,96 @@ +##################################################################################################################################################################### +# xView2 # +# Copyright 2019 Carnegie Mellon University. # +# NO WARRANTY. THIS CARNEGIE MELLON UNIVERSITY AND SOFTWARE ENGINEERING INSTITUTE MATERIAL IS FURNISHED ON AN "AS-IS" BASIS. CARNEGIE MELLON UNIVERSITY MAKES NO # +# WARRANTIES OF ANY KIND, EITHER EXPRESSED OR IMPLIED, AS TO ANY MATTER INCLUDING, BUT NOT LIMITED TO, WARRANTY OF FITNESS FOR PURPOSE OR MERCHANTABILITY, # +# EXCLUSIVITY, OR RESULTS OBTAINED FROM USE OF THE MATERIAL. CARNEGIE MELLON UNIVERSITY DOES NOT MAKE ANY WARRANTY OF ANY KIND WITH RESPECT TO FREEDOM FROM PATENT, # +# TRADEMARK, OR COPYRIGHT INFRINGEMENT. # +# Released under a MIT (SEI)-style license, please see LICENSE.md or contact permission@sei.cmu.edu for full terms. # +# [DISTRIBUTION STATEMENT A] This material has been approved for public release and unlimited distribution. Please see Copyright notice for non-US Government use # +# and distribution. # +# This Software includes and/or makes use of the following Third-Party Software subject to its own license: # +# 1. SpaceNet (https://github.com/motokimura/spacenet_building_detection/blob/master/LICENSE) Copyright 2017 Motoki Kimura. # +# DM19-0988 # +##################################################################################################################################################################### + +import json + +def combine_output(pred_polygons, pred_classification, output_file): + """ + :param pred_polygons: the file path to the localization inference output json + :param pre_classification: the file path to the classification inference output json + :param output_file: the file path to store the combined json file + """ + + # Skeleton of the json with null values + output_json = { + "features": { + "lng_lat": [], + "xy": [] + }, + "metadata": { + "sensor": "", + "provider_asset_type": "", + "gsd": 0, + "capture_date": "", + "off_nadir_angle": 0, + "pan_resolution": 0, + "sun_azimuth": 0, + "sun_elevation": 0, + "target_azimuth": 0, + "disaster": "", + "disaster_type": "", + "catalog_id": "", + "original_width": 0, + "original_height": 0, + "width": 0, + "height": 0, + "id": "", + "img_name": "" + } + } + + # Open the classification json + with open(pred_classification) as labels: + label_json = json.load(labels) + + # Open the localization json + with open(pred_polygons) as polys: + poly_json = json.load(polys) + + # Match UUIDs from the two jsons and combine in output_json skeleton + for p in poly_json['features']['xy']: + p['properties']['subtype'] = label_json[p['properties']['uid']] + output_json['features']['xy'].append(p) + + # Finally save out the combined json file + with open(output_file, 'w') as out: + json.dump(output_json, out) + +if __name__ == '__main__': + import argparse + + # Parse command line arguments + parser = argparse.ArgumentParser( + description= + """combine_jsons.py: combines the outputs of localization and classification inference into a single output json""" + ) + parser.add_argument('--polys', + required=True, + metavar='/path/to/input/polygons.json', + help="Full path to the json from polygonize.py") + parser.add_argument('--classes', + required=True, + metavar='/path/to/classifications.json', + help="Full path to the json from tensor_inf.py" + ) + parser.add_argument('--output', + required=True, + metavar='/path/to/pred.json', + help="Full path to save the final single output file to" + ) + + args = parser.parse_args() + + # Combining the json based off the uuid assigned at the polygonize stage + combine_output(args.polys, args.classes, args.output) diff --git a/utils/data_finalize.sh b/utils/data_finalize.sh new file mode 100755 index 0000000..6d36593 --- /dev/null +++ b/utils/data_finalize.sh @@ -0,0 +1,100 @@ +#!/bin/bash + +##################################################################################################################################################################### +# xView2 # +# Copyright 2019 Carnegie Mellon University. # +# NO WARRANTY. THIS CARNEGIE MELLON UNIVERSITY AND SOFTWARE ENGINEERING INSTITUTE MATERIAL IS FURNISHED ON AN "AS-IS" BASIS. CARNEGIE MELLON UNIVERSITY MAKES NO # +# WARRANTIES OF ANY KIND, EITHER EXPRESSED OR IMPLIED, AS TO ANY MATTER INCLUDING, BUT NOT LIMITED TO, WARRANTY OF FITNESS FOR PURPOSE OR MERCHANTABILITY, # +# EXCLUSIVITY, OR RESULTS OBTAINED FROM USE OF THE MATERIAL. CARNEGIE MELLON UNIVERSITY DOES NOT MAKE ANY WARRANTY OF ANY KIND WITH RESPECT TO FREEDOM FROM PATENT, # +# TRADEMARK, OR COPYRIGHT INFRINGEMENT. # +# Released under a MIT (SEI)-style license, please see LICENSE.md or contact permission@sei.cmu.edu for full terms. # +# [DISTRIBUTION STATEMENT A] This material has been approved for public release and unlimited distribution. Please see Copyright notice for non-US Government use # +# and distribution. # +# This Software includes and/or makes use of the following Third-Party Software subject to its own license: # +# 1. SpaceNet (https://github.com/motokimura/spacenet_building_detection/blob/master/LICENSE) Copyright 2017 Motoki Kimura. # +# DM19-0988 # +##################################################################################################################################################################### + + +set -euo pipefail + +# this function is called when Ctrl-C is sent +function trap_ctrlc () +{ + # perform cleanup here + echo "Ctrl-C or Error caught...performing clean up" + + if [ -d /tmp/inference ]; then + rm -rf /tmp/inference + fi + + exit 99 +} + +# initialise trap to call trap_ctrlc function +# when signal 2 (SIGINT) is received +trap "trap_ctrlc" 2 9 13 3 17 + +help_message () { + printf "${0}: Moves files around for the spacenet model to train\n\t-i /path/to/xBD/ \n\t-s split percentage to go to train\n\t-x /path/to/xview-2/repository/\n\t(Note: this script expects mask_polygons.py to have ran first to create labels)\n\n" +} + +if [ $# -lt 3 ]; then + help_message + exit 1 +fi + +while getopts "i:s:x:h" OPTION +do + case $OPTION in + h) + help_message + exit 1 + ;; + i) + input="$OPTARG" + ;; + s) + split="$OPTARG" + ;; + x) + XBDIR="$OPTARG" + ;; + esac +done + + +# Get list of disasters to iterate over +disasters=`/bin/ls -1 "$input"` + +# Making the spacenet training directory +mkdir -p "$input"/spacenet_gt/images +mkdir -p "$input"/spacenet_gt/labels +mkdir -p "$input"/spacenet_gt/dataSet + +# for each disaster, copy the pre images and labels to the spacenet training directory +for disaster in $disasters; do + masks=`/bin/ls -1 "$input"/"$disaster"/masks` + for mask in $masks; do + cp "$input"/"$disaster"/masks/$mask "$input"/spacenet_gt/labels + cp "$input"/"$disaster"/images/$mask "$input"/spacenet_gt/images + done +done + +# Listing all files to do the split +cd "$input"/spacenet_gt/dataSet/ +touch all_images.txt +/bin/ls -1 "$input"/spacenet_gt/images > all_images.txt + +line_count=`cat all_images.txt | wc -l` +lines_to_split=$(bc -l <<< "$line_count"*"$split") +split -l `awk -F. '{print $1}' <<< $lines_to_split` all_images.txt + +mv ./xaa train.txt +mv ./xab val.txt +rm all_images.txt + +# Running the mean creation code over the images +python "$XBDIR"/spacenet/src/features/compute_mean.py "$input"/spacenet_gt/dataSet/train.txt --root "$input"/spacenet_gt/images/ --output "$input"/spacenet_gt/dataSet/mean.npy + +echo "Done!" diff --git a/utils/inference.sh b/utils/inference.sh new file mode 100755 index 0000000..a29d53b --- /dev/null +++ b/utils/inference.sh @@ -0,0 +1,174 @@ +#!/bin/bash + +##################################################################################################################################################################### +# xView2 # +# Copyright 2019 Carnegie Mellon University. # +# NO WARRANTY. THIS CARNEGIE MELLON UNIVERSITY AND SOFTWARE ENGINEERING INSTITUTE MATERIAL IS FURNISHED ON AN "AS-IS" BASIS. CARNEGIE MELLON UNIVERSITY MAKES NO # +# WARRANTIES OF ANY KIND, EITHER EXPRESSED OR IMPLIED, AS TO ANY MATTER INCLUDING, BUT NOT LIMITED TO, WARRANTY OF FITNESS FOR PURPOSE OR MERCHANTABILITY, # +# EXCLUSIVITY, OR RESULTS OBTAINED FROM USE OF THE MATERIAL. CARNEGIE MELLON UNIVERSITY DOES NOT MAKE ANY WARRANTY OF ANY KIND WITH RESPECT TO FREEDOM FROM PATENT, # +# TRADEMARK, OR COPYRIGHT INFRINGEMENT. # +# Released under a MIT (SEI)-style license, please see LICENSE.md or contact permission@sei.cmu.edu for full terms. # +# [DISTRIBUTION STATEMENT A] This material has been approved for public release and unlimited distribution. Please see Copyright notice for non-US Government use # +# and distribution. # +# This Software includes and/or makes use of the following Third-Party Software subject to its own license: # +# 1. SpaceNet (https://github.com/motokimura/spacenet_building_detection/blob/master/LICENSE) Copyright 2017 Motoki Kimura. # +# DM19-0988 # +##################################################################################################################################################################### + +set -euo pipefail + +# this function is called when Ctrl-C is sent +function trap_ctrlc () +{ + # perform cleanup here + echo "Ctrl-C or Error caught...performing clean up" + + if [ -d /tmp/inference ]; then + rm -rf /tmp/inference + fi + + exit 99 +} + +# initialise trap to call trap_ctrlc function +# when signal 2 (SIGINT) is received +trap "trap_ctrlc" 2 9 13 3 17 + +help_message () { + printf "${0}: Runs the polygonization in inference mode\n\t-x: path to xview-2 repository\n\t-i: /full/path/to/input/pre-disaster/image.png\n\t-p: /full/path/to/input/post-disaster/image.png\n\t-o: /path/to/output.png\n\t-l: path/to/localization_weights\n\t-c: path/to/classification_weights\n\t-e /path/to/virtual/env/activate\n\t-y continue with local environment and without interactive prompt\n\n" +} + +input="" +input_post="" +inference_base="/tmp/inference" +LOGFILE="/tmp/inference_log" +XBDIR="" +virtual_env="" +localization_weights="" +classification_weights="" +continue_answer="n" + +if [ "$#" -lt 13 ]; then + help_message + exit 1 +fi + +while getopts "i:p:o:x:l:e:c:hy" OPTION +do + case $OPTION in + h) + help_message + exit 0 + ;; + y) + continue_answer="y" + ;; + o) + output_file="$OPTARG" + ;; + x) + XBDIR="$OPTARG" + virtual_env="$XBDIR/bin/activate" + ;; + i) + input="$OPTARG" + ;; + p) + input_post="$OPTARG" + ;; + l) + localization_weights="$OPTARG" + ;; + c) + classification_weights="$OPTARG" + ;; + e) + virtual_env="$OPTARG" + ;; + ?) + help_message + exit 0 + ;; + esac +done + +# Create the output directory if it doesn't exist +mkdir -p "$inference_base" + +if ! [ -f "$LOGFILE" ]; then + touch "$LOGFILE" +fi + +printf "==========\n" >> "$LOGFILE" +echo `date +%Y%m%dT%H%M%S` >> "$LOGFILE" +printf "\n" >> "$LOGFILE" + +input_image=${input##*/} + +label_temp="$inference_base"/"${input_image%.*}"/labels +mkdir -p "$label_temp" + +printf "\n" + +printf "\n" + +# Run in inference mode +# Because of the models _have_ to be in the correct directory, they use relative paths to find the source (e.g. "../src") +# sourcing the virtual environment packages if they exist +# this is *necessary* or all packages must be installed globally +if [ -f "$virtual_env" ]; then + source "$virtual_env" +else + if [ "$continue_answer" = "n" ]; then + printf "Error: cannot source virtual environment \n\tDo you have all the dependencies installed and want to continue? [Y/N]: " + read continue_answer + if [ "$continue_answer" == "N" ]; then + exit 2 + fi + fi +fi + +cd "$XBDIR"/spacenet/inference/ + +# Quietly running the localization inference to output a json with the predicted polygons from the supplied input image +printf "Running localization\n" +python ./inference.py --input "$input" --weights "$localization_weights" --mean "$XBDIR"/weights/mean.npy --output "$label_temp"/"${input_image%.*}".json >> "$LOGFILE" 2>&1 + +printf "\n" >> "$LOGFILE" + +# Classification inferences start below +cd "$XBDIR"/model + +# Replace the pre image here with the post +# We need to do this so the classification inference pulls the images from the post +# Since post is where the damage occurs +printf "Grabbing post image file for classification\n" +disaster_post_file="$input_post" + +mkdir -p "$inference_base"/output_polygons + +printf "Running classification\n" + +# Extracting polygons from post image +python ./process_data_inference.py --input_img "$disaster_post_file" --label_path "$label_temp"/"${input_image%.*}".json --output_dir "$inference_base"/output_polygons --output_csv "$inference_base"/output.csv >> "$LOGFILE" 2>&1 + +# Classifying extracted polygons +python ./damage_inference.py --test_data "$inference_base"/output_polygons --test_csv "$inference_base"/output.csv --model_weights "$classification_weights" --output_json /tmp/inference/classification_inference.json >> "$LOGFILE" 2>&1 + +printf "\n" >> "$LOGFILE" + +# Combining the predicted polygons with the predicted labels, based off a UUID generated during the localization inference stage +printf "Formatting json and scoring image\n" +python "$XBDIR"/utils/combine_jsons.py --polys "$label_temp"/"${input_image%.*}".json --classes /tmp/inference/classification_inference.json --output "$inference_base/inference.json" >> "$LOGFILE" 2>&1 +printf "\n" >> "$LOGFILE" + +# Transforming the inference json file to the image required for scoring +printf "Finalizing output to $output_file\n" +python "$XBDIR"/utils/inference_image_output.py --input "$inference_base"/inference.json --output "$output_file" >> "$LOGFILE" 2>&1 + +#Cleaning up by removing the temporary working directory we created +printf "Cleaning up\n" +rm -rf "$inference_base" +printf "==========\n" >> "$LOGFILE" +printf "Done!\n" + diff --git a/utils/inference_image_output.py b/utils/inference_image_output.py new file mode 100644 index 0000000..b880ea8 --- /dev/null +++ b/utils/inference_image_output.py @@ -0,0 +1,96 @@ +##################################################################################################################################################################### +# xView2 # +# Copyright 2019 Carnegie Mellon University. # +# NO WARRANTY. THIS CARNEGIE MELLON UNIVERSITY AND SOFTWARE ENGINEERING INSTITUTE MATERIAL IS FURNISHED ON AN "AS-IS" BASIS. CARNEGIE MELLON UNIVERSITY MAKES NO # +# WARRANTIES OF ANY KIND, EITHER EXPRESSED OR IMPLIED, AS TO ANY MATTER INCLUDING, BUT NOT LIMITED TO, WARRANTY OF FITNESS FOR PURPOSE OR MERCHANTABILITY, # +# EXCLUSIVITY, OR RESULTS OBTAINED FROM USE OF THE MATERIAL. CARNEGIE MELLON UNIVERSITY DOES NOT MAKE ANY WARRANTY OF ANY KIND WITH RESPECT TO FREEDOM FROM PATENT, # +# TRADEMARK, OR COPYRIGHT INFRINGEMENT. # +# Released under a MIT (SEI)-style license, please see LICENSE.md or contact permission@sei.cmu.edu for full terms. # +# [DISTRIBUTION STATEMENT A] This material has been approved for public release and unlimited distribution. Please see Copyright notice for non-US Government use # +# and distribution. # +# This Software includes and/or makes use of the following Third-Party Software subject to its own license: # +# 1. SpaceNet (https://github.com/motokimura/spacenet_building_detection/blob/master/LICENSE) Copyright 2017 Motoki Kimura. # +# DM19-0988 # +##################################################################################################################################################################### + + +import json +from shapely import wkt +from shapely.geometry import Polygon +import numpy as np +from cv2 import fillPoly, imwrite + +def open_json(json_file_path): + """ + :param json_file_path: path to open inference json file + :returns: the json data dictionary of localized polygon and their classifications + """ + + with open(json_file_path) as jf: + json_data = json.load(jf) + inference_data = json_data['features']['xy'] + return inference_data + +def create_image(inference_data): + """ + :params inference_data: json data dictionary of localized polygon and their classifications + :returns: an numpy array of 8-bit grey scale image with polygons filled in according to the key provided + """ + + damage_key = {'no-damage': 1, 'minor-damage': 2, 'major-damage': 3, 'destroyed': 4} + + mask_img = np.zeros((1024,1024,1), np.uint8) + + for poly in inference_data: + damage = poly['properties']['subtype'] + coords = wkt.loads(poly['wkt']) + poly_np = np.array(coords.exterior.coords, np.int32) + + fillPoly(mask_img, [poly_np], damage_key[damage]) + + return mask_img + +def save_image(polygons, output_path): + """ + :param polygons: np array with filled in polygons from create_image() + :param output_path: path to save the final output inference image + """ + + # Output the filled in polygons to an image file + imwrite(output_path, polygons) + +def create_inference_image(json_input_path, image_output_path): + """ + :param json_input_path: Path to output inference json file + :param image_outut_pat: Path to save the final inference image + """ + + # Getting the inference data from the localization and classification + inference_data = open_json(json_input_path) + + # Filling in the polygons and readying the image format + polygon_array = create_image(inference_data) + + # Saving the image to the desired location + save_image(polygon_array, image_output_path) + +if __name__ == '__main__': + import argparse + + # Parse command line arguments + parser = argparse.ArgumentParser( + description= + """inference_image_output.py: Takes the inference localization and classification final outputs in json from and outputs an image ready to be scored based off the challenge parameters""") + parser.add_argument('--input', + required=True, + metavar='/path/to/final/inference.json', + help="Full path to the final inference json") + parser.add_argument('--output', + required=True, + metavar='/path/to/inference.png', + help="Full path to save the image to") + + args = parser.parse_args() + + # Creating the scoring image + create_inference_image(args.input, args.output) diff --git a/utils/mask_polygons.py b/utils/mask_polygons.py new file mode 100644 index 0000000..87530a7 --- /dev/null +++ b/utils/mask_polygons.py @@ -0,0 +1,275 @@ +##################################################################################################################################################################### +# xView2 # +# Copyright 2019 Carnegie Mellon University. # +# NO WARRANTY. THIS CARNEGIE MELLON UNIVERSITY AND SOFTWARE ENGINEERING INSTITUTE MATERIAL IS FURNISHED ON AN "AS-IS" BASIS. CARNEGIE MELLON UNIVERSITY MAKES NO # +# WARRANTIES OF ANY KIND, EITHER EXPRESSED OR IMPLIED, AS TO ANY MATTER INCLUDING, BUT NOT LIMITED TO, WARRANTY OF FITNESS FOR PURPOSE OR MERCHANTABILITY, # +# EXCLUSIVITY, OR RESULTS OBTAINED FROM USE OF THE MATERIAL. CARNEGIE MELLON UNIVERSITY DOES NOT MAKE ANY WARRANTY OF ANY KIND WITH RESPECT TO FREEDOM FROM PATENT, # +# TRADEMARK, OR COPYRIGHT INFRINGEMENT. # +# Released under a MIT (SEI)-style license, please see LICENSE.md or contact permission@sei.cmu.edu for full terms. # +# [DISTRIBUTION STATEMENT A] This material has been approved for public release and unlimited distribution. Please see Copyright notice for non-US Government use # +# and distribution. # +# This Software includes and/or makes use of the following Third-Party Software subject to its own license: # +# 1. SpaceNet (https://github.com/motokimura/spacenet_building_detection/blob/master/LICENSE) Copyright 2017 Motoki Kimura. # +# DM19-0988 # +##################################################################################################################################################################### + + +import json +from os import path, walk, makedirs +from sys import exit, stderr + +from cv2 import fillPoly, imwrite +import numpy as np +from shapely import wkt +from shapely.geometry import mapping, Polygon +from skimage.io import imread +from tqdm import tqdm +import imantics + +# This removes the massive amount of scikit warnings of "low contrast images" +import warnings +warnings.filterwarnings("ignore", category=UserWarning) + + +def get_dimensions(file_path): + """ + :param file_path: The path of the file + :return: returns (width,height,channels) + """ + # Open the image we are going to mask + pil_img = imread(file_path) + img = np.array(pil_img) + w, h, c = img.shape + return (w, h, c) + + +def mask_polygons_separately(size, shapes): + """ + :param size: A tuple of the (width,height,channels) + :param shapes: A list of points in the polygon from get_feature_info + :returns: a dict of masked polygons with the shapes filled in from cv2.fillPoly + """ + # For each WKT polygon, read the WKT format and fill the polygon as an image + masked_polys = {} + + for u in shapes: + sh = shapes[u] + mask_img = np.zeros(size, np.uint8) + i = fillPoly(mask_img, [sh], (255, 255, 255)) + masked_polys[u] = i + + return masked_polys + +def mask_polygons_together(size, shapes): + """ + :param size: A tuple of the (width,height,channels) + :param shapes: A list of points in the polygon from get_feature_info + :returns: A numpy array with the polygons filled 255s where there's a building and 0 where not + """ + # For each WKT polygon, read the WKT format and fill the polygon as an image + mask_img = np.zeros(size, np.uint8) + + for u in shapes: + blank = np.zeros(size, np.uint8) + poly = shapes[u] + fillPoly(blank, [poly], (1, 1, 1)) + mask_img += blank + + # Here we are taking the overlap (+=) and squashing it back to 0 + mask_img[mask_img > 1] = 0 + + # Finally we are taking all 1s and making it pure white (255) + mask_img[mask_img == 1] = 255 + + return mask_img + +def mask_polygons_together_with_border(size, shapes, border): + """ + :param size: A tuple of the (width,height,channels) + :param shapes: A list of points in the polygon from get_feature_info + :returns: a dict of masked polygons with the shapes filled in from cv2.fillPoly + """ + + # For each WKT polygon, read the WKT format and fill the polygon as an image + mask_img = np.zeros(size, np.uint8) + + for u in shapes: + blank = np.zeros(size, np.uint8) + # Each polygon stored in shapes is a np.ndarray + poly = shapes[u] + + # Creating a shapely polygon object out of the numpy array + polygon = Polygon(poly) + + # Getting the center points from the polygon and the polygon points + (poly_center_x, poly_center_y) = polygon.centroid.coords[0] + polygon_points = polygon.exterior.coords + + # Setting a new polygon with each X,Y manipulated based off the center point + shrunk_polygon = [] + for (x,y) in polygon_points: + if x < poly_center_x: + x += border + elif x > poly_center_x: + x -= border + + if y < poly_center_y: + y += border + elif y > poly_center_y: + y -= border + + shrunk_polygon.append([x,y]) + + # Transforming the polygon back to a np.ndarray + ns_poly = np.array(shrunk_polygon, np.int32) + + # Filling the shrunken polygon to add a border between close polygons + fillPoly(blank, [ns_poly], (1, 1, 1)) + mask_img += blank + + mask_img[mask_img > 1] = 0 + mask_img[mask_img == 1] = 255 + return mask_img + +def save_masks(masks, output_path, mask_file_name): + """ + :param masks: dictionary of UID:masked polygons from mask_polygons_separately() + :param output_path: path to save the masks + :param mask_file_name: the file name the masks should have + """ + # For each filled polygon, write out a separate file, increasing the name + for m in masks: + final_out = path.join(output_path, + mask_file_name + '_{}.png'.format(m)) + imwrite(final_out, masks[m]) + +def save_one_mask(masks, output_path, mask_file_name): + """ + :param masks: list of masked polygons from the mask_polygons_separately function + :param output_path: path to save the masks + :param mask_file_name: the file name the masks should have + """ + # For each filled polygon, write the mask shape out to the file per image + mask_file_name = path.join(output_path, mask_file_name + '.png') + imwrite(mask_file_name, masks) + + +def read_json(json_path): + """ + :param json_path: path to load json from + :returns: a python dictionary of json features + """ + annotations = json.load(open(json_path)) + return annotations + + +def get_feature_info(feature): + """ + :param feature: a python dictionary of json labels + :returns: a list mapping of polygons contained in the image + """ + # Getting each polygon points from the json file and adding it to a dictionary of uid:polygons + props = {} + + for feat in feature['features']['xy']: + feat_shape = wkt.loads(feat['wkt']) + coords = list(mapping(feat_shape)['coordinates'][0]) + props[feat['properties']['uid']] = (np.array(coords, np.int32)) + + return props + + +def mask_chips(json_path, images_directory, output_directory, single_file, border): + """ + :param json_path: path to find multiple json files for the chips + :param images_directory: path to the directory containing the images to be masked + :param output_directory: path to the directory where masks are to be saved + :param single_file: a boolean value to see if masks should be saved a single file or multiple + """ + # For each feature in the json we will create a separate mask + # Getting all files in the directory provided for jsons + jsons = [j for j in next(walk(json_path))[2] if '_pre' in j] + + # After removing non-json items in dir (if any) + for j in tqdm([j for j in jsons if j.endswith('json')], + unit='poly', + leave=False): + # Our chips start off in life as PNGs + chip_image_id = path.splitext(j)[0] + '.png' + mask_file = path.splitext(j)[0] + + # Loading the per chip json + j_full_path = path.join(json_path, j) + chip_json = read_json(j_full_path) + + # Getting the full chip path, and loading the size dimensions + chip_file = path.join(images_directory, chip_image_id) + chip_size = get_dimensions(chip_file) + + # Reading in the polygons from the json file + polys = get_feature_info(chip_json) + + # Getting a list of the polygons and saving masks as separate or single image files + if len(polys) > 0: + if single_file: + if border > 0: + masked_polys = mask_polygons_together_with_border(chip_size, polys, border) + else: + masked_polys = mask_polygons_together(chip_size, polys) + save_one_mask(masked_polys, output_directory, mask_file) + else: + masked_polys = mask_polygons_separately(chip_size, polys) + save_masks(masked_polys, output_directory, mask_file) + + +if __name__ == "__main__": + import argparse + + # Parse command line arguments + parser = argparse.ArgumentParser( + description= + """mask_polygons.py: Takes in xBD dataset and masks polygons in the image (make sure you've ran chip_masks.py first)\n\n + WARNING: This could lead to hundreds of output images per input\n""") + + parser.add_argument('--input', + required=True, + metavar="/path/to/xBD/", + help='Path to parent dataset directory "xBD"') + parser.add_argument('--single-file', + action='store_true', + help='use to save all masked polygon instances to a single file rather than one polygon per mask file') + parser.add_argument('--border', + default=0, + type=int, + metavar="positive integer for pixel border (e.g. 1)", + help='Positive integer used to shrink the polygon by') + + args = parser.parse_args() + + # Getting the list of the disaster types under the xBD directory + disasters = next(walk(args.input))[1] + + for disaster in tqdm(disasters, desc='Masking', unit='disaster'): + # Create the full path to the images, labels, and mask output directories + image_dir = path.join(args.input, disaster, 'images') + json_dir = path.join(args.input, disaster, 'labels') + output_dir = path.join(args.input, disaster, 'masks') + + if not path.isdir(image_dir): + print( + "Error, could not find image files in {}.\n\n" + .format(image_dir), + file=stderr) + exit(2) + + if not path.isdir(json_dir): + print( + "Error, could not find labels in {}.\n\n" + .format(json_dir), + file=stderr) + exit(3) + + if not path.isdir(output_dir): + makedirs(output_dir) + + mask_chips(json_dir, image_dir, output_dir, args.single_file, args.border) diff --git a/utils/view_polygons.ipynb b/utils/view_polygons.ipynb new file mode 100644 index 0000000..25b9098 --- /dev/null +++ b/utils/view_polygons.ipynb @@ -0,0 +1,367 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "#####################################################################################################################################################################\n", + "# xView2 #\n", + "# Copyright 2019 Carnegie Mellon University. #\n", + "# NO WARRANTY. THIS CARNEGIE MELLON UNIVERSITY AND SOFTWARE ENGINEERING INSTITUTE MATERIAL IS FURNISHED ON AN \"AS-IS\" BASIS. CARNEGIE MELLON UNIVERSITY MAKES NO #\n", + "# WARRANTIES OF ANY KIND, EITHER EXPRESSED OR IMPLIED, AS TO ANY MATTER INCLUDING, BUT NOT LIMITED TO, WARRANTY OF FITNESS FOR PURPOSE OR MERCHANTABILITY, # \n", + "# EXCLUSIVITY, OR RESULTS OBTAINED FROM USE OF THE MATERIAL. CARNEGIE MELLON UNIVERSITY DOES NOT MAKE ANY WARRANTY OF ANY KIND WITH RESPECT TO FREEDOM FROM PATENT, # \n", + "# TRADEMARK, OR COPYRIGHT INFRINGEMENT. #\n", + "# Released under a MIT (SEI)-style license, please see LICENSE.md or contact permission@sei.cmu.edu for full terms. #\n", + "# [DISTRIBUTION STATEMENT A] This material has been approved for public release and unlimited distribution. Please see Copyright notice for non-US Government use #\n", + "# and distribution. #\n", + "# This Software includes and/or makes use of the following Third-Party Software subject to its own license: #\n", + "# 1. SpaceNet (https://github.com/motokimura/spacenet_building_detection/blob/master/LICENSE) Copyright 2017 Motoki Kimura. #\n", + "# DM19-0988 #\n", + "#####################################################################################################################################################################" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "# To Run this notebook, start at the first cell with the license information and click run 4 times to show \n", + "# the field blocks, then input the *full path* to the label, and image. Finally, click \"Create next input\", \n", + "# you'll then see a full sized image with labels overlaid, you will also get different color labels if the\n", + "# label file as damage labels under ['features']['xy'][i]['properties']['subtype'] where i is the polygon \n", + "# in the ['xy'] list " + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "hidden": true + }, + "outputs": [], + "source": [ + "from ipywidgets import Layout\n", + "from IPython.display import Javascript, HTML\n", + "import ipywidgets as widgets\n", + "\n", + "def run_all(ev):\n", + " display(Javascript('IPython.notebook.execute_cells_below()'))\n", + "\n", + "path_to_label = widgets.Text(\n", + " placeholder='Label path here',\n", + " description='Label:',\n", + " disabled=False,\n", + " layout=Layout(width='100%')\n", + ")\n", + "path_to_image = widgets.Text(\n", + " placeholder='Image path here',\n", + " description='Image:',\n", + " disabled=False,\n", + " layout=Layout(width='100%')\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "The raw code for this IPython notebook is by default hidden for easier reading.\n", + "To toggle on/off the raw code, click here." + ], + "text/plain": [ + "" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\n", + "HTML('''\n", + "The raw code for this IPython notebook is by default hidden for easier reading.\n", + "To toggle on/off the raw code, click here.''')" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "application/javascript": [ + "IPython.notebook.execute_cells_below()" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "d2be852d23c1454ca6d8b7871ecf5eb5", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Text(value='/tmp/inference/inference.json', description='Label:', layout=Layout(width='100%'), placeholder='La…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "a4df1b7f08c6433eaba706e160b7956b", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Text(value='/Users/rthosfelt/Downloads/xBD/tuscaloosa-tornado/images/tuscaloosa-tornado_00000027_pre_disaster.…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "1bcd23b318194309bcee1c297b83c4c2", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Button(description='Create next input', style=ButtonStyle())" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "4df43ae86c7847c595ef8deab722b358", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Text(value='', description='Label:', layout=Layout(width='100%'), placeholder='Label path here')" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "6d55945ff4664d3bb3341993e0df13f7", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Text(value='', description='Image:', layout=Layout(width='100%'), placeholder='Image path here')" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "9fa49996441e4368bb3c477207577b08", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Button(description='Create next input', style=ButtonStyle())" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "display(path_to_label, path_to_image)\n", + "button = widgets.Button(description=\"Create next input\")\n", + "button.on_click(run_all)\n", + "display(button)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "import json \n", + "from PIL import Image, ImageDraw\n", + "from IPython.display import display\n", + "from shapely import wkt" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "ename": "FileNotFoundError", + "evalue": "[Errno 2] No such file or directory: ''", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mFileNotFoundError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0mpath_to_image_value\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpath_to_image\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath_to_label_value\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'rb'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mimage_json_file\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 6\u001b[0m \u001b[0mimage_json\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mjson\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mimage_json_file\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: ''" + ] + } + ], + "source": [ + "# Opening and loading polygons from label json \n", + "path_to_label_value = path_to_label.value\n", + "path_to_image_value = path_to_image.value\n", + "\n", + "with open(path_to_label_value, 'rb') as image_json_file:\n", + " image_json = json.load(image_json_file)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "coords = image_json['features']['xy']\n", + "wkt_polygons = []\n", + "\n", + "for coord in coords:\n", + " if 'subtype' in coord['properties']:\n", + " damage = coord['properties']['subtype']\n", + " else:\n", + " damage = 'no-damage'\n", + " wkt_polygons.append((damage, coord['wkt']))\n", + " \n", + "polygons = []\n", + "\n", + "for damage, swkt in wkt_polygons:\n", + " polygons.append((damage, wkt.loads(swkt)))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": false + }, + "outputs": [], + "source": [ + "# Loading image\n", + "img = Image.open(path_to_image_value)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "draw = ImageDraw.Draw(img, 'RGBA')\n", + "\n", + "damage_dict = {\n", + " \"no-damage\": (0, 255, 0, 100),\n", + " \"minor-damage\": (0, 0, 255, 125),\n", + " \"major-damage\": (255, 69, 0, 125),\n", + " \"destroyed\": (255, 0, 0, 125),\n", + " \"un-classified\": (255, 255, 255, 125)\n", + "}\n", + "\n", + "for damage, polygon in polygons:\n", + " x,y = polygon.exterior.coords.xy\n", + " coords = list(zip(x,y))\n", + " draw.polygon(coords, damage_dict[damage])\n", + "\n", + "del draw\n", + "\n", + "display(img)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.4" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/weights/mean.npy b/weights/mean.npy new file mode 100644 index 0000000..5320154 Binary files /dev/null and b/weights/mean.npy differ