Add CropHarvest Dataset

docs/api/datasets.rst

@@ -206,6 +206,11 @@ COWC
 .. autoclass:: COWCCounting
 .. autoclass:: COWCDetection
 
+CropHarvest
+^^^^^^^^^^^
+
+.. autoclass:: CropHarvest
+
 Kenya Crop Type
 ^^^^^^^^^^^^^^^
 

docs/api/non_geo_datasets.csv

@@ -6,6 +6,7 @@ Dataset,Task,Source,License,# Samples,# Classes,Size (px),Resolution (m),Bands
 `ChaBuD`_,CD,Sentinel-2,"OpenRAIL",356,2,512x512,10,MSI
 `Cloud Cover Detection`_,S,Sentinel-2,"CC-BY-4.0","22,728",2,512x512,10,MSI
 `COWC`_,"C, R","CSUAV AFRL, ISPRS, LINZ, AGRC","AGPL-3.0-only","388,435",2,256x256,0.15,RGB
+`CropHarvest`_,"C","Sentinel-1/2, SRTM, ERA5","CC-BY-SA-4.0","70,213",349,1x1,10,"SAR, MSI, SRTM"
 `Kenya Crop Type`_,S,Sentinel-2,"CC-BY-SA-4.0","4,688",7,"3,035x2,016",10,MSI
 `DeepGlobe Land Cover`_,S,DigitalGlobe +Vivid,-,803,7,"2,448x2,448",0.5,RGB
 `DFC2022`_,S,Aerial,"CC-BY-4.0","3,981",15,"2,000x2,000",0.5,RGB
@@ -49,4 +50,4 @@ Dataset,Task,Source,License,# Samples,# Classes,Size (px),Resolution (m),Bands
 `VHR-10`_,I,"Google Earth, Vaihingen","MIT",800,10,"358--1,728",0.08--2,RGB
 `Western USA Live Fuel Moisture`_,R,"Landsat8, Sentinel-1","CC-BY-NC-ND-4.0",2615,-,-,-,-
 `xView2`_,CD,Maxar,"CC-BY-NC-SA-4.0","3,732",4,"1,024x1,024",0.8,RGB
-`ZueriCrop`_,"I, T",Sentinel-2,-,116K,48,24x24,10,MSI
+`ZueriCrop`_,"I, T",Sentinel-2,-,116K,48,24x24,10,MSI

tests/data/cropharvest/data.py

@@ -0,0 +1,153 @@
+#!/usr/bin/env python3
+
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+import hashlib
+import json
+import os
+import shutil
+
+import h5py
+import numpy as np
+
+SIZE = 32
+
+np.random.seed(0)
+
+PATHS = [
+    os.path.join("cropharvest", "features", "arrays", "0_TestDataset1.h5"),
+    os.path.join("cropharvest", "features", "arrays", "1_TestDataset1.h5"),
+    os.path.join("cropharvest", "features", "arrays", "2_TestDataset1.h5"),
+    os.path.join("cropharvest", "features", "arrays", "0_TestDataset2.h5"),
+    os.path.join("cropharvest", "features", "arrays", "1_TestDataset2.h5"),
+]
+
+
+def create_geojson():
+    geojson = {
+        "type": "FeatureCollection",
+        "crs": {},
+        "features": [
+            {
+                "type": "Feature",
+                "properties": {
+                    "dataset": "TestDataset1",
+                    "index": 0,
+                    "is_crop": 1,
+                    "label": "soybean",
+                },
+                "geometry": {
+                    "type": "Polygon",
+                    "coordinates": [
+                        [[0.0, 0.0], [0.0, 1.0], [1.0, 1.0], [1.0, 0.0], [0.0, 0.0]]
+                    ],
+                },
+            },
+            {
+                "type": "Feature",
+                "properties": {
+                    "dataset": "TestDataset1",
+                    "index": 0,
+                    "is_crop": 1,
+                    "label": "alfalfa",
+                },
+                "geometry": {
+                    "type": "Polygon",
+                    "coordinates": [
+                        [[0.0, 0.0], [0.0, 1.0], [1.0, 1.0], [1.0, 0.0], [0.0, 0.0]]
+                    ],
+                },
+            },
+            {
+                "type": "Feature",
+                "properties": {
+                    "dataset": "TestDataset1",
+                    "index": 1,
+                    "is_crop": 1,
+                    "label": None,
+                },
+                "geometry": {
+                    "type": "Polygon",
+                    "coordinates": [
+                        [[0.0, 0.0], [0.0, 1.0], [1.0, 1.0], [1.0, 0.0], [0.0, 0.0]]
+                    ],
+                },
+            },
+            {
+                "type": "Feature",
+                "properties": {
+                    "dataset": "TestDataset2",
+                    "index": 2,
+                    "is_crop": 1,
+                    "label": "maize",
+                },
+                "geometry": {
+                    "type": "Polygon",
+                    "coordinates": [
+                        [[0.0, 0.0], [0.0, 1.0], [1.0, 1.0], [1.0, 0.0], [0.0, 0.0]]
+                    ],
+                },
+            },
+            {
+                "type": "Feature",
+                "properties": {
+                    "dataset": "TestDataset2",
+                    "index": 1,
+                    "is_crop": 0,
+                    "label": None,
+                },
+                "geometry": {
+                    "type": "Polygon",
+                    "coordinates": [
+                        [[0.0, 0.0], [0.0, 1.0], [1.0, 1.0], [1.0, 0.0], [0.0, 0.0]]
+                    ],
+                },
+            },
+        ],
+    }
+    return geojson
+
+
+def create_file(path: str) -> None:
+    Z = np.random.randint(4000, size=(12, 18), dtype=np.int64)
+    with h5py.File(path, "w") as f:
+        f.create_dataset("array", data=Z)
+
+
+if __name__ == "__main__":
+    directory = "cropharvest"
+
+    # remove old data
+    to_remove = [
+        os.path.join(directory, "features"),
+        os.path.join(directory, "features.tar.gz"),
+        os.path.join(directory, "labels.geojson"),
+    ]
+    for path in to_remove:
+        if os.path.isdir(path):
+            shutil.rmtree(path)
+
+    label_path = os.path.join(directory, "labels.geojson")
+    geojson = create_geojson()
+    os.makedirs(os.path.dirname(label_path), exist_ok=True)
+
+    with open(label_path, "w") as f:
+        json.dump(geojson, f)
+
+    for path in PATHS:
+        os.makedirs(os.path.dirname(path), exist_ok=True)
+        create_file(path)
+
+    # compress data
+    source_dir = os.path.join(directory, "features")
+    shutil.make_archive(source_dir, "gztar", directory, "features")
+
+    # compute checksum
+    with open(label_path, "rb") as f:
+        md5 = hashlib.md5(f.read()).hexdigest()
+        print(f"{label_path}: {md5}")
+
+    with open(os.path.join(directory, "features.tar.gz"), "rb") as f:
+        md5 = hashlib.md5(f.read()).hexdigest()
+        print(f"zipped features: {md5}")

tests/data/cropharvest/labels.geojson

@@ -0,0 +1 @@
+{"type": "FeatureCollection", "crs": {}, "features": [{"type": "Feature", "properties": {"dataset": "TestDataset1", "index": 0, "is_crop": 1, "label": "soybean"}, "geometry": {"type": "Polygon", "coordinates": [[[0.0, 0.0], [0.0, 1.0], [1.0, 1.0], [1.0, 0.0], [0.0, 0.0]]]}}, {"type": "Feature", "properties": {"dataset": "TestDataset1", "index": 0, "is_crop": 1, "label": "alfalfa"}, "geometry": {"type": "Polygon", "coordinates": [[[0.0, 0.0], [0.0, 1.0], [1.0, 1.0], [1.0, 0.0], [0.0, 0.0]]]}}, {"type": "Feature", "properties": {"dataset": "TestDataset1", "index": 1, "is_crop": 1, "label": null}, "geometry": {"type": "Polygon", "coordinates": [[[0.0, 0.0], [0.0, 1.0], [1.0, 1.0], [1.0, 0.0], [0.0, 0.0]]]}}, {"type": "Feature", "properties": {"dataset": "TestDataset2", "index": 2, "is_crop": 1, "label": "maize"}, "geometry": {"type": "Polygon", "coordinates": [[[0.0, 0.0], [0.0, 1.0], [1.0, 1.0], [1.0, 0.0], [0.0, 0.0]]]}}, {"type": "Feature", "properties": {"dataset": "TestDataset2", "index": 1, "is_crop": 0, "label": null}, "geometry": {"type": "Polygon", "coordinates": [[[0.0, 0.0], [0.0, 1.0], [1.0, 1.0], [1.0, 0.0], [0.0, 0.0]]]}}]}

tests/datasets/test_cropharvest.py

@@ -0,0 +1,100 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+import builtins
+import os
+import shutil
+from pathlib import Path
+from typing import Any
+
+import matplotlib.pyplot as plt
+import pytest
+import torch
+import torch.nn as nn
+from pytest import MonkeyPatch
+
+import torchgeo.datasets.utils
+from torchgeo.datasets import CropHarvest, DatasetNotFoundError
+
+pytest.importorskip("h5py", minversion="3")
+
+
+def download_url(url: str, root: str, filename: str, md5: str) -> None:
+    shutil.copy(url, os.path.join(root, filename))
+
+
+class TestCropHarvest:
+    @pytest.fixture
+    def mock_missing_module(self, monkeypatch: MonkeyPatch) -> None:
+        import_orig = builtins.__import__
+
+        def mocked_import(name: str, *args: Any, **kwargs: Any) -> Any:
+            if name == "h5py":
+                raise ImportError()
+            return import_orig(name, *args, **kwargs)
+
+        monkeypatch.setattr(builtins, "__import__", mocked_import)
+
+    @pytest.fixture
+    def dataset(self, monkeypatch: MonkeyPatch, tmp_path: Path) -> CropHarvest:
+        monkeypatch.setattr(torchgeo.datasets.cropharvest, "download_url", download_url)
+        monkeypatch.setitem(
+            CropHarvest.file_dict["features"], "md5", "ef6f4f00c0b3b50ed8380b0044928572"
+        )
+        monkeypatch.setitem(
+            CropHarvest.file_dict["labels"], "md5", "1d93b6bfcec7b6797b75acbd9d284b92"
+        )
+        monkeypatch.setitem(
+            CropHarvest.file_dict["features"],
+            "url",
+            os.path.join("tests", "data", "cropharvest", "features.tar.gz"),
+        )
+        monkeypatch.setitem(
+            CropHarvest.file_dict["labels"],
+            "url",
+            os.path.join("tests", "data", "cropharvest", "labels.geojson"),
+        )
+
+        root = str(tmp_path)
+        transforms = nn.Identity()
+
+        dataset = CropHarvest(root, transforms, download=True, checksum=True)
+        return dataset
+
+    def test_getitem(self, dataset: CropHarvest) -> None:
+        x = dataset[0]
+        assert isinstance(x, dict)
+        assert isinstance(x["array"], torch.Tensor)
+        assert isinstance(x["label"], torch.Tensor)
+        assert x["array"].shape == (12, 18)
+        y = dataset[2]
+        assert y["label"] == 1
+
+    def test_len(self, dataset: CropHarvest) -> None:
+        assert len(dataset) == 5
+
+    def test_already_downloaded(self, dataset: CropHarvest, tmp_path: Path) -> None:
+        CropHarvest(root=str(tmp_path), download=False)
+
+    def test_downloaded_zipped(self, dataset: CropHarvest, tmp_path: Path) -> None:
+        feature_path = os.path.join(tmp_path, "features")
+        shutil.rmtree(feature_path)
+        CropHarvest(root=str(tmp_path), download=True)
+
+    def test_not_downloaded(self, tmp_path: Path) -> None:
+        with pytest.raises(DatasetNotFoundError, match="Dataset not found"):
+            CropHarvest(str(tmp_path))
+
+    def test_plot(self, dataset: CropHarvest) -> None:
+        x = dataset[0].copy()
+        dataset.plot(x, subtitle="Test")
+        plt.close()
+
+    def test_mock_missing_module(
+        self, dataset: CropHarvest, tmp_path: Path, mock_missing_module: None
+    ) -> None:
+        with pytest.raises(
+            ImportError,
+            match="h5py is not installed and is required to use this dataset",
+        ):
+            CropHarvest(root=str(tmp_path), download=True)[0]

torchgeo/datasets/__init__.py

@@ -28,6 +28,7 @@
 from .cloud_cover import CloudCoverDetection
 from .cms_mangrove_canopy import CMSGlobalMangroveCanopy
 from .cowc import COWC, COWCCounting, COWCDetection
+from .cropharvest import CropHarvest
 from .cv4a_kenya_crop_type import CV4AKenyaCropType
 from .cyclone import TropicalCyclone
 from .deepglobelandcover import DeepGlobeLandCover
@@ -150,6 +151,7 @@
     "ChesapeakeWV",
     "ChesapeakeCVPR",
     "CMSGlobalMangroveCanopy",
+    "CropHarvest",
     "EDDMapS",
     "Esri2020",
     "EUDEM",