diff --git a/megaflow/__pycache__/__init__.cpython-310.pyc b/megaflow/__pycache__/__init__.cpython-310.pyc index 8a8efc2..f9c827b 100644 Binary files a/megaflow/__pycache__/__init__.cpython-310.pyc and b/megaflow/__pycache__/__init__.cpython-310.pyc differ diff --git a/megaflow/common/__pycache__/__init__.cpython-310.pyc b/megaflow/common/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000..deca66d Binary files /dev/null and b/megaflow/common/__pycache__/__init__.cpython-310.pyc differ diff --git a/megaflow/common/__pycache__/utils.cpython-310.pyc b/megaflow/common/__pycache__/utils.cpython-310.pyc new file mode 100644 index 0000000..94be7f8 Binary files /dev/null and b/megaflow/common/__pycache__/utils.cpython-310.pyc differ diff --git a/megaflow/dataset/MegaFlow2D.py b/megaflow/dataset/MegaFlow2D.py index 6278a1c..7d40731 100644 --- a/megaflow/dataset/MegaFlow2D.py +++ b/megaflow/dataset/MegaFlow2D.py @@ -4,6 +4,7 @@ import multiprocessing as mp from threading import Thread import h5py +from zipfile import ZipFile import torch from torch import Tensor @@ -33,7 +34,9 @@ def __init__(self, root, download, transform, pre_transform, split_scheme='mixed self.transforms = transform self.pre_transform = pre_transform if download: - self.download() + self.download() + # give a warning that the package does not check the integrity of the downloaded data + Warning('The package does not check the integrity of the downloaded data. The downloading operation is always executed if the flag download is True. Please disable the download flag if you have already downloaded the data') self.data_list = self.get_data_list # self.processed_las_data_dir = os.path.join(self.root, 'processed', 'las') # self.processed_has_data_dir = os.path.join(self.root, 'processed', 'has') @@ -104,12 +107,27 @@ def len(self): return 0 else: return len(self.data_list) + + def _extract_zip(self, path, folder): + zips = ["data.zip.00{}".format(i) for i in range(1, 6)] + + with open(os.path.join(path, "data.zip"), "ab") as f: + for zipName in zips: + with open(os.path.join(path, zipName), "rb") as z: + f.write(z.read()) + + z.close() + os.remove(os.path.join(path, zipName)) + + with ZipFile(os.path.join(path, "data.zip"), "r") as zipObj: + zipObj.extractall(folder) + os.remove(os.path.join(path, "data.zip")) def download(self): for i in range(1, 6): url = 'https://huggingface.co/datasets/cmudrc/MegaFlow2D/resolve/main/data.zip.00{}'.format(i) path = download_url(url, self.root) - extract_zip(path, self.root) + self._extract_zip(self.root, self.root) def process(self): # Read mesh solution into graph structure diff --git a/megaflow/dataset/__pycache__/MegaFlow2D.cpython-310.pyc b/megaflow/dataset/__pycache__/MegaFlow2D.cpython-310.pyc index d01f5fc..4f0b03f 100644 Binary files a/megaflow/dataset/__pycache__/MegaFlow2D.cpython-310.pyc and b/megaflow/dataset/__pycache__/MegaFlow2D.cpython-310.pyc differ diff --git a/megaflow/dataset/__pycache__/__init__.cpython-310.pyc b/megaflow/dataset/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000..3ad36d3 Binary files /dev/null and b/megaflow/dataset/__pycache__/__init__.cpython-310.pyc differ diff --git a/test.py b/test.py index 0c489e6..2889ab9 100644 --- a/test.py +++ b/test.py @@ -1,4 +1,5 @@ from megaflow.dataset.MegaFlow2D import MegaFlow2D -# Create a dataset object -dataset = MegaFlow2D(root='D:/Work/data', download=True) \ No newline at end of file +if __name__ == '__main__': + # Create a dataset object + dataset = MegaFlow2D(root='D:/Work/data', download=True, transform='normalize', pre_transform=None) \ No newline at end of file