diff options
Diffstat (limited to 'src/python')
-rw-r--r-- | src/python/CMakeLists.txt | 5 | ||||
-rw-r--r-- | src/python/doc/datasets.inc (renamed from src/python/doc/datasets_generators.inc) | 4 | ||||
-rw-r--r-- | src/python/doc/datasets.rst (renamed from src/python/doc/datasets_generators.rst) | 36 | ||||
-rw-r--r-- | src/python/doc/img/bunny.png | bin | 0 -> 48040 bytes | |||
-rw-r--r-- | src/python/doc/img/spiral_2d.png | bin | 0 -> 279276 bytes | |||
-rw-r--r-- | src/python/doc/index.rst | 6 | ||||
-rw-r--r-- | src/python/gudhi/datasets/remote.py | 223 | ||||
-rw-r--r-- | src/python/test/test_remote_datasets.py | 87 |
8 files changed, 352 insertions, 9 deletions
diff --git a/src/python/CMakeLists.txt b/src/python/CMakeLists.txt index af0b6115..c3768475 100644 --- a/src/python/CMakeLists.txt +++ b/src/python/CMakeLists.txt @@ -591,6 +591,11 @@ if(PYTHONINTERP_FOUND) add_gudhi_py_test(test_dtm_rips_complex) endif() + # Fetch remote datasets + if(WITH_GUDHI_REMOTE_TEST) + add_gudhi_py_test(test_remote_datasets) + endif() + # persistence graphical tools if(MATPLOTLIB_FOUND) add_gudhi_py_test(test_persistence_graphical_tools) diff --git a/src/python/doc/datasets_generators.inc b/src/python/doc/datasets.inc index 8d169275..95a87678 100644 --- a/src/python/doc/datasets_generators.inc +++ b/src/python/doc/datasets.inc @@ -2,7 +2,7 @@ :widths: 30 40 30 +-----------------------------------+--------------------------------------------+--------------------------------------------------------------------------------------+ - | .. figure:: | Datasets generators (points). | :Authors: Hind Montassif | + | .. figure:: | Datasets either generated or fetched. | :Authors: Hind Montassif | | img/sphere_3d.png | | | | | | :Since: GUDHI 3.5.0 | | | | | @@ -10,5 +10,5 @@ | | | | | | | :Requires: `CGAL <installation.html#cgal>`_ | +-----------------------------------+--------------------------------------------+--------------------------------------------------------------------------------------+ - | * :doc:`datasets_generators` | + | * :doc:`datasets` | +-----------------------------------+-----------------------------------------------------------------------------------------------------------------------------------+ diff --git a/src/python/doc/datasets_generators.rst b/src/python/doc/datasets.rst index 260c3882..2d11a19d 100644 --- a/src/python/doc/datasets_generators.rst +++ b/src/python/doc/datasets.rst @@ -3,12 +3,14 @@ .. To get rid of WARNING: document isn't included in any toctree -=========================== -Datasets generators manual -=========================== +================ +Datasets manual +================ -We provide the generation of different customizable datasets to use as inputs for Gudhi complexes and data structures. +Datasets generators +=================== +We provide the generation of different customizable datasets to use as inputs for Gudhi complexes and data structures. Points generators ------------------ @@ -103,3 +105,29 @@ Example .. autofunction:: gudhi.datasets.generators.points.torus + + +Fetching datasets +================= + +We provide some ready-to-use datasets that are not available by default when getting GUDHI, and need to be fetched explicitly. + +By **default**, the fetched datasets directory is set to a folder named **'gudhi_data'** in the **user home folder**. +Alternatively, it can be set using the **'GUDHI_DATA'** environment variable. + +.. autofunction:: gudhi.datasets.remote.fetch_bunny + +.. figure:: ./img/bunny.png + :figclass: align-center + + 3D Stanford bunny with 35947 vertices. + + +.. autofunction:: gudhi.datasets.remote.fetch_spiral_2d + +.. figure:: ./img/spiral_2d.png + :figclass: align-center + + 2D spiral with 114562 vertices. + +.. autofunction:: gudhi.datasets.remote.clear_data_home diff --git a/src/python/doc/img/bunny.png b/src/python/doc/img/bunny.png Binary files differnew file mode 100644 index 00000000..769aa530 --- /dev/null +++ b/src/python/doc/img/bunny.png diff --git a/src/python/doc/img/spiral_2d.png b/src/python/doc/img/spiral_2d.png Binary files differnew file mode 100644 index 00000000..abd247cd --- /dev/null +++ b/src/python/doc/img/spiral_2d.png diff --git a/src/python/doc/index.rst b/src/python/doc/index.rst index 2d7921ae..35f4ba46 100644 --- a/src/python/doc/index.rst +++ b/src/python/doc/index.rst @@ -92,7 +92,7 @@ Clustering .. include:: clustering.inc -Datasets generators -******************* +Datasets +******** -.. include:: datasets_generators.inc +.. include:: datasets.inc diff --git a/src/python/gudhi/datasets/remote.py b/src/python/gudhi/datasets/remote.py new file mode 100644 index 00000000..f6d3fe56 --- /dev/null +++ b/src/python/gudhi/datasets/remote.py @@ -0,0 +1,223 @@ +# This file is part of the Gudhi Library - https://gudhi.inria.fr/ - which is released under MIT. +# See file LICENSE or go to https://gudhi.inria.fr/licensing/ for full license details. +# Author(s): Hind Montassif +# +# Copyright (C) 2021 Inria +# +# Modification(s): +# - YYYY/MM Author: Description of the modification + +from os.path import join, split, exists, expanduser +from os import makedirs, remove, environ + +from urllib.request import urlretrieve +import hashlib +import shutil + +import numpy as np + +def _get_data_home(data_home = None): + """ + Return the path of the remote datasets directory. + This folder is used to store remotely fetched datasets. + By default the datasets directory is set to a folder named 'gudhi_data' in the user home folder. + Alternatively, it can be set by the 'GUDHI_DATA' environment variable. + The '~' symbol is expanded to the user home folder. + If the folder does not already exist, it is automatically created. + + Parameters + ---------- + data_home : string + The path to remote datasets directory. + Default is `None`, meaning that the data home directory will be set to "~/gudhi_data", + if the 'GUDHI_DATA' environment variable does not exist. + + Returns + ------- + data_home: string + The path to remote datasets directory. + """ + if data_home is None: + data_home = environ.get("GUDHI_DATA", join("~", "gudhi_data")) + data_home = expanduser(data_home) + makedirs(data_home, exist_ok=True) + return data_home + + +def clear_data_home(data_home = None): + """ + Delete the data home cache directory and all its content. + + Parameters + ---------- + data_home : string, default is None. + The path to remote datasets directory. + If `None` and the 'GUDHI_DATA' environment variable does not exist, + the default directory to be removed is set to "~/gudhi_data". + """ + data_home = _get_data_home(data_home) + shutil.rmtree(data_home) + +def _checksum_sha256(file_path): + """ + Compute the file checksum using sha256. + + Parameters + ---------- + file_path: string + Full path of the created file including filename. + + Returns + ------- + The hex digest of file_path. + """ + sha256_hash = hashlib.sha256() + chunk_size = 4096 + with open(file_path,"rb") as f: + # Read and update hash string value in blocks of 4K + while True: + buffer = f.read(chunk_size) + if not buffer: + break + sha256_hash.update(buffer) + return sha256_hash.hexdigest() + +def _fetch_remote(url, file_path, file_checksum = None): + """ + Fetch the wanted dataset from the given url and save it in file_path. + + Parameters + ---------- + url : string + The url to fetch the dataset from. + file_path : string + Full path of the downloaded file including filename. + file_checksum : string + The file checksum using sha256 to check against the one computed on the downloaded file. + Default is 'None', which means the checksum is not checked. + + Raises + ------ + IOError + If the computed SHA256 checksum of file does not match the one given by the user. + """ + + # Get the file + urlretrieve(url, file_path) + + if file_checksum is not None: + checksum = _checksum_sha256(file_path) + if file_checksum != checksum: + # Remove file and raise error + remove(file_path) + raise IOError("{} has a SHA256 checksum : {}, " + "different from expected : {}." + "The file may be corrupted or the given url may be wrong !".format(file_path, checksum, file_checksum)) + +def _get_archive_path(file_path, label): + """ + Get archive path based on file_path given by user and label. + + Parameters + ---------- + file_path: string + Full path of the file to get including filename, or None. + label: string + Label used along with 'data_home' to get archive path, in case 'file_path' is None. + + Returns + ------- + Full path of archive including filename. + """ + if file_path is None: + archive_path = join(_get_data_home(), label) + dirname = split(archive_path)[0] + makedirs(dirname, exist_ok=True) + else: + archive_path = file_path + dirname = split(archive_path)[0] + makedirs(dirname, exist_ok=True) + + return archive_path + +def fetch_spiral_2d(file_path = None): + """ + Load the spiral_2d dataset. + + Note that if the dataset already exists in the target location, it is not downloaded again, + and the corresponding array is returned from cache. + + Parameters + ---------- + file_path : string + Full path of the downloaded file including filename. + + Default is None, meaning that it's set to "data_home/points/spiral_2d/spiral_2d.npy". + + The "data_home" directory is set by default to "~/gudhi_data", + unless the 'GUDHI_DATA' environment variable is set. + + Returns + ------- + points: numpy array + Array of shape (114562, 2). + """ + file_url = "https://raw.githubusercontent.com/GUDHI/gudhi-data/main/points/spiral_2d/spiral_2d.npy" + file_checksum = '2226024da76c073dd2f24b884baefbfd14928b52296df41ad2d9b9dc170f2401' + + archive_path = _get_archive_path(file_path, "points/spiral_2d/spiral_2d.npy") + + if not exists(archive_path): + _fetch_remote(file_url, archive_path, file_checksum) + + return np.load(archive_path, mmap_mode='r') + +def fetch_bunny(file_path = None, accept_license = False): + """ + Load the Stanford bunny dataset. + + This dataset contains 35947 vertices. + + Note that if the dataset already exists in the target location, it is not downloaded again, + and the corresponding array is returned from cache. + + Parameters + ---------- + file_path : string + Full path of the downloaded file including filename. + + Default is None, meaning that it's set to "data_home/points/bunny/bunny.npy". + In this case, the LICENSE file would be downloaded as "data_home/points/bunny/bunny.LICENSE". + + The "data_home" directory is set by default to "~/gudhi_data", + unless the 'GUDHI_DATA' environment variable is set. + + accept_license : boolean + Flag to specify if user accepts the file LICENSE and prevents from printing the corresponding license terms. + + Default is False. + + Returns + ------- + points: numpy array + Array of shape (35947, 3). + """ + + file_url = "https://raw.githubusercontent.com/GUDHI/gudhi-data/main/points/bunny/bunny.npy" + file_checksum = 'f382482fd89df8d6444152dc8fd454444fe597581b193fd139725a85af4a6c6e' + license_url = "https://raw.githubusercontent.com/GUDHI/gudhi-data/main/points/bunny/bunny.LICENSE" + license_checksum = 'b763dbe1b2fc6015d05cbf7bcc686412a2eb100a1f2220296e3b4a644c69633a' + + archive_path = _get_archive_path(file_path, "points/bunny/bunny.npy") + + if not exists(archive_path): + _fetch_remote(file_url, archive_path, file_checksum) + license_path = join(split(archive_path)[0], "bunny.LICENSE") + _fetch_remote(license_url, license_path, license_checksum) + # Print license terms unless accept_license is set to True + if not accept_license: + if exists(license_path): + with open(license_path, 'r') as f: + print(f.read()) + + return np.load(archive_path, mmap_mode='r') diff --git a/src/python/test/test_remote_datasets.py b/src/python/test/test_remote_datasets.py new file mode 100644 index 00000000..e5d2de82 --- /dev/null +++ b/src/python/test/test_remote_datasets.py @@ -0,0 +1,87 @@ +# This file is part of the Gudhi Library - https://gudhi.inria.fr/ - which is released under MIT. +# See file LICENSE or go to https://gudhi.inria.fr/licensing/ for full license details. +# Author(s): Hind Montassif +# +# Copyright (C) 2021 Inria +# +# Modification(s): +# - YYYY/MM Author: Description of the modification + +from gudhi.datasets import remote + +import shutil +import io +import sys +import pytest + +from os.path import isdir, expanduser, exists +from os import remove, environ + +def test_data_home(): + # Test _get_data_home and clear_data_home on new empty folder + empty_data_home = remote._get_data_home(data_home="empty_folder_for_test") + assert isdir(empty_data_home) + + remote.clear_data_home(data_home=empty_data_home) + assert not isdir(empty_data_home) + +def test_fetch_remote(): + # Test fetch with a wrong checksum + with pytest.raises(OSError): + remote._fetch_remote("https://raw.githubusercontent.com/GUDHI/gudhi-data/main/points/spiral_2d/spiral_2d.npy", "tmp_spiral_2d.npy", file_checksum = 'XXXXXXXXXX') + assert not exists("tmp_spiral_2d.npy") + +def _get_bunny_license_print(accept_license = False): + capturedOutput = io.StringIO() + # Redirect stdout + sys.stdout = capturedOutput + + bunny_arr = remote.fetch_bunny("./tmp_for_test/bunny.npy", accept_license) + assert bunny_arr.shape == (35947, 3) + del bunny_arr + remove("./tmp_for_test/bunny.npy") + + # Reset redirect + sys.stdout = sys.__stdout__ + return capturedOutput + +def test_print_bunny_license(): + # Test not printing bunny.npy LICENSE when accept_license = True + assert "" == _get_bunny_license_print(accept_license = True).getvalue() + # Test printing bunny.LICENSE file when fetching bunny.npy with accept_license = False (default) + with open("./tmp_for_test/bunny.LICENSE") as f: + assert f.read().rstrip("\n") == _get_bunny_license_print().getvalue().rstrip("\n") + shutil.rmtree("./tmp_for_test") + +def test_fetch_remote_datasets_wrapped(): + # Test fetch_spiral_2d and fetch_bunny wrapping functions with data directory different from default (twice, to test case of already fetched files) + # Default case is not tested because it would fail in case the user sets the 'GUDHI_DATA' environment variable locally + for i in range(2): + spiral_2d_arr = remote.fetch_spiral_2d("./another_fetch_folder_for_test/spiral_2d.npy") + assert spiral_2d_arr.shape == (114562, 2) + + bunny_arr = remote.fetch_bunny("./another_fetch_folder_for_test/bunny.npy") + assert bunny_arr.shape == (35947, 3) + + # Check that the directory was created + assert isdir("./another_fetch_folder_for_test") + # Check downloaded files + assert exists("./another_fetch_folder_for_test/spiral_2d.npy") + assert exists("./another_fetch_folder_for_test/bunny.npy") + assert exists("./another_fetch_folder_for_test/bunny.LICENSE") + + # Remove test folders + del spiral_2d_arr + del bunny_arr + shutil.rmtree("./another_fetch_folder_for_test") + +def test_gudhi_data_env(): + # Set environment variable "GUDHI_DATA" + environ["GUDHI_DATA"] = "./test_folder_from_env_var" + bunny_arr = remote.fetch_bunny() + assert bunny_arr.shape == (35947, 3) + assert exists("./test_folder_from_env_var/points/bunny/bunny.npy") + assert exists("./test_folder_from_env_var/points/bunny/bunny.LICENSE") + # Remove test folder + del bunny_arr + shutil.rmtree("./test_folder_from_env_var") |