summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/cmake/modules/GUDHI_options.cmake11
-rw-r--r--src/common/doc/installation.h2
-rw-r--r--src/python/CMakeLists.txt4
-rw-r--r--src/python/doc/datasets.inc (renamed from src/python/doc/datasets_generators.inc)4
-rw-r--r--src/python/doc/datasets.rst (renamed from src/python/doc/datasets_generators.rst)21
-rw-r--r--src/python/doc/index.rst6
-rw-r--r--src/python/gudhi/datasets/remote.py157
-rw-r--r--src/python/test/test_remote_datasets.py86
8 files changed, 277 insertions, 14 deletions
diff --git a/src/cmake/modules/GUDHI_options.cmake b/src/cmake/modules/GUDHI_options.cmake
index 3cd0a489..c75b72f5 100644
--- a/src/cmake/modules/GUDHI_options.cmake
+++ b/src/cmake/modules/GUDHI_options.cmake
@@ -1,5 +1,6 @@
-option(WITH_GUDHI_BENCHMARK "Activate/desactivate benchmark compilation" OFF)
-option(WITH_GUDHI_EXAMPLE "Activate/desactivate examples compilation and installation" OFF)
-option(WITH_GUDHI_PYTHON "Activate/desactivate python module compilation and installation" ON)
-option(WITH_GUDHI_TEST "Activate/desactivate examples compilation and installation" ON)
-option(WITH_GUDHI_UTILITIES "Activate/desactivate utilities compilation and installation" ON)
+option(WITH_GUDHI_BENCHMARK "Activate/deactivate benchmark compilation" OFF)
+option(WITH_GUDHI_EXAMPLE "Activate/deactivate examples compilation and installation" OFF)
+option(WITH_GUDHI_REMOTE_TEST "Activate/deactivate datasets fetching test which uses the Internet" OFF)
+option(WITH_GUDHI_PYTHON "Activate/deactivate python module compilation and installation" ON)
+option(WITH_GUDHI_TEST "Activate/deactivate examples compilation and installation" ON)
+option(WITH_GUDHI_UTILITIES "Activate/deactivate utilities compilation and installation" ON)
diff --git a/src/common/doc/installation.h b/src/common/doc/installation.h
index ef668dfb..6c42fbb4 100644
--- a/src/common/doc/installation.h
+++ b/src/common/doc/installation.h
@@ -40,6 +40,8 @@ make \endverbatim
* `make test` is using <a href="https://cmake.org/cmake/help/latest/manual/ctest.1.html">Ctest</a> (CMake test driver
* program). If some of the tests are failing, please send us the result of the following command:
* \verbatim ctest --output-on-failure \endverbatim
+ * Testing fetching datasets feature requires the use of the internet and is disabled by default. If you want to include this test, set WITH_GUDHI_REMOTE_TEST to ON when building in the previous step (note that this test is included in the python module):
+ * \verbatim cmake -DCMAKE_BUILD_TYPE=Release -DWITH_GUDHI_TEST=ON -DWITH_GUDHI_REMOTE_TEST=ON --DWITH_GUDHI_PYTHON=ON .. \endverbatim
*
* \subsection documentationgeneration Documentation
* To generate the documentation, <a target="_blank" href="http://www.doxygen.org/">Doxygen</a> is required.
diff --git a/src/python/CMakeLists.txt b/src/python/CMakeLists.txt
index 8eb7478e..e3119cfc 100644
--- a/src/python/CMakeLists.txt
+++ b/src/python/CMakeLists.txt
@@ -580,6 +580,10 @@ if(PYTHONINTERP_FOUND)
add_gudhi_py_test(test_dtm_rips_complex)
endif()
+ # Fetch remote datasets
+ if(WITH_GUDHI_REMOTE_TEST)
+ add_gudhi_py_test(test_remote_datasets)
+ endif()
# Set missing or not modules
set(GUDHI_MODULES ${GUDHI_MODULES} "python" CACHE INTERNAL "GUDHI_MODULES")
diff --git a/src/python/doc/datasets_generators.inc b/src/python/doc/datasets.inc
index 8d169275..95a87678 100644
--- a/src/python/doc/datasets_generators.inc
+++ b/src/python/doc/datasets.inc
@@ -2,7 +2,7 @@
:widths: 30 40 30
+-----------------------------------+--------------------------------------------+--------------------------------------------------------------------------------------+
- | .. figure:: | Datasets generators (points). | :Authors: Hind Montassif |
+ | .. figure:: | Datasets either generated or fetched. | :Authors: Hind Montassif |
| img/sphere_3d.png | | |
| | | :Since: GUDHI 3.5.0 |
| | | |
@@ -10,5 +10,5 @@
| | | |
| | | :Requires: `CGAL <installation.html#cgal>`_ |
+-----------------------------------+--------------------------------------------+--------------------------------------------------------------------------------------+
- | * :doc:`datasets_generators` |
+ | * :doc:`datasets` |
+-----------------------------------+-----------------------------------------------------------------------------------------------------------------------------------+
diff --git a/src/python/doc/datasets_generators.rst b/src/python/doc/datasets.rst
index 260c3882..4fa8a628 100644
--- a/src/python/doc/datasets_generators.rst
+++ b/src/python/doc/datasets.rst
@@ -3,12 +3,14 @@
.. To get rid of WARNING: document isn't included in any toctree
-===========================
-Datasets generators manual
-===========================
+================
+Datasets manual
+================
-We provide the generation of different customizable datasets to use as inputs for Gudhi complexes and data structures.
+Datasets generators
+===================
+We provide the generation of different customizable datasets to use as inputs for Gudhi complexes and data structures.
Points generators
------------------
@@ -103,3 +105,14 @@ Example
.. autofunction:: gudhi.datasets.generators.points.torus
+
+
+Fetching datasets
+=================
+
+We provide some ready-to-use datasets that are not available by default when getting GUDHI, and need to be fetched explicitly.
+
+.. automodule:: gudhi.datasets.remote
+ :members:
+ :special-members:
+ :show-inheritance:
diff --git a/src/python/doc/index.rst b/src/python/doc/index.rst
index 2d7921ae..35f4ba46 100644
--- a/src/python/doc/index.rst
+++ b/src/python/doc/index.rst
@@ -92,7 +92,7 @@ Clustering
.. include:: clustering.inc
-Datasets generators
-*******************
+Datasets
+********
-.. include:: datasets_generators.inc
+.. include:: datasets.inc
diff --git a/src/python/gudhi/datasets/remote.py b/src/python/gudhi/datasets/remote.py
new file mode 100644
index 00000000..ef797417
--- /dev/null
+++ b/src/python/gudhi/datasets/remote.py
@@ -0,0 +1,157 @@
+# This file is part of the Gudhi Library - https://gudhi.inria.fr/ - which is released under MIT.
+# See file LICENSE or go to https://gudhi.inria.fr/licensing/ for full license details.
+# Author(s): Hind Montassif
+#
+# Copyright (C) 2021 Inria
+#
+# Modification(s):
+# - YYYY/MM Author: Description of the modification
+
+from os.path import join, exists
+from os import makedirs
+
+from urllib.request import urlretrieve
+import hashlib
+
+import numpy as np
+
+def _checksum_sha256(file_path):
+ """
+ Compute the file checksum using sha256.
+
+ Parameters
+ ----------
+ file_path: string
+ Full path of the created file.
+
+ Returns
+ -------
+ The hex digest of file_path.
+ """
+ sha256_hash = hashlib.sha256()
+ chunk_size = 4096
+ with open(file_path,"rb") as f:
+ # Read and update hash string value in blocks of 4K
+ while True:
+ buffer = f.read(chunk_size)
+ if not buffer:
+ break
+ sha256_hash.update(buffer)
+ return sha256_hash.hexdigest()
+
+def _fetch_remote(url, filename, dirname = "remote_datasets", file_checksum = None, accept_license = False):
+ """
+ Fetch the wanted dataset from the given url and save it in file_path.
+
+ Parameters
+ ----------
+ url : string
+ The url to fetch the dataset from.
+ filename : string
+ The name to give to downloaded file.
+ dirname : string
+ The directory to save the file to. Default is "remote_datasets".
+ file_checksum : string
+ The file checksum using sha256 to check against the one computed on the downloaded file.
+ Default is 'None'.
+ accept_license : boolean
+ Flag to specify if user accepts the file LICENSE and prevents from printing the corresponding license terms.
+ Default is False.
+
+ Returns
+ -------
+ file_path: string
+ Full path of the created file.
+ """
+
+ file_path = join(dirname, filename)
+
+ # Get the file
+ urlretrieve(url, file_path)
+
+ if file_checksum is not None:
+ checksum = _checksum_sha256(file_path)
+ if file_checksum != checksum:
+ raise IOError("{} has a SHA256 checksum : {}, "
+ "different from expected : {}."
+ "The file may be corrupted or the given url may be wrong !".format(file_path, checksum, file_checksum))
+
+ # Print license terms unless accept_license is set to True
+ if not accept_license:
+ license_file = join(dirname, "LICENSE")
+ if exists(license_file) and (file_path != license_file):
+ with open(license_file, 'r') as f:
+ print(f.read())
+
+ return file_path
+
+def fetch_spiral_2d(filename = "spiral_2d.csv", dirname = "remote_datasets"):
+ """
+ Fetch "spiral_2d.csv" remotely.
+
+ Parameters
+ ----------
+ filename : string
+ The name to give to downloaded file. Default is "spiral_2d.csv".
+ dirname : string
+ The directory to save the file to. Default is "remote_datasets".
+
+ Returns
+ -------
+ points: array
+ Array of points stored in "spiral_2d.csv".
+ """
+ file_url = "https://raw.githubusercontent.com/GUDHI/gudhi-data/main/points/spiral_2d.csv"
+ file_checksum = '37530355d980d957c4ec06b18c775f90a91e446107d06c6201c9b4000b077f38'
+
+ archive_path = join(dirname, filename)
+
+ if not exists(archive_path):
+ # Create directory if not existing
+ if not exists(dirname):
+ makedirs(dirname)
+
+ file_path_pkl = _fetch_remote(file_url, filename, dirname, file_checksum)
+
+ return np.loadtxt(file_path_pkl)
+ else:
+ return np.loadtxt(archive_path)
+
+def fetch_bunny(filename = "bunny.npy", dirname = "remote_datasets/bunny", accept_license = False):
+ """
+ Fetch "bunny.npy" remotely and its LICENSE file.
+
+ Parameters
+ ----------
+ filename : string
+ The name to give to downloaded file. Default is "bunny.npy".
+ dirname : string
+ The directory to save the file to. Default is "remote_datasets/bunny".
+ accept_license : boolean
+ Flag to specify if user accepts the file LICENSE and prevents from printing the corresponding license terms.
+ Default is False.
+
+ Returns
+ -------
+ points: array
+ Array of points stored in "bunny.npy".
+ """
+
+ file_url = "https://raw.githubusercontent.com/GUDHI/gudhi-data/main/points/bunny/bunny.npy"
+ file_checksum = '13f7842ebb4b45370e50641ff28c88685703efa5faab14edf0bb7d113a965e1b'
+ license_url = "https://raw.githubusercontent.com/GUDHI/gudhi-data/main/points/bunny/LICENSE"
+ license_checksum = 'aeb1bad319b7d74fa0b8076358182f9c6b1284c67cc07dc67cbc9bc73025d956'
+
+ archive_path = join(dirname, filename)
+
+ if not exists(archive_path):
+ # Create directory if not existing
+ if not exists(dirname):
+ makedirs(dirname)
+
+ license_path = _fetch_remote(license_url, "LICENSE", dirname, license_checksum)
+ file_path_pkl = _fetch_remote(file_url, filename, dirname, file_checksum, accept_license)
+
+ return np.load(file_path_pkl, mmap_mode='r')
+ else:
+ return np.load(archive_path, mmap_mode='r')
diff --git a/src/python/test/test_remote_datasets.py b/src/python/test/test_remote_datasets.py
new file mode 100644
index 00000000..56a273b4
--- /dev/null
+++ b/src/python/test/test_remote_datasets.py
@@ -0,0 +1,86 @@
+# This file is part of the Gudhi Library - https://gudhi.inria.fr/ - which is released under MIT.
+# See file LICENSE or go to https://gudhi.inria.fr/licensing/ for full license details.
+# Author(s): Hind Montassif
+#
+# Copyright (C) 2021 Inria
+#
+# Modification(s):
+# - YYYY/MM Author: Description of the modification
+
+
+from gudhi.datasets import remote
+import re
+from os.path import isfile, exists
+from os import makedirs
+import io
+import sys
+import pytest
+
+def _check_dir_file_names(path_file_dw, filename, dirname):
+ assert isfile(path_file_dw)
+
+ names_dw = re.split(r' |/|\\', path_file_dw)
+ # Case where inner directories are created in "remote_datasets/"; e.g: "remote_datasets/bunny"
+ if len(names_dw) >= 3:
+ for i in range(len(names_dw)-1):
+ assert re.split(r' |/|\\', dirname)[i] == names_dw[i]
+ assert filename == names_dw[i+1]
+ else:
+ assert dirname == names_dw[0]
+ assert filename == names_dw[1]
+
+def _check_fetch_output(url, filename, dirname = "remote_datasets", file_checksum = None):
+ if not exists(dirname):
+ makedirs(dirname)
+ path_file_dw = remote._fetch_remote(url, filename, dirname, file_checksum)
+ _check_dir_file_names(path_file_dw, filename, dirname)
+
+def _get_bunny_license_print(accept_license = False):
+ capturedOutput = io.StringIO()
+ # Redirect stdout
+ sys.stdout = capturedOutput
+
+ if not exists("remote_datasets/bunny"):
+ makedirs("remote_datasets/bunny")
+ remote._fetch_remote("https://raw.githubusercontent.com/GUDHI/gudhi-data/main/points/bunny/bunny.npy", "bunny.npy", "remote_datasets/bunny",
+ '13f7842ebb4b45370e50641ff28c88685703efa5faab14edf0bb7d113a965e1b', accept_license)
+ # Reset redirect
+ sys.stdout = sys.__stdout__
+ return capturedOutput
+
+def test_fetch_remote_datasets():
+ # Test fetch with a wrong checksum
+ with pytest.raises(OSError):
+ _check_fetch_output("https://raw.githubusercontent.com/GUDHI/gudhi-data/main/points/spiral_2d.csv", "spiral_2d.csv", file_checksum = 'XXXXXXXXXX')
+
+ # Test files download from given urls with checksums provided
+ _check_fetch_output("https://raw.githubusercontent.com/GUDHI/gudhi-data/main/points/spiral_2d.csv", "spiral_2d.csv",
+ file_checksum = '37530355d980d957c4ec06b18c775f90a91e446107d06c6201c9b4000b077f38')
+
+ _check_fetch_output("https://raw.githubusercontent.com/GUDHI/gudhi-data/main/points/sphere3D_pts_on_grid.off", "sphere3D_pts_on_grid.off",
+ file_checksum = '32f96d2cafb1177f0dd5e0a019b6ff5658e14a619a7815ae55ad0fc5e8bd3f88')
+
+ # Test files download from given urls without checksums
+ _check_fetch_output("https://raw.githubusercontent.com/GUDHI/gudhi-data/main/points/spiral_2d.csv", "spiral_2d.csv")
+
+ _check_fetch_output("https://raw.githubusercontent.com/GUDHI/gudhi-data/main/points/sphere3D_pts_on_grid.off", "sphere3D_pts_on_grid.off")
+
+ # Test spiral_2d.csv wrapping function
+ spiral_2d_arr = remote.fetch_spiral_2d()
+ assert spiral_2d_arr.shape == (114562, 2)
+
+ # Test printing existing LICENSE file when fetching bunny.npy with accept_license = False (default)
+ # Fetch LICENSE file
+ if not exists("remote_datasets/bunny"):
+ makedirs("remote_datasets/bunny")
+ remote._fetch_remote("https://raw.githubusercontent.com/GUDHI/gudhi-data/main/points/bunny/LICENSE", "LICENSE", "remote_datasets/bunny",
+ 'aeb1bad319b7d74fa0b8076358182f9c6b1284c67cc07dc67cbc9bc73025d956')
+ with open("remote_datasets/bunny/LICENSE") as f:
+ assert f.read() == _get_bunny_license_print().getvalue().rstrip("\n")
+
+ # Test not printing bunny.npy LICENSE when accept_license = True
+ assert "" == _get_bunny_license_print(accept_license = True).getvalue()
+
+ # Test fetch_bunny wrapping function
+ bunny_arr = remote.fetch_bunny()
+ assert bunny_arr.shape == (35947, 3)
generated by cgit v1.2.3 (git 2.39.1) at 2024-07-07 00:16:38 +0000