diff options
-rw-r--r-- | .travis.yml | 48 | ||||
-rw-r--r-- | MANIFEST.in | 2 | ||||
-rw-r--r-- | Makefile | 19 | ||||
-rw-r--r-- | README.md | 7 | ||||
-rw-r--r-- | RELEASES.md | 68 | ||||
-rw-r--r-- | ot/__init__.py | 4 | ||||
-rw-r--r-- | ot/lp/__init__.py | 14 | ||||
-rw-r--r-- | ot/utils.py | 31 | ||||
-rw-r--r-- | requirements.txt | 1 | ||||
-rwxr-xr-x | setup.py | 15 |
10 files changed, 160 insertions, 49 deletions
diff --git a/.travis.yml b/.travis.yml index 72fd29a..5b3a26e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,27 +1,35 @@ +dist: xenial # required for Python >= 3.7 language: python matrix: -# allow_failures: -# - os: osx - include: -# - os: osx -# language: generic - - os: linux - sudo: required - python: 3.4 - - os: linux - sudo: required - python: 3.5 - - os: linux - sudo: required - python: 3.6 - - os: linux - sudo: required - python: 2.7 + # allow_failures: + # - os: osx + # - os: windows + include: + - os: linux + sudo: required + python: 3.5 + - os: linux + sudo: required + python: 3.6 + - os: linux + sudo: required + python: 3.7 + - os: linux + sudo: required + python: 2.7 + # - os: osx + # sudo: required + # language: generic + # - name: "Python 3.7.3 on Windows" + # os: windows # Windows 10.0.17134 N/A Build 17134 + # language: shell # 'language: python' is an error on Travis CI Windows + # before_install: choco install python + # env: PATH=/c/Python37:/c/Python37/Scripts:$PATH +# before_script: # configure a headless display to test plot generation +# - "export DISPLAY=:99.0" +# - sleep 3 # give xvfb some time to start before_install: - ./.travis/before_install.sh -before_script: # configure a headless display to test plot generation - - "export DISPLAY=:99.0" - - sleep 3 # give xvfb some time to start # command to install dependencies install: - pip install -r requirements.txt diff --git a/MANIFEST.in b/MANIFEST.in index e0acb7a..df4e139 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,5 @@ -graft ot/lp/ include README.md +include RELEASES.md include LICENSE include ot/lp/core.h include ot/lp/EMD.h @@ -3,6 +3,8 @@ PYTHON=python3 branch := $(shell git symbolic-ref --short -q HEAD) + + help : @echo "The following make targets are available:" @echo " help - print this message" @@ -13,6 +15,7 @@ help : @echo " sremove - remove the package (system with sudo)" @echo " clean - remove any temporary files" @echo " notebook - launch ipython notebook" + build : $(PYTHON) setup.py build @@ -47,14 +50,15 @@ test : FORCE pep8 pytest : FORCE $(PYTHON) -m pytest -v test/ --doctest-modules --ignore ot/gpu/ --cov=ot -uploadpypi : - #python setup.py register - $(PYTHON) setup.py sdist upload -r pypi +release : + twine upload dist/* + +release_test : + twine upload --repository-url https://test.pypi.org/legacy/ dist/* rdoc : pandoc --from=markdown --to=rst --output=docs/source/readme.rst README.md - notebook : ipython notebook --matplotlib=inline --notebook-dir=notebooks/ @@ -73,6 +77,13 @@ autopep8 : aautopep8 : autopep8 -air test ot examples --jobs -1 + +wheels : + CIBW_BEFORE_BUILD="pip install numpy cython" cibuildwheel --platform linux --output-dir dist + +dist : wheels + $(PYTHON) setup.py sdist + pydocstyle : pydocstyle ot @@ -45,7 +45,7 @@ year={2017} ## Installation -The library has been tested on Linux, MacOSX and Windows. It requires a C++ compiler for using the EMD solver and relies on the following Python modules: +The library has been tested on Linux, MacOSX and Windows. It requires a C++ compiler for building/installing the EMD solver and relies on the following Python modules: - Numpy (>=1.11) - Scipy (>=1.0) @@ -152,7 +152,8 @@ Here is a list of the Python notebooks available [here](https://github.com/rflam * [Wasserstein Discriminant Analysis](https://github.com/rflamary/POT/blob/master/notebooks/plot_WDA.ipynb) * [Gromov Wasserstein](https://github.com/rflamary/POT/blob/master/notebooks/plot_gromov.ipynb) * [Gromov Wasserstein Barycenter](https://github.com/rflamary/POT/blob/master/notebooks/plot_gromov_barycenter.ipynb) - +* [Fused Gromov Wasserstein](https://github.com/rflamary/POT/blob/master/notebooks/plot_fgw.ipynb) +* [Fused Gromov Wasserstein Barycenter](https://github.com/rflamary/POT/blob/master/notebooks/plot_barycenter_fgw.ipynb) You can also see the notebooks with [Jupyter nbviewer](https://nbviewer.jupyter.org/github/rflamary/POT/tree/master/notebooks/). @@ -166,8 +167,6 @@ This toolbox has been created and is maintained by The contributors to this library are -* [RĂ©mi Flamary](http://remi.flamary.com/) -* [Nicolas Courty](http://people.irisa.fr/Nicolas.Courty/) * [Alexandre Gramfort](http://alexandre.gramfort.net/) * [Laetitia Chapel](http://people.irisa.fr/Laetitia.Chapel/) * [Michael Perrot](http://perso.univ-st-etienne.fr/pem82055/) (Mapping estimation) diff --git a/RELEASES.md b/RELEASES.md index a617441..66eee19 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -1,6 +1,74 @@ # POT Releases +## 0.6 Year 3 +*July 2019* + +This is the first official stable release of POT and this means a jump to 0.6! +The library has been used in +the wild for a while now and we have reached a state where a lot of fundamental +OT solvers are available and tested. It has been quite stable in the last months +but kept the beta flag in its Pypi classifiers until now. + +Note that this release will be the last one supporting officially Python 2.7 (See +https://python3statement.org/ for more reasons). For next release we will keep +the travis tests for Python 2 but will make them non necessary for merge in 2020. + +The features are never complete in a toolbox designed for solving mathematical +problems and research but with the new contributions we now implement algorithms and solvers +from 24 scientific papers (listed in the README.md file). New features include a +direct implementation of the [empirical Sinkhorn divergence](https://pot.readthedocs.io/en/latest/all.html#ot.bregman.empirical_sinkhorn_divergence) +, a new efficient (Cython implementation) solver for [EMD in 1D](https://pot.readthedocs.io/en/latest/all.html#ot.lp.emd_1d) +and corresponding [Wasserstein +1D](https://pot.readthedocs.io/en/latest/all.html#ot.lp.wasserstein_1d). We now also +have implementations for [Unbalanced OT](https://github.com/rflamary/POT/blob/master/notebooks/plot_UOT_1D.ipynb) +and a solver for [Unbalanced OT barycenters](https://github.com/rflamary/POT/blob/master/notebooks/plot_UOT_barycenter_1D.ipynb). +A new variant of Gromov-Wasserstein divergence called [Fused +Gromov-Wasserstein](https://pot.readthedocs.io/en/latest/all.html?highlight=fused_#ot.gromov.fused_gromov_wasserstein) + has been also contributed with exemples of use on [structured data](https://github.com/rflamary/POT/blob/master/notebooks/plot_fgw.ipynb) +and computing [barycenters of labeld graphs](https://github.com/rflamary/POT/blob/master/notebooks/plot_barycenter_fgw.ipynb). + + +A lot of work has been done on the documentation with several new +examples corresponding to the new features and a lot of corrections for the +docstrings. But the most visible change is a new +[quick start guide](https://pot.readthedocs.io/en/latest/quickstart.html) for +POT that gives several pointers about which function or classes allow to solve which +specific OT problem. When possible a link is provided to relevant examples. + +We will also provide with this release some pre-compiled Python wheels for Linux +64bit on +github and pip. This will simplify the install process that before required a C +compiler and numpy/cython already installed. + +Finally we would like to acknowledge and thank the numerous contributors of POT +that has helped in the past build the foundation and are still contributing to +bring new features and solvers to the library. + + + +#### Features + +* Add compiled manylinux 64bits wheels to pip releases (PR #91) +* Add quick start guide (PR #88) +* Make doctest work on travis (PR #90) +* Update documentation (PR #79, PR #84) +* Solver for EMD in 1D (PR #89) +* Solvers for regularized unbalanced OT (PR #87, PR#99) +* Solver for Fused Gromov-Wasserstein (PR #86) +* Add empirical Sinkhorn and empirical Sinkhorn divergences (PR #80) + + +#### Closed issues + +- Issue #59 fail when using "pip install POT" (new details in doc+ hopefully + wheels) +- Issue #85 Cannot run gpu modules +- Issue #75 Greenkhorn do not return log (solved in PR #76) +- Issue #82 Gromov-Wasserstein fails when the cost matrices are slightly different +- Issue #72 Macosx build problem + + ## 0.5.0 Year 2 *Sep 2018* diff --git a/ot/__init__.py b/ot/__init__.py index df0ef8a..89c7936 100644 --- a/ot/__init__.py +++ b/ot/__init__.py @@ -71,9 +71,9 @@ from .da import sinkhorn_lpl1_mm # utils functions from .utils import dist, unif, tic, toc, toq -__version__ = "0.5.1" +__version__ = "0.6.0" -__all__ = ["emd", "emd2", 'emd_1d', "sinkhorn", "sinkhorn2", "utils", 'datasets', +__all__ = ['emd', 'emd2', 'emd_1d', 'sinkhorn', 'sinkhorn2', 'utils', 'datasets', 'bregman', 'lp', 'tic', 'toc', 'toq', 'gromov', 'emd_1d', 'emd2_1d', 'wasserstein_1d', 'dist', 'unif', 'barycenter', 'sinkhorn_lpl1_mm', 'da', 'optim', diff --git a/ot/lp/__init__.py b/ot/lp/__init__.py index 17f1731..0c92810 100644 --- a/ot/lp/__init__.py +++ b/ot/lp/__init__.py @@ -11,7 +11,7 @@ Solvers for the original linear program OT problem # License: MIT License import multiprocessing - +import sys import numpy as np from scipy.sparse import coo_matrix @@ -151,6 +151,8 @@ def emd2(a, b, M, processes=multiprocessing.cpu_count(), Target histogram (uniform weight if empty list) M : (ns,nt) numpy.ndarray, float64 Loss matrix (c-order array with type float64) + processes : int, optional (default=nb cpu) + Nb of processes used for multiple emd computation (not used on windows) numItermax : int, optional (default=100000) The maximum number of iterations before stopping the optimization algorithm if it has not converged. @@ -200,6 +202,10 @@ def emd2(a, b, M, processes=multiprocessing.cpu_count(), b = np.asarray(b, dtype=np.float64) M = np.asarray(M, dtype=np.float64) + # problem with pikling Forks + if sys.platform.endswith('win32'): + processes=1 + # if empty array given then use uniform distributions if len(a) == 0: a = np.ones((M.shape[0],), dtype=np.float64) / M.shape[0] @@ -228,7 +234,11 @@ def emd2(a, b, M, processes=multiprocessing.cpu_count(), return f(b) nb = b.shape[1] - res = parmap(f, [b[:, i] for i in range(nb)], processes) + if processes>1: + res = parmap(f, [b[:, i] for i in range(nb)], processes) + else: + res = list(map(f, [b[:, i].copy() for i in range(nb)])) + return res diff --git a/ot/utils.py b/ot/utils.py index d4127e3..b71458b 100644 --- a/ot/utils.py +++ b/ot/utils.py @@ -210,23 +210,28 @@ def fun(f, q_in, q_out): def parmap(f, X, nprocs=multiprocessing.cpu_count()): - """ paralell map for multiprocessing """ - q_in = multiprocessing.Queue(1) - q_out = multiprocessing.Queue() + """ paralell map for multiprocessing (only map on windows)""" - proc = [multiprocessing.Process(target=fun, args=(f, q_in, q_out)) - for _ in range(nprocs)] - for p in proc: - p.daemon = True - p.start() + if not sys.platform.endswith('win32'): - sent = [q_in.put((i, x)) for i, x in enumerate(X)] - [q_in.put((None, None)) for _ in range(nprocs)] - res = [q_out.get() for _ in range(len(sent))] + q_in = multiprocessing.Queue(1) + q_out = multiprocessing.Queue() - [p.join() for p in proc] + proc = [multiprocessing.Process(target=fun, args=(f, q_in, q_out)) + for _ in range(nprocs)] + for p in proc: + p.daemon = True + p.start() - return [x for i, x in sorted(res)] + sent = [q_in.put((i, x)) for i, x in enumerate(X)] + [q_in.put((None, None)) for _ in range(nprocs)] + res = [q_out.get() for _ in range(len(sent))] + + [p.join() for p in proc] + + return [x for i, x in sorted(res)] + else: + return list(map(f, X)) def check_params(**kwargs): diff --git a/requirements.txt b/requirements.txt index 97d165b..5a3432b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,4 +5,5 @@ matplotlib sphinx-gallery autograd pymanopt +cvxopt pytest @@ -53,17 +53,26 @@ setup(name='POT', license = 'MIT', scripts=[], data_files=[], - requires=["numpy","scipy","cython","matplotlib"], - install_requires=["numpy","scipy","cython","matplotlib"], + requires=["numpy","scipy","cython"], + install_requires=["numpy","scipy","cython"], classifiers=[ - 'Development Status :: 4 - Beta', + 'Development Status :: 5 - Production/Stable', 'Intended Audience :: Developers', + 'Intended Audience :: Education', + 'Intended Audience :: Science/Research', + 'License :: OSI Approved :: MIT License', 'Environment :: Console', 'Operating System :: OS Independent', 'Operating System :: MacOS', 'Operating System :: POSIX', 'Programming Language :: Python', + 'Programming Language :: C++', + 'Programming Language :: C', + 'Programming Language :: Cython', 'Topic :: Utilities', + 'Topic :: Scientific/Engineering :: Artificial Intelligence', + 'Topic :: Scientific/Engineering :: Mathematics', + 'Topic :: Scientific/Engineering :: Information Analysis', 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', |