From ca6f74ef0ae8784133a87ea4bd4905532a5721ae Mon Sep 17 00:00:00 2001 From: Rémi Flamary Date: Mon, 24 Oct 2016 15:51:48 +0200 Subject: bregman as module --- ot/bregman.py | 91 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 ot/bregman.py (limited to 'ot/bregman.py') diff --git a/ot/bregman.py b/ot/bregman.py new file mode 100644 index 0000000..8b97e1e --- /dev/null +++ b/ot/bregman.py @@ -0,0 +1,91 @@ +# -*- coding: utf-8 -*- +""" +Created on Fri Oct 21 09:40:21 2016 + +@author: rflamary +""" + +import numpy as np + + +def sinkhorn(a,b, M, reg,numItermax = 1000,stopThr=1e-9): + """ + Solve the optimal transport problem (OT) + + .. math:: + \gamma = arg\min_\gamma <\gamma,M>_F + reg\cdot\Omega(\gamma) + + s.t. \gamma 1 = a + + \gamma^T 1= b + + \gamma\geq 0 + where : + + - M is the metric cost matrix + - Omega is the entropic regularization term + - a and b are the sample weights + + Parameters + ---------- + a : (ns,) ndarray + samples in the source domain + b : (nt,) ndarray + samples in the target domain + M : (ns,nt) ndarray + loss matrix + reg: float() + Regularization term >0 + + + Returns + ------- + gamma: (ns x nt) ndarray + Optimal transportation matrix for the given parameters + + """ + # init data + Nini = len(a) + Nfin = len(b) + + + cpt = 0 + + # we assume that no distances are null except those of the diagonal of distances + u = np.ones(Nini)/Nini + v = np.ones(Nfin)/Nfin + uprev=np.zeros(Nini) + vprev=np.zeros(Nini) + + #print reg + + K = np.exp(-M/reg) + #print np.min(K) + + Kp = np.dot(np.diag(1/a),K) + transp = K + cpt = 0 + err=1 + while (err>stopThr and cpttol_error and cpttol_error and cpt Date: Mon, 24 Oct 2016 16:32:01 +0200 Subject: correction barcenter --- examples/demo_OT_2D_samples.py | 2 +- ot/__init__.py | 4 ++-- ot/bregman.py | 18 +++++++++++------- 3 files changed, 14 insertions(+), 10 deletions(-) (limited to 'ot/bregman.py') diff --git a/examples/demo_OT_2D_samples.py b/examples/demo_OT_2D_samples.py index f91bbb2..992352c 100644 --- a/examples/demo_OT_2D_samples.py +++ b/examples/demo_OT_2D_samples.py @@ -62,7 +62,7 @@ pl.title('OT matrix') #%% sinkhorn -lambd=.8e-1 +lambd=1e-1 Gs=ot.sinkhorn(a,b,M,lambd) diff --git a/ot/__init__.py b/ot/__init__.py index ce9d157..dd74590 100644 --- a/ot/__init__.py +++ b/ot/__init__.py @@ -8,9 +8,9 @@ import bregman # OT functions from emd import emd -from bregman import sinkhorn +from bregman import sinkhorn,barycenter # utils functions from utils import dist,unif -__all__ = ["emd","sinkhorn","utils",'datasets','bregman','plot','dist','unif'] +__all__ = ["emd","sinkhorn","utils",'datasets','bregman','plot','dist','unif','barycenter'] diff --git a/ot/bregman.py b/ot/bregman.py index 1761029..e46506b 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -116,7 +116,7 @@ def barycenter(A,M,reg, weights=None, numItermax = 1000, tol_error=1e-4,log=dict #compute Mmax once for all #M = M/np.median(M) # suggested by G. Peyre - K = np.exp(-reg*M) + K = np.exp(-M/reg) cpt = 0 err=1 @@ -124,7 +124,8 @@ def barycenter(A,M,reg, weights=None, numItermax = 1000, tol_error=1e-4,log=dict UKv=np.dot(K,np.divide(A.T,np.sum(K,axis=0)).T) u = (geometricMean(UKv)/UKv.T).T - log = {'niter':0, 'all_err':[]} + log['niter']=0 + log['all_err']=[] while (err>tol_error and cpttol_error and cpt Date: Mon, 24 Oct 2016 16:34:26 +0200 Subject: update demo barycenter --- examples/demo_barycenter_1D.py | 5 ++--- ot/bregman.py | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) (limited to 'ot/bregman.py') diff --git a/examples/demo_barycenter_1D.py b/examples/demo_barycenter_1D.py index 200444b..c9f63a2 100644 --- a/examples/demo_barycenter_1D.py +++ b/examples/demo_barycenter_1D.py @@ -42,9 +42,8 @@ pl.title('Distributions') bary_l2=A.mean(1) # wasserstein -reg=1e-2 -log=dict() -bary_wass=ot.bregman.barycenter(A,M,reg,log=log) +reg=1e-3 +bary_wass,log=ot.bregman.barycenter(A,M,reg) pl.figure(2) pl.clf() diff --git a/ot/bregman.py b/ot/bregman.py index e46506b..8362dc4 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -136,7 +136,7 @@ def barycenter(A,M,reg, weights=None, numItermax = 1000, tol_error=1e-4,log=dict log['all_err'].append(err) log['niter']=cpt - return geometricBar(weights,UKv) + return geometricBar(weights,UKv),log def unmixBregman(distrib,D,M,M0,h0,reg,reg0,alpha,numItermax = 1000, tol_error=1e-3,log=dict()): @@ -181,4 +181,4 @@ def unmixBregman(distrib,D,M,M0,h0,reg,reg0,alpha,numItermax = 1000, tol_error=1 log['niter']=cpt - return np.sum(K0,axis=1) + return np.sum(K0,axis=1),log -- cgit v1.2.3 From eff2e5304269482e1ce2e7d6e1271e068134bfa4 Mon Sep 17 00:00:00 2001 From: Rémi Flamary Date: Mon, 24 Oct 2016 16:45:03 +0200 Subject: cleanup --- Makefile | 5 +++++ ot/bregman.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) (limited to 'ot/bregman.py') diff --git a/Makefile b/Makefile index 030f422..ead5e3a 100644 --- a/Makefile +++ b/Makefile @@ -31,6 +31,11 @@ sremove : tr '\n' '\0' < files.txt | sudo xargs -0 rm -f -- rm files.txt +doc : + cd docs + make html + cd .. + clean : $(PYTHON) setup.py clean diff --git a/ot/bregman.py b/ot/bregman.py index 8362dc4..a74cd19 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -139,7 +139,7 @@ def barycenter(A,M,reg, weights=None, numItermax = 1000, tol_error=1e-4,log=dict return geometricBar(weights,UKv),log -def unmixBregman(distrib,D,M,M0,h0,reg,reg0,alpha,numItermax = 1000, tol_error=1e-3,log=dict()): +def unmix(distrib,D,M,M0,h0,reg,reg0,alpha,numItermax = 1000, tol_error=1e-3,log=dict()): """ distrib : distribution to unmix D : Dictionnary -- cgit v1.2.3 From 176ff069483b9ba630af8a00ce5edc104168c0a2 Mon Sep 17 00:00:00 2001 From: Rémi Flamary Date: Mon, 24 Oct 2016 16:53:34 +0200 Subject: comments in modules --- ot/bregman.py | 4 +--- ot/datasets.py | 4 ++++ ot/plot.py | 4 ++++ ot/utils.py | 4 +++- 4 files changed, 12 insertions(+), 4 deletions(-) (limited to 'ot/bregman.py') diff --git a/ot/bregman.py b/ot/bregman.py index a74cd19..81804a7 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- """ -Created on Fri Oct 21 09:40:21 2016 - -@author: rflamary +Bregman projection for regularized Otimal transport """ import numpy as np diff --git a/ot/datasets.py b/ot/datasets.py index 3ebc2a1..cebfdac 100644 --- a/ot/datasets.py +++ b/ot/datasets.py @@ -1,3 +1,7 @@ +""" +Simple example datasets for OT +""" + import numpy as np import scipy as sp diff --git a/ot/plot.py b/ot/plot.py index ce5444e..d172c90 100644 --- a/ot/plot.py +++ b/ot/plot.py @@ -1,3 +1,7 @@ +""" +Functions for plotting OT matrices +""" + import numpy as np import matplotlib.pylab as pl diff --git a/ot/utils.py b/ot/utils.py index 5feb4c6..46c3775 100644 --- a/ot/utils.py +++ b/ot/utils.py @@ -1,4 +1,6 @@ - +""" +Various function that can be usefull +""" import numpy as np from scipy.spatial.distance import cdist -- cgit v1.2.3 From 7b9f4e9d4b198183929c60eceec1a419b5212e2d Mon Sep 17 00:00:00 2001 From: Rémi Flamary Date: Tue, 25 Oct 2016 10:24:55 +0200 Subject: set numpy doc format --- docs/source/conf.py | 2 +- docs/source/index.rst | 25 +++++++++++++++++++++++++ ot/bregman.py | 31 +++++++++++++++++++++++-------- 3 files changed, 49 insertions(+), 9 deletions(-) (limited to 'ot/bregman.py') diff --git a/docs/source/conf.py b/docs/source/conf.py index 0caefae..e114e2c 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -38,7 +38,7 @@ extensions = [ 'sphinx.ext.coverage', 'sphinx.ext.mathjax', 'sphinx.ext.ifconfig', - 'sphinx.ext.viewcode', + 'sphinx.ext.viewcode','sphinx.ext.autodoc', 'sphinxcontrib.napoleon' ] # Add any paths that contain templates here, relative to this directory. diff --git a/docs/source/index.rst b/docs/source/index.rst index e96f544..8613bfd 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -11,16 +11,41 @@ Contents: .. toctree:: :maxdepth: 2 + +Module ot +========= + +This module provide easy access to solvers for the most common OT problems + .. automodule:: ot :members: + +Module ot.emd +========= .. automodule:: ot.emd :members: + +Module ot.bregman +========= + .. automodule:: ot.bregman :members: + +Module ot.utils +========= + .. automodule:: ot.utils :members: + +Module ot.datasets +========= + .. automodule:: ot.datasets :members: + +Module ot.plot +========= + .. automodule:: ot.plot :members: diff --git a/ot/bregman.py b/ot/bregman.py index 81804a7..17ec06f 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -8,7 +8,9 @@ import numpy as np def sinkhorn(a,b, M, reg,numItermax = 1000,stopThr=1e-9): """ - Solve the optimal transport problem (OT) + Solve the entropic regularization optimal transport problem and return the OT matrix + + The function solves the following optimization problem: .. math:: \gamma = arg\min_\gamma <\gamma,M>_F + reg\cdot\Omega(\gamma) @@ -20,17 +22,20 @@ def sinkhorn(a,b, M, reg,numItermax = 1000,stopThr=1e-9): \gamma\geq 0 where : - - M is the metric cost matrix - - Omega is the entropic regularization term - - a and b are the sample weights + - M is the (ns,nt) metric cost matrix + - :math:`\Omega` is the entropic regularization term :math:`\Omega(\gamma)=\sum_{i,j} \gamma_{i,j}\log(\gamma_{i,j})` + - a and b are source and target weights (sum to 1) + + The algorithm used for solving the problem is the Sinkhorn-Knopp matrix scaling algorithm as proposed in [1]_ + Parameters ---------- - a : (ns,) ndarray - samples in the source domain - b : (nt,) ndarray + a : np.ndarray (ns,) + samples weights in the source domain + b : np.ndarray (nt,) samples in the target domain - M : (ns,nt) ndarray + M : np.ndarray (ns,nt) loss matrix reg: float() Regularization term >0 @@ -41,6 +46,16 @@ def sinkhorn(a,b, M, reg,numItermax = 1000,stopThr=1e-9): gamma: (ns x nt) ndarray Optimal transportation matrix for the given parameters + References + ---------- + + .. [1] M. Cuturi, Sinkhorn Distances : Lightspeed Computation of Optimal Transport, Advances in Neural Information Processing Systems (NIPS) 26, 2013 + + + See Also + -------- + ot.emd.emd : Unregularized optimal ransport + """ # init data Nini = len(a) -- cgit v1.2.3 From cf2d92e151f816e6ddcfc4b64cbda1f8f7bde9df Mon Sep 17 00:00:00 2001 From: Rémi Flamary Date: Fri, 28 Oct 2016 08:53:16 +0200 Subject: complete doc emd --- ot/bregman.py | 5 ++- ot/lp/__init__.py | 61 +++++++++++++++++++++++++- ot/lp/emd.cpp | 126 +++++++++++++++++++++++++++--------------------------- ot/lp/emd.pyx | 12 +++--- 4 files changed, 132 insertions(+), 72 deletions(-) (limited to 'ot/bregman.py') diff --git a/ot/bregman.py b/ot/bregman.py index 17ec06f..0d2c099 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -37,7 +37,7 @@ def sinkhorn(a,b, M, reg,numItermax = 1000,stopThr=1e-9): samples in the target domain M : np.ndarray (ns,nt) loss matrix - reg: float() + reg: float Regularization term >0 @@ -54,7 +54,8 @@ def sinkhorn(a,b, M, reg,numItermax = 1000,stopThr=1e-9): See Also -------- - ot.emd.emd : Unregularized optimal ransport + ot.lp.emd : Unregularized OT + ot.optim.cg : General regularized OT """ # init data diff --git a/ot/lp/__init__.py b/ot/lp/__init__.py index 46662b7..568e370 100644 --- a/ot/lp/__init__.py +++ b/ot/lp/__init__.py @@ -1,3 +1,62 @@ -from .emd import emd +from .emd import emd_c +import numpy as np + +def emd(a,b,M): + """ + Solves the Earth Movers distance problem and returns the optimal transport matrix + + gamm=emd(a,b,M) + + .. math:: + \gamma = arg\min_\gamma <\gamma,M>_F + + s.t. \gamma 1 = a + + \gamma^T 1= b + + \gamma\geq 0 + where : + + - M is the metric cost matrix + - a and b are the sample weights + + Parameters + ---------- + a : (ns,) ndarray, float64 + Source histogram (uniform weigth if empty list) + b : (nt,) ndarray, float64 + Target histogram (uniform weigth if empty list) + M : (ns,nt) ndarray, float64 + loss matrix + + Examples + -------- + + Simple example with obvious solution. The function :func:emd accepts lists and + perform automatic conversion tu numpy arrays + + >>> a=[.5,.5] + >>> b=[.5,.5] + >>> M=[[0.,1.],[1.,0.]] + >>> ot.emd(a,b,M) + array([[ 0.5, 0. ], + [ 0. , 0.5]]) + + Returns + ------- + gamma: (ns x nt) ndarray + Optimal transportation matrix for the given parameters + + """ + a=np.asarray(a,dtype=np.float64) + b=np.asarray(b,dtype=np.float64) + + if len(a)==0: + a=np.ones((M.shape[0],),dtype=np.float64)/M.shape[0] + if len(b)==0: + b=np.ones((M.shape[1],),dtype=np.float64)/M.shape[1] + + return emd_c(a,b,M) + diff --git a/ot/lp/emd.cpp b/ot/lp/emd.cpp index 2343af6..26d243f 100644 --- a/ot/lp/emd.cpp +++ b/ot/lp/emd.cpp @@ -6,11 +6,11 @@ "depends": [ "/usr/lib/python2.7/dist-packages/numpy/core/include/numpy/arrayobject.h", "/usr/lib/python2.7/dist-packages/numpy/core/include/numpy/ufuncobject.h", - "ot/emd/EMD.h" + "ot/lp/EMD.h" ], "include_dirs": [ "/usr/lib/python2.7/dist-packages/numpy/core/include", - "/home/rflamary/PYTHON/POT/ot/emd" + "/home/rflamary/PYTHON/POT/ot/lp" ], "language": "c++" } @@ -260,8 +260,8 @@ static CYTHON_INLINE float __PYX_NAN() { #endif #endif -#define __PYX_HAVE__ot__emd__emd -#define __PYX_HAVE_API__ot__emd__emd +#define __PYX_HAVE__ot__lp__emd +#define __PYX_HAVE_API__ot__lp__emd #include "string.h" #include "stdio.h" #include "stdlib.h" @@ -497,7 +497,7 @@ static const char *__pyx_filename; static const char *__pyx_f[] = { - "ot/emd/emd.pyx", + "ot/lp/emd.pyx", "__init__.pxd", "type.pxd", }; @@ -1129,12 +1129,12 @@ static CYTHON_INLINE char *__pyx_f_5numpy__util_dtypestring(PyArray_Descr *, cha /* Module declarations from 'cython' */ -/* Module declarations from 'ot.emd.emd' */ +/* Module declarations from 'ot.lp.emd' */ static __Pyx_TypeInfo __Pyx_TypeInfo_double = { "double", NULL, sizeof(double), { 0 }, 0, 'R', 0, 0 }; -#define __Pyx_MODULE_NAME "ot.emd.emd" -int __pyx_module_is_main_ot__emd__emd = 0; +#define __Pyx_MODULE_NAME "ot.lp.emd" +int __pyx_module_is_main_ot__lp__emd = 0; -/* Implementation of 'ot.emd.emd' */ +/* Implementation of 'ot.lp.emd' */ static PyObject *__pyx_builtin_ValueError; static PyObject *__pyx_builtin_range; static PyObject *__pyx_builtin_RuntimeError; @@ -1161,21 +1161,21 @@ static char __pyx_k_Zg[] = "Zg"; static char __pyx_k_n1[] = "n1"; static char __pyx_k_n2[] = "n2"; static char __pyx_k_np[] = "np"; -static char __pyx_k_emd[] = "emd"; static char __pyx_k_cost[] = "cost"; static char __pyx_k_main[] = "__main__"; static char __pyx_k_ones[] = "ones"; static char __pyx_k_test[] = "__test__"; +static char __pyx_k_emd_c[] = "emd_c"; static char __pyx_k_numpy[] = "numpy"; static char __pyx_k_range[] = "range"; static char __pyx_k_zeros[] = "zeros"; static char __pyx_k_import[] = "__import__"; +static char __pyx_k_ot_lp_emd[] = "ot.lp.emd"; static char __pyx_k_ValueError[] = "ValueError"; -static char __pyx_k_ot_emd_emd[] = "ot.emd.emd"; static char __pyx_k_RuntimeError[] = "RuntimeError"; static char __pyx_k_ndarray_is_not_C_contiguous[] = "ndarray is not C contiguous"; static char __pyx_k_Created_on_Thu_Sep_11_08_42_08[] = "\nCreated on Thu Sep 11 08:42:08 2014\n\n@author: rflamary\n"; -static char __pyx_k_home_rflamary_PYTHON_POT_ot_emd[] = "/home/rflamary/PYTHON/POT/ot/emd/emd.pyx"; +static char __pyx_k_home_rflamary_PYTHON_POT_ot_lp[] = "/home/rflamary/PYTHON/POT/ot/lp/emd.pyx"; static char __pyx_k_unknown_dtype_code_in_numpy_pxd[] = "unknown dtype code in numpy.pxd (%d)"; static char __pyx_k_Format_string_allocated_too_shor[] = "Format string allocated too short, see comment in numpy.pxd"; static char __pyx_k_Non_native_byte_order_not_suppor[] = "Non-native byte order not supported"; @@ -1191,8 +1191,8 @@ static PyObject *__pyx_n_s_ValueError; static PyObject *__pyx_n_s_a; static PyObject *__pyx_n_s_b; static PyObject *__pyx_n_s_cost; -static PyObject *__pyx_n_s_emd; -static PyObject *__pyx_kp_s_home_rflamary_PYTHON_POT_ot_emd; +static PyObject *__pyx_n_s_emd_c; +static PyObject *__pyx_kp_s_home_rflamary_PYTHON_POT_ot_lp; static PyObject *__pyx_n_s_import; static PyObject *__pyx_n_s_main; static PyObject *__pyx_n_s_n1; @@ -1202,12 +1202,12 @@ static PyObject *__pyx_kp_u_ndarray_is_not_Fortran_contiguou; static PyObject *__pyx_n_s_np; static PyObject *__pyx_n_s_numpy; static PyObject *__pyx_n_s_ones; -static PyObject *__pyx_n_s_ot_emd_emd; +static PyObject *__pyx_n_s_ot_lp_emd; static PyObject *__pyx_n_s_range; static PyObject *__pyx_n_s_test; static PyObject *__pyx_kp_u_unknown_dtype_code_in_numpy_pxd; static PyObject *__pyx_n_s_zeros; -static PyObject *__pyx_pf_2ot_3emd_3emd_emd(CYTHON_UNUSED PyObject *__pyx_self, PyArrayObject *__pyx_v_a, PyArrayObject *__pyx_v_b, PyArrayObject *__pyx_v_M); /* proto */ +static PyObject *__pyx_pf_2ot_2lp_3emd_emd_c(CYTHON_UNUSED PyObject *__pyx_self, PyArrayObject *__pyx_v_a, PyArrayObject *__pyx_v_b, PyArrayObject *__pyx_v_M); /* proto */ static int __pyx_pf_5numpy_7ndarray___getbuffer__(PyArrayObject *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags); /* proto */ static void __pyx_pf_5numpy_7ndarray_2__releasebuffer__(PyArrayObject *__pyx_v_self, Py_buffer *__pyx_v_info); /* proto */ static PyObject *__pyx_tuple_; @@ -1219,19 +1219,19 @@ static PyObject *__pyx_tuple__6; static PyObject *__pyx_tuple__7; static PyObject *__pyx_codeobj__8; -/* "ot/emd/emd.pyx":21 +/* "ot/lp/emd.pyx":21 * @cython.boundscheck(False) * @cython.wraparound(False) - * def emd( np.ndarray[double, ndim=1, mode="c"] a,np.ndarray[double, ndim=1, mode="c"] b,np.ndarray[double, ndim=2, mode="c"] M): # <<<<<<<<<<<<<< + * def emd_c( np.ndarray[double, ndim=1, mode="c"] a,np.ndarray[double, ndim=1, mode="c"] b,np.ndarray[double, ndim=2, mode="c"] M): # <<<<<<<<<<<<<< * """ * Solves the Earth Movers distance problem and returns the optimal transport matrix */ /* Python wrapper */ -static PyObject *__pyx_pw_2ot_3emd_3emd_1emd(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ -static char __pyx_doc_2ot_3emd_3emd_emd[] = "\n Solves the Earth Movers distance problem and returns the optimal transport matrix\n \n gamm=emd(a,b,M)\n \n .. math::\n \\gamma = arg\\min_\\gamma <\\gamma,M>_F \n \n s.t. \\gamma 1 = a\n \n \\gamma^T 1= b \n \n \\gamma\\geq 0\n where :\n \n - M is the metric cost matrix\n - a and b are the sample weights\n \n Parameters\n ----------\n a : (ns,) ndarray\n samples in the source domain (uniform waigth if empty)\n b : (nt,) ndarray\n samples in the target domain (uniform waigth if empty)\n M : (ns,nt) ndarray\n loss matrix \n \n \n Returns\n -------\n gamma: (ns x nt) ndarray\n Optimal transportation matrix for the given parameters\n \n "; -static PyMethodDef __pyx_mdef_2ot_3emd_3emd_1emd = {"emd", (PyCFunction)__pyx_pw_2ot_3emd_3emd_1emd, METH_VARARGS|METH_KEYWORDS, __pyx_doc_2ot_3emd_3emd_emd}; -static PyObject *__pyx_pw_2ot_3emd_3emd_1emd(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { +static PyObject *__pyx_pw_2ot_2lp_3emd_1emd_c(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static char __pyx_doc_2ot_2lp_3emd_emd_c[] = "\n Solves the Earth Movers distance problem and returns the optimal transport matrix\n \n gamm=emd(a,b,M)\n \n .. math::\n \\gamma = arg\\min_\\gamma <\\gamma,M>_F \n \n s.t. \\gamma 1 = a\n \n \\gamma^T 1= b \n \n \\gamma\\geq 0\n where :\n \n - M is the metric cost matrix\n - a and b are the sample weights\n \n Parameters\n ----------\n a : (ns,) ndarray\n source histogram \n b : (nt,) ndarray\n target histogram\n M : (ns,nt) ndarray\n loss matrix \n \n \n Returns\n -------\n gamma: (ns x nt) ndarray\n Optimal transportation matrix for the given parameters\n \n "; +static PyMethodDef __pyx_mdef_2ot_2lp_3emd_1emd_c = {"emd_c", (PyCFunction)__pyx_pw_2ot_2lp_3emd_1emd_c, METH_VARARGS|METH_KEYWORDS, __pyx_doc_2ot_2lp_3emd_emd_c}; +static PyObject *__pyx_pw_2ot_2lp_3emd_1emd_c(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { PyArrayObject *__pyx_v_a = 0; PyArrayObject *__pyx_v_b = 0; PyArrayObject *__pyx_v_M = 0; @@ -1240,7 +1240,7 @@ static PyObject *__pyx_pw_2ot_3emd_3emd_1emd(PyObject *__pyx_self, PyObject *__p int __pyx_clineno = 0; PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations - __Pyx_RefNannySetupContext("emd (wrapper)", 0); + __Pyx_RefNannySetupContext("emd_c (wrapper)", 0); { static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_a,&__pyx_n_s_b,&__pyx_n_s_M,0}; PyObject* values[3] = {0,0,0}; @@ -1262,16 +1262,16 @@ static PyObject *__pyx_pw_2ot_3emd_3emd_1emd(PyObject *__pyx_self, PyObject *__p case 1: if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_b)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("emd", 1, 3, 3, 1); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 21; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + __Pyx_RaiseArgtupleInvalid("emd_c", 1, 3, 3, 1); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 21; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } case 2: if (likely((values[2] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_M)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("emd", 1, 3, 3, 2); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 21; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + __Pyx_RaiseArgtupleInvalid("emd_c", 1, 3, 3, 2); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 21; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "emd") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 21; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "emd_c") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 21; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } } else if (PyTuple_GET_SIZE(__pyx_args) != 3) { goto __pyx_L5_argtuple_error; @@ -1286,16 +1286,16 @@ static PyObject *__pyx_pw_2ot_3emd_3emd_1emd(PyObject *__pyx_self, PyObject *__p } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("emd", 1, 3, 3, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 21; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + __Pyx_RaiseArgtupleInvalid("emd_c", 1, 3, 3, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 21; __pyx_clineno = __LINE__; goto __pyx_L3_error;} __pyx_L3_error:; - __Pyx_AddTraceback("ot.emd.emd.emd", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_AddTraceback("ot.lp.emd.emd_c", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); return NULL; __pyx_L4_argument_unpacking_done:; if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_a), __pyx_ptype_5numpy_ndarray, 1, "a", 0))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 21; __pyx_clineno = __LINE__; goto __pyx_L1_error;} if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_b), __pyx_ptype_5numpy_ndarray, 1, "b", 0))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 21; __pyx_clineno = __LINE__; goto __pyx_L1_error;} if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_M), __pyx_ptype_5numpy_ndarray, 1, "M", 0))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 21; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_r = __pyx_pf_2ot_3emd_3emd_emd(__pyx_self, __pyx_v_a, __pyx_v_b, __pyx_v_M); + __pyx_r = __pyx_pf_2ot_2lp_3emd_emd_c(__pyx_self, __pyx_v_a, __pyx_v_b, __pyx_v_M); /* function exit code */ goto __pyx_L0; @@ -1306,7 +1306,7 @@ static PyObject *__pyx_pw_2ot_3emd_3emd_1emd(PyObject *__pyx_self, PyObject *__p return __pyx_r; } -static PyObject *__pyx_pf_2ot_3emd_3emd_emd(CYTHON_UNUSED PyObject *__pyx_self, PyArrayObject *__pyx_v_a, PyArrayObject *__pyx_v_b, PyArrayObject *__pyx_v_M) { +static PyObject *__pyx_pf_2ot_2lp_3emd_emd_c(CYTHON_UNUSED PyObject *__pyx_self, PyArrayObject *__pyx_v_a, PyArrayObject *__pyx_v_b, PyArrayObject *__pyx_v_M) { int __pyx_v_n1; int __pyx_v_n2; float __pyx_v_cost; @@ -1338,7 +1338,7 @@ static PyObject *__pyx_pf_2ot_3emd_3emd_emd(CYTHON_UNUSED PyObject *__pyx_self, int __pyx_lineno = 0; const char *__pyx_filename = NULL; int __pyx_clineno = 0; - __Pyx_RefNannySetupContext("emd", 0); + __Pyx_RefNannySetupContext("emd_c", 0); __Pyx_INCREF((PyObject *)__pyx_v_a); __Pyx_INCREF((PyObject *)__pyx_v_b); __pyx_pybuffer_G.pybuffer.buf = NULL; @@ -1373,7 +1373,7 @@ static PyObject *__pyx_pf_2ot_3emd_3emd_emd(CYTHON_UNUSED PyObject *__pyx_self, } __pyx_pybuffernd_M.diminfo[0].strides = __pyx_pybuffernd_M.rcbuffer->pybuffer.strides[0]; __pyx_pybuffernd_M.diminfo[0].shape = __pyx_pybuffernd_M.rcbuffer->pybuffer.shape[0]; __pyx_pybuffernd_M.diminfo[1].strides = __pyx_pybuffernd_M.rcbuffer->pybuffer.strides[1]; __pyx_pybuffernd_M.diminfo[1].shape = __pyx_pybuffernd_M.rcbuffer->pybuffer.shape[1]; - /* "ot/emd/emd.pyx":56 + /* "ot/lp/emd.pyx":56 * * """ * cdef int n1= M.shape[0] # <<<<<<<<<<<<<< @@ -1382,7 +1382,7 @@ static PyObject *__pyx_pf_2ot_3emd_3emd_emd(CYTHON_UNUSED PyObject *__pyx_self, */ __pyx_v_n1 = (__pyx_v_M->dimensions[0]); - /* "ot/emd/emd.pyx":57 + /* "ot/lp/emd.pyx":57 * """ * cdef int n1= M.shape[0] * cdef int n2= M.shape[1] # <<<<<<<<<<<<<< @@ -1391,7 +1391,7 @@ static PyObject *__pyx_pf_2ot_3emd_3emd_emd(CYTHON_UNUSED PyObject *__pyx_self, */ __pyx_v_n2 = (__pyx_v_M->dimensions[1]); - /* "ot/emd/emd.pyx":59 + /* "ot/lp/emd.pyx":59 * cdef int n2= M.shape[1] * * cdef float cost=0 # <<<<<<<<<<<<<< @@ -1400,7 +1400,7 @@ static PyObject *__pyx_pf_2ot_3emd_3emd_emd(CYTHON_UNUSED PyObject *__pyx_self, */ __pyx_v_cost = 0.0; - /* "ot/emd/emd.pyx":60 + /* "ot/lp/emd.pyx":60 * * cdef float cost=0 * cdef np.ndarray[double, ndim=2, mode="c"] G=np.zeros([n1, n2]) # <<<<<<<<<<<<<< @@ -1464,7 +1464,7 @@ static PyObject *__pyx_pf_2ot_3emd_3emd_emd(CYTHON_UNUSED PyObject *__pyx_self, __pyx_v_G = ((PyArrayObject *)__pyx_t_1); __pyx_t_1 = 0; - /* "ot/emd/emd.pyx":62 + /* "ot/lp/emd.pyx":62 * cdef np.ndarray[double, ndim=2, mode="c"] G=np.zeros([n1, n2]) * * if not len(a): # <<<<<<<<<<<<<< @@ -1475,7 +1475,7 @@ static PyObject *__pyx_pf_2ot_3emd_3emd_emd(CYTHON_UNUSED PyObject *__pyx_self, __pyx_t_8 = ((!(__pyx_t_7 != 0)) != 0); if (__pyx_t_8) { - /* "ot/emd/emd.pyx":63 + /* "ot/lp/emd.pyx":63 * * if not len(a): * a=np.ones((n1,))/n1 # <<<<<<<<<<<<<< @@ -1548,7 +1548,7 @@ static PyObject *__pyx_pf_2ot_3emd_3emd_emd(CYTHON_UNUSED PyObject *__pyx_self, __Pyx_DECREF_SET(__pyx_v_a, ((PyArrayObject *)__pyx_t_4)); __pyx_t_4 = 0; - /* "ot/emd/emd.pyx":62 + /* "ot/lp/emd.pyx":62 * cdef np.ndarray[double, ndim=2, mode="c"] G=np.zeros([n1, n2]) * * if not len(a): # <<<<<<<<<<<<<< @@ -1557,7 +1557,7 @@ static PyObject *__pyx_pf_2ot_3emd_3emd_emd(CYTHON_UNUSED PyObject *__pyx_self, */ } - /* "ot/emd/emd.pyx":65 + /* "ot/lp/emd.pyx":65 * a=np.ones((n1,))/n1 * * if not len(b): # <<<<<<<<<<<<<< @@ -1568,7 +1568,7 @@ static PyObject *__pyx_pf_2ot_3emd_3emd_emd(CYTHON_UNUSED PyObject *__pyx_self, __pyx_t_8 = ((!(__pyx_t_7 != 0)) != 0); if (__pyx_t_8) { - /* "ot/emd/emd.pyx":66 + /* "ot/lp/emd.pyx":66 * * if not len(b): * b=np.ones((n2,))/n2 # <<<<<<<<<<<<<< @@ -1641,7 +1641,7 @@ static PyObject *__pyx_pf_2ot_3emd_3emd_emd(CYTHON_UNUSED PyObject *__pyx_self, __Pyx_DECREF_SET(__pyx_v_b, ((PyArrayObject *)__pyx_t_3)); __pyx_t_3 = 0; - /* "ot/emd/emd.pyx":65 + /* "ot/lp/emd.pyx":65 * a=np.ones((n1,))/n1 * * if not len(b): # <<<<<<<<<<<<<< @@ -1650,7 +1650,7 @@ static PyObject *__pyx_pf_2ot_3emd_3emd_emd(CYTHON_UNUSED PyObject *__pyx_self, */ } - /* "ot/emd/emd.pyx":69 + /* "ot/lp/emd.pyx":69 * * # calling the function * EMD_wrap(n1,n2, a.data, b.data, M.data, G.data, &cost) # <<<<<<<<<<<<<< @@ -1659,7 +1659,7 @@ static PyObject *__pyx_pf_2ot_3emd_3emd_emd(CYTHON_UNUSED PyObject *__pyx_self, */ EMD_wrap(__pyx_v_n1, __pyx_v_n2, ((double *)__pyx_v_a->data), ((double *)__pyx_v_b->data), ((double *)__pyx_v_M->data), ((double *)__pyx_v_G->data), ((double *)(&__pyx_v_cost))); - /* "ot/emd/emd.pyx":71 + /* "ot/lp/emd.pyx":71 * EMD_wrap(n1,n2, a.data, b.data, M.data, G.data, &cost) * * return G # <<<<<<<<<<<<<< @@ -1669,10 +1669,10 @@ static PyObject *__pyx_pf_2ot_3emd_3emd_emd(CYTHON_UNUSED PyObject *__pyx_self, __pyx_r = ((PyObject *)__pyx_v_G); goto __pyx_L0; - /* "ot/emd/emd.pyx":21 + /* "ot/lp/emd.pyx":21 * @cython.boundscheck(False) * @cython.wraparound(False) - * def emd( np.ndarray[double, ndim=1, mode="c"] a,np.ndarray[double, ndim=1, mode="c"] b,np.ndarray[double, ndim=2, mode="c"] M): # <<<<<<<<<<<<<< + * def emd_c( np.ndarray[double, ndim=1, mode="c"] a,np.ndarray[double, ndim=1, mode="c"] b,np.ndarray[double, ndim=2, mode="c"] M): # <<<<<<<<<<<<<< * """ * Solves the Earth Movers distance problem and returns the optimal transport matrix */ @@ -1691,7 +1691,7 @@ static PyObject *__pyx_pf_2ot_3emd_3emd_emd(CYTHON_UNUSED PyObject *__pyx_self, __Pyx_SafeReleaseBuffer(&__pyx_pybuffernd_a.rcbuffer->pybuffer); __Pyx_SafeReleaseBuffer(&__pyx_pybuffernd_b.rcbuffer->pybuffer); __Pyx_ErrRestore(__pyx_type, __pyx_value, __pyx_tb);} - __Pyx_AddTraceback("ot.emd.emd.emd", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_AddTraceback("ot.lp.emd.emd_c", __pyx_clineno, __pyx_lineno, __pyx_filename); __pyx_r = NULL; goto __pyx_L2; __pyx_L0:; @@ -3884,8 +3884,8 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = { {&__pyx_n_s_a, __pyx_k_a, sizeof(__pyx_k_a), 0, 0, 1, 1}, {&__pyx_n_s_b, __pyx_k_b, sizeof(__pyx_k_b), 0, 0, 1, 1}, {&__pyx_n_s_cost, __pyx_k_cost, sizeof(__pyx_k_cost), 0, 0, 1, 1}, - {&__pyx_n_s_emd, __pyx_k_emd, sizeof(__pyx_k_emd), 0, 0, 1, 1}, - {&__pyx_kp_s_home_rflamary_PYTHON_POT_ot_emd, __pyx_k_home_rflamary_PYTHON_POT_ot_emd, sizeof(__pyx_k_home_rflamary_PYTHON_POT_ot_emd), 0, 0, 1, 0}, + {&__pyx_n_s_emd_c, __pyx_k_emd_c, sizeof(__pyx_k_emd_c), 0, 0, 1, 1}, + {&__pyx_kp_s_home_rflamary_PYTHON_POT_ot_lp, __pyx_k_home_rflamary_PYTHON_POT_ot_lp, sizeof(__pyx_k_home_rflamary_PYTHON_POT_ot_lp), 0, 0, 1, 0}, {&__pyx_n_s_import, __pyx_k_import, sizeof(__pyx_k_import), 0, 0, 1, 1}, {&__pyx_n_s_main, __pyx_k_main, sizeof(__pyx_k_main), 0, 0, 1, 1}, {&__pyx_n_s_n1, __pyx_k_n1, sizeof(__pyx_k_n1), 0, 0, 1, 1}, @@ -3895,7 +3895,7 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = { {&__pyx_n_s_np, __pyx_k_np, sizeof(__pyx_k_np), 0, 0, 1, 1}, {&__pyx_n_s_numpy, __pyx_k_numpy, sizeof(__pyx_k_numpy), 0, 0, 1, 1}, {&__pyx_n_s_ones, __pyx_k_ones, sizeof(__pyx_k_ones), 0, 0, 1, 1}, - {&__pyx_n_s_ot_emd_emd, __pyx_k_ot_emd_emd, sizeof(__pyx_k_ot_emd_emd), 0, 0, 1, 1}, + {&__pyx_n_s_ot_lp_emd, __pyx_k_ot_lp_emd, sizeof(__pyx_k_ot_lp_emd), 0, 0, 1, 1}, {&__pyx_n_s_range, __pyx_k_range, sizeof(__pyx_k_range), 0, 0, 1, 1}, {&__pyx_n_s_test, __pyx_k_test, sizeof(__pyx_k_test), 0, 0, 1, 1}, {&__pyx_kp_u_unknown_dtype_code_in_numpy_pxd, __pyx_k_unknown_dtype_code_in_numpy_pxd, sizeof(__pyx_k_unknown_dtype_code_in_numpy_pxd), 0, 1, 0, 0}, @@ -3981,17 +3981,17 @@ static int __Pyx_InitCachedConstants(void) { __Pyx_GOTREF(__pyx_tuple__6); __Pyx_GIVEREF(__pyx_tuple__6); - /* "ot/emd/emd.pyx":21 + /* "ot/lp/emd.pyx":21 * @cython.boundscheck(False) * @cython.wraparound(False) - * def emd( np.ndarray[double, ndim=1, mode="c"] a,np.ndarray[double, ndim=1, mode="c"] b,np.ndarray[double, ndim=2, mode="c"] M): # <<<<<<<<<<<<<< + * def emd_c( np.ndarray[double, ndim=1, mode="c"] a,np.ndarray[double, ndim=1, mode="c"] b,np.ndarray[double, ndim=2, mode="c"] M): # <<<<<<<<<<<<<< * """ * Solves the Earth Movers distance problem and returns the optimal transport matrix */ __pyx_tuple__7 = PyTuple_Pack(7, __pyx_n_s_a, __pyx_n_s_b, __pyx_n_s_M, __pyx_n_s_n1, __pyx_n_s_n2, __pyx_n_s_cost, __pyx_n_s_G); if (unlikely(!__pyx_tuple__7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 21; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_tuple__7); __Pyx_GIVEREF(__pyx_tuple__7); - __pyx_codeobj__8 = (PyObject*)__Pyx_PyCode_New(3, 0, 7, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__7, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_home_rflamary_PYTHON_POT_ot_emd, __pyx_n_s_emd, 21, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 21; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_codeobj__8 = (PyObject*)__Pyx_PyCode_New(3, 0, 7, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__7, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_home_rflamary_PYTHON_POT_ot_lp, __pyx_n_s_emd_c, 21, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 21; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_RefNannyFinishContext(); return 0; __pyx_L1_error:; @@ -4073,14 +4073,14 @@ PyMODINIT_FUNC PyInit_emd(void) #if PY_MAJOR_VERSION < 3 && (__PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT) if (__Pyx_init_sys_getdefaultencoding_params() < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} #endif - if (__pyx_module_is_main_ot__emd__emd) { + if (__pyx_module_is_main_ot__lp__emd) { if (PyObject_SetAttrString(__pyx_m, "__name__", __pyx_n_s_main) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} } #if PY_MAJOR_VERSION >= 3 { PyObject *modules = PyImport_GetModuleDict(); if (unlikely(!modules)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - if (!PyDict_GetItemString(modules, "ot.emd.emd")) { - if (unlikely(PyDict_SetItemString(modules, "ot.emd.emd", __pyx_m) < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (!PyDict_GetItemString(modules, "ot.lp.emd")) { + if (unlikely(PyDict_SetItemString(modules, "ot.lp.emd", __pyx_m) < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} } } #endif @@ -4112,7 +4112,7 @@ PyMODINIT_FUNC PyInit_emd(void) if (__Pyx_patch_abc() < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} #endif - /* "ot/emd/emd.pyx":7 + /* "ot/lp/emd.pyx":7 * @author: rflamary * """ * import numpy as np # <<<<<<<<<<<<<< @@ -4124,19 +4124,19 @@ PyMODINIT_FUNC PyInit_emd(void) if (PyDict_SetItem(__pyx_d, __pyx_n_s_np, __pyx_t_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 7; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "ot/emd/emd.pyx":21 + /* "ot/lp/emd.pyx":21 * @cython.boundscheck(False) * @cython.wraparound(False) - * def emd( np.ndarray[double, ndim=1, mode="c"] a,np.ndarray[double, ndim=1, mode="c"] b,np.ndarray[double, ndim=2, mode="c"] M): # <<<<<<<<<<<<<< + * def emd_c( np.ndarray[double, ndim=1, mode="c"] a,np.ndarray[double, ndim=1, mode="c"] b,np.ndarray[double, ndim=2, mode="c"] M): # <<<<<<<<<<<<<< * """ * Solves the Earth Movers distance problem and returns the optimal transport matrix */ - __pyx_t_1 = PyCFunction_NewEx(&__pyx_mdef_2ot_3emd_3emd_1emd, NULL, __pyx_n_s_ot_emd_emd); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 21; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyCFunction_NewEx(&__pyx_mdef_2ot_2lp_3emd_1emd_c, NULL, __pyx_n_s_ot_lp_emd); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 21; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_emd, __pyx_t_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 21; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (PyDict_SetItem(__pyx_d, __pyx_n_s_emd_c, __pyx_t_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 21; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "ot/emd/emd.pyx":1 + /* "ot/lp/emd.pyx":1 * # -*- coding: utf-8 -*- # <<<<<<<<<<<<<< * """ * Created on Thu Sep 11 08:42:08 2014 @@ -4161,11 +4161,11 @@ PyMODINIT_FUNC PyInit_emd(void) __Pyx_XDECREF(__pyx_t_1); if (__pyx_m) { if (__pyx_d) { - __Pyx_AddTraceback("init ot.emd.emd", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_AddTraceback("init ot.lp.emd", __pyx_clineno, __pyx_lineno, __pyx_filename); } Py_DECREF(__pyx_m); __pyx_m = 0; } else if (!PyErr_Occurred()) { - PyErr_SetString(PyExc_ImportError, "init ot.emd.emd"); + PyErr_SetString(PyExc_ImportError, "init ot.lp.emd"); } __pyx_L0:; __Pyx_RefNannyFinishContext(); diff --git a/ot/lp/emd.pyx b/ot/lp/emd.pyx index 753b195..de2d4a9 100644 --- a/ot/lp/emd.pyx +++ b/ot/lp/emd.pyx @@ -18,7 +18,7 @@ cdef extern from "EMD.h": @cython.boundscheck(False) @cython.wraparound(False) -def emd( np.ndarray[double, ndim=1, mode="c"] a,np.ndarray[double, ndim=1, mode="c"] b,np.ndarray[double, ndim=2, mode="c"] M): +def emd_c( np.ndarray[double, ndim=1, mode="c"] a,np.ndarray[double, ndim=1, mode="c"] b,np.ndarray[double, ndim=2, mode="c"] M): """ Solves the Earth Movers distance problem and returns the optimal transport matrix @@ -39,11 +39,11 @@ def emd( np.ndarray[double, ndim=1, mode="c"] a,np.ndarray[double, ndim=1, mode= Parameters ---------- - a : (ns,) ndarray - samples in the source domain (uniform waigth if empty) - b : (nt,) ndarray - samples in the target domain (uniform waigth if empty) - M : (ns,nt) ndarray + a : (ns,) ndarray, float64 + source histogram + b : (nt,) ndarray, float64 + target histogram + M : (ns,nt) ndarray, float64 loss matrix -- cgit v1.2.3 From 0e8363ca13034537ca7bc64acd982f39b7a42123 Mon Sep 17 00:00:00 2001 From: Rémi Flamary Date: Fri, 28 Oct 2016 09:14:19 +0200 Subject: doc sinkhorn --- docs/source/conf.py | 2 +- docs/source/index.rst | 2 +- examples/demo_OT_1D.py | 6 +++--- ot/bregman.py | 26 ++++++++++++++++++++++++-- ot/lp/__init__.py | 30 ++++++++++++++++++++++-------- 5 files changed, 51 insertions(+), 15 deletions(-) (limited to 'ot/bregman.py') diff --git a/docs/source/conf.py b/docs/source/conf.py index 51b71c9..1c1bb40 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -118,7 +118,7 @@ todo_include_todos = True # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = 'alabaster' +html_theme = 'sphinx_rtd_theme' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the diff --git a/docs/source/index.rst b/docs/source/index.rst index c95e847..cd1029e 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -3,7 +3,7 @@ You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. -Welcome to POT's documentation! +POT's documentation! =============================== Contents: diff --git a/examples/demo_OT_1D.py b/examples/demo_OT_1D.py index fc2cbcf..6eaa2ff 100644 --- a/examples/demo_OT_1D.py +++ b/examples/demo_OT_1D.py @@ -8,7 +8,7 @@ Demo for 1D optimal transport import numpy as np import matplotlib.pylab as pl import ot - +from ot.datasets import get_1D_gauss as gauss #%% parameters @@ -19,8 +19,8 @@ n=100 # nb bins x=np.arange(n,dtype=np.float64) # Gaussian distributions -a=ot.datasets.get_1D_gauss(n,m=20,s=20) # m= mean, s= std -b=ot.datasets.get_1D_gauss(n,m=60,s=60) +a=gauss(n,m=20,s=20) # m= mean, s= std +b=gauss(n,m=60,s=60) # loss matrix M=ot.dist(x.reshape((n,1)),x.reshape((n,1))) diff --git a/ot/bregman.py b/ot/bregman.py index 0d2c099..5d93fd6 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -26,7 +26,7 @@ def sinkhorn(a,b, M, reg,numItermax = 1000,stopThr=1e-9): - :math:`\Omega` is the entropic regularization term :math:`\Omega(\gamma)=\sum_{i,j} \gamma_{i,j}\log(\gamma_{i,j})` - a and b are source and target weights (sum to 1) - The algorithm used for solving the problem is the Sinkhorn-Knopp matrix scaling algorithm as proposed in [1]_ + The algorithm used for solving the problem is the Sinkhorn-Knopp matrix scaling algorithm as proposed in [2]_ Parameters @@ -46,10 +46,22 @@ def sinkhorn(a,b, M, reg,numItermax = 1000,stopThr=1e-9): gamma: (ns x nt) ndarray Optimal transportation matrix for the given parameters + + Examples + -------- + + >>> a=[.5,.5] + >>> b=[.5,.5] + >>> M=[[0.,1.],[1.,0.]] + >>> ot.sinkhorn(a,b,M,1) + array([[ 0.36552929, 0.13447071], + [ 0.13447071, 0.36552929]]) + + References ---------- - .. [1] M. Cuturi, Sinkhorn Distances : Lightspeed Computation of Optimal Transport, Advances in Neural Information Processing Systems (NIPS) 26, 2013 + .. [2] M. Cuturi, Sinkhorn Distances : Lightspeed Computation of Optimal Transport, Advances in Neural Information Processing Systems (NIPS) 26, 2013 See Also @@ -58,6 +70,16 @@ def sinkhorn(a,b, M, reg,numItermax = 1000,stopThr=1e-9): ot.optim.cg : General regularized OT """ + + a=np.asarray(a,dtype=np.float64) + b=np.asarray(b,dtype=np.float64) + M=np.asarray(M,dtype=np.float64) + + if len(a)==0: + a=np.ones((M.shape[0],),dtype=np.float64)/M.shape[0] + if len(b)==0: + b=np.ones((M.shape[1],),dtype=np.float64)/M.shape[1] + # init data Nini = len(a) Nfin = len(b) diff --git a/ot/lp/__init__.py b/ot/lp/__init__.py index 568e370..72b4cb8 100644 --- a/ot/lp/__init__.py +++ b/ot/lp/__init__.py @@ -7,7 +7,6 @@ def emd(a,b,M): """ Solves the Earth Movers distance problem and returns the optimal transport matrix - gamm=emd(a,b,M) .. math:: \gamma = arg\min_\gamma <\gamma,M>_F @@ -21,6 +20,8 @@ def emd(a,b,M): - M is the metric cost matrix - a and b are the sample weights + + Uses the algorithm proposed in [1]_ Parameters ---------- @@ -31,11 +32,17 @@ def emd(a,b,M): M : (ns,nt) ndarray, float64 loss matrix + Returns + ------- + gamma: (ns x nt) ndarray + Optimal transportation matrix for the given parameters + + Examples -------- - Simple example with obvious solution. The function :func:emd accepts lists and - perform automatic conversion tu numpy arrays + Simple example with obvious solution. The function emd accepts lists and + perform automatic conversion to numpy arrays >>> a=[.5,.5] >>> b=[.5,.5] @@ -43,15 +50,22 @@ def emd(a,b,M): >>> ot.emd(a,b,M) array([[ 0.5, 0. ], [ 0. , 0.5]]) + + References + ---------- - Returns - ------- - gamma: (ns x nt) ndarray - Optimal transportation matrix for the given parameters - + .. [1] Bonneel, N., Van De Panne, M., Paris, S., & Heidrich, W. (2011, December). Displacement interpolation using Lagrangian mass transport. In ACM Transactions on Graphics (TOG) (Vol. 30, No. 6, p. 158). ACM. + + See Also + -------- + ot.bregman.sinkhorn : Entropic regularized OT + ot.optim.cg : General regularized OT + + """ a=np.asarray(a,dtype=np.float64) b=np.asarray(b,dtype=np.float64) + M=np.asarray(M,dtype=np.float64) if len(a)==0: a=np.ones((M.shape[0],),dtype=np.float64)/M.shape[0] -- cgit v1.2.3 From a0d8139af3407e567e1dc9a5e8c10d9218ddd185 Mon Sep 17 00:00:00 2001 From: Rémi Flamary Date: Fri, 28 Oct 2016 09:54:11 +0200 Subject: doc index updated --- docs/source/all.rst | 47 ++++++++++++++++++++++++++++++++++ docs/source/examples.rst | 14 ++++++++++ docs/source/index.rst | 66 +++++++++++++++++++++++------------------------- ot/bregman.py | 5 ++++ 4 files changed, 97 insertions(+), 35 deletions(-) create mode 100644 docs/source/all.rst create mode 100644 docs/source/examples.rst (limited to 'ot/bregman.py') diff --git a/docs/source/all.rst b/docs/source/all.rst new file mode 100644 index 0000000..30f5add --- /dev/null +++ b/docs/source/all.rst @@ -0,0 +1,47 @@ + + +Python modules +============== + +ot +-- + +This module provide easy access to solvers for the most common OT problems + +.. automodule:: ot + :members: + +ot.lp +----- +.. automodule:: ot.lp + :members: + +ot.bregman +---------- + +.. automodule:: ot.bregman + :members: + +ot.optim +-------- + +.. automodule:: ot.optim + :members: + +ot.utils +-------- + +.. automodule:: ot.utils + :members: + +ot.datasets +----------- + +.. automodule:: ot.datasets + :members: + +ot.plot +------- + +.. automodule:: ot.plot + :members: diff --git a/docs/source/examples.rst b/docs/source/examples.rst new file mode 100644 index 0000000..606b2f5 --- /dev/null +++ b/docs/source/examples.rst @@ -0,0 +1,14 @@ + + +Examples +============ + +1D Optimal transport +--------------------- + +.. literalinclude:: ../../examples/demo_OT_1D.py + +2D Optimal transport on empirical distributions +----------------------------------------------- + +.. literalinclude:: ../../examples/demo_OT_2D_samples.py diff --git a/docs/source/index.rst b/docs/source/index.rst index cd1029e..8452f00 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -3,60 +3,56 @@ You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. -POT's documentation! -=============================== +POT: Python Optimal Transport +============================= -Contents: -.. toctree:: - :maxdepth: 2 +This open source Python library provide several solvers for optimization problems related to Optimal Transport for signal, image processing and machine learning. -Module list -=========== +It provides the following solvers: +* OT solver for the linear program/ Earth Movers Distance [1]. +* Entropic regularization OT solver with Sinkhorn Knopp Algorithm [2]. +* Bregman projections for Wasserstein barycenter [3] and unmixing [4]. +* Optimal transport for domain adaptation with group lasso regularization [5] +* Conditional gradient [6] and Generalized conditional gradient for regularized OT [7]. -Module ot ---------- +Some demonstrations (both in Python and Jupyter Notebook format) are available in the examples folder. -This module provide easy access to solvers for the most common OT problems -.. automodule:: ot - :members: +Contents +-------- -Module ot.emd -------------- -.. automodule:: ot.emd - :members: +.. toctree:: + :maxdepth: 2 -Module ot.bregman ------------------ + self + all + examples -.. automodule:: ot.bregman - :members: +Examples +-------- -Module ot.utils ---------------- -.. automodule:: ot.utils - :members: -Module ot.datasets ------------------- -.. automodule:: ot.datasets - :members: +References +---------- -Module ot.plot --------------- +[1] Bonneel, N., Van De Panne, M., Paris, S., & Heidrich, W. (2011, December). Displacement interpolation using Lagrangian mass transport. In ACM Transactions on Graphics (TOG) (Vol. 30, No. 6, p. 158). ACM. -.. automodule:: ot.plot - :members: +[2] Cuturi, M. (2013). Sinkhorn distances: Lightspeed computation of optimal transport. In Advances in Neural Information Processing Systems (pp. 2292-2300). +[3] Benamou, J. D., Carlier, G., Cuturi, M., Nenna, L., & Peyré, G. (2015). Iterative Bregman projections for regularized transportation problems. SIAM Journal on Scientific Computing, 37(2), A1111-A1138. -Examples -======== +[4] S. Nakhostin, N. Courty, R. Flamary, D. Tuia, T. Corpetti, Supervised planetary unmixing with optimal transport, Whorkshop on Hyperspectral Image and Signal Processing : Evolution in Remote Sensing (WHISPERS), 2016. + +[5] N. Courty; R. Flamary; D. Tuia; A. Rakotomamonjy, "Optimal Transport for Domain Adaptation," in IEEE Transactions on Pattern Analysis and Machine Intelligence , vol.PP, no.99, pp.1-1 + +[6] Ferradans, S., Papadakis, N., Peyré, G., & Aujol, J. F. (2014). Regularized discrete optimal transport. SIAM Journal on Imaging Sciences, 7(3), 1853-1882. + +[7] Rakotomamonjy, A., Flamary, R., & Courty, N. (2015). Generalized conditional gradient: analysis of convergence and applications. arXiv preprint arXiv:1510.06567. -.. literalinclude:: ../../examples/demo_OT_1D.py Indices and tables ================== diff --git a/ot/bregman.py b/ot/bregman.py index 5d93fd6..b749b13 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -39,6 +39,11 @@ def sinkhorn(a,b, M, reg,numItermax = 1000,stopThr=1e-9): loss matrix reg: float Regularization term >0 + numItermax: int, optional + Max number of iterations + stopThr: float, optional + Stop threshol on error (>0) + Returns -- cgit v1.2.3 From 8cd50c55f398cc371db2ef334c803dec99cc209a Mon Sep 17 00:00:00 2001 From: Rémi Flamary Date: Fri, 28 Oct 2016 10:58:04 +0200 Subject: update doc optim+bregman; add log to sinkhorn --- ot/__init__.py | 7 ++++--- ot/bregman.py | 27 +++++++++++++++++++++------ ot/lp/emd.cpp | 2 +- ot/optim.py | 58 ++++++++++++++++++++++++++++++++++++++++++++++++++++------ 4 files changed, 78 insertions(+), 16 deletions(-) (limited to 'ot/bregman.py') diff --git a/ot/__init__.py b/ot/__init__.py index 87119e5..863f408 100644 --- a/ot/__init__.py +++ b/ot/__init__.py @@ -1,13 +1,14 @@ # Python Optimal Transport toolbox # All submodules and packages +from . import lp +from . import bregman +from . import optim from . import utils from . import datasets from . import plot -from . import bregman -from . import lp from . import da -from . import optim + # OT functions diff --git a/ot/bregman.py b/ot/bregman.py index b749b13..08f965b 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -1,12 +1,12 @@ # -*- coding: utf-8 -*- """ -Bregman projection for regularized Otimal transport +Bregman projections for regularized OT """ import numpy as np -def sinkhorn(a,b, M, reg,numItermax = 1000,stopThr=1e-9): +def sinkhorn(a,b, M, reg,numItermax = 1000,stopThr=1e-9,verbose=False,log=False): """ Solve the entropic regularization optimal transport problem and return the OT matrix @@ -43,14 +43,18 @@ def sinkhorn(a,b, M, reg,numItermax = 1000,stopThr=1e-9): Max number of iterations stopThr: float, optional Stop threshol on error (>0) - + verbose : int, optional + Print information along iterations + log : int, optional + record log if True Returns ------- gamma: (ns x nt) ndarray Optimal transportation matrix for the given parameters - + log: dict + log dictionary return only if log==True in parameters Examples -------- @@ -91,6 +95,8 @@ def sinkhorn(a,b, M, reg,numItermax = 1000,stopThr=1e-9): cpt = 0 + if log: + log={'loss':[]} # we assume that no distances are null except those of the diagonal of distances u = np.ones(Nini)/Nini @@ -124,10 +130,19 @@ def sinkhorn(a,b, M, reg,numItermax = 1000,stopThr=1e-9): # we can speed up the process by checking for the error only all the 10th iterations transp = np.dot(np.diag(u),np.dot(K,np.diag(v))) err = np.linalg.norm((np.sum(transp,axis=0)-b))**2 + if log: + log['loss'].append(err) + + if verbose: + if cpt%200 ==0: + print('{:5s}|{:12s}'.format('It.','Err')+'\n'+'-'*19) + print('{:5d}|{:8e}|'.format(cpt,err)) cpt = cpt +1 #print 'err=',err,' cpt=',cpt - - return np.dot(np.diag(u),np.dot(K,np.diag(v))) + if log: + return np.dot(np.diag(u),np.dot(K,np.diag(v))),log + else: + return np.dot(np.diag(u),np.dot(K,np.diag(v))) def geometricBar(weights,alldistribT): diff --git a/ot/lp/emd.cpp b/ot/lp/emd.cpp index 26d243f..6db54bb 100644 --- a/ot/lp/emd.cpp +++ b/ot/lp/emd.cpp @@ -1229,7 +1229,7 @@ static PyObject *__pyx_codeobj__8; /* Python wrapper */ static PyObject *__pyx_pw_2ot_2lp_3emd_1emd_c(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ -static char __pyx_doc_2ot_2lp_3emd_emd_c[] = "\n Solves the Earth Movers distance problem and returns the optimal transport matrix\n \n gamm=emd(a,b,M)\n \n .. math::\n \\gamma = arg\\min_\\gamma <\\gamma,M>_F \n \n s.t. \\gamma 1 = a\n \n \\gamma^T 1= b \n \n \\gamma\\geq 0\n where :\n \n - M is the metric cost matrix\n - a and b are the sample weights\n \n Parameters\n ----------\n a : (ns,) ndarray\n source histogram \n b : (nt,) ndarray\n target histogram\n M : (ns,nt) ndarray\n loss matrix \n \n \n Returns\n -------\n gamma: (ns x nt) ndarray\n Optimal transportation matrix for the given parameters\n \n "; +static char __pyx_doc_2ot_2lp_3emd_emd_c[] = "\n Solves the Earth Movers distance problem and returns the optimal transport matrix\n \n gamm=emd(a,b,M)\n \n .. math::\n \\gamma = arg\\min_\\gamma <\\gamma,M>_F \n \n s.t. \\gamma 1 = a\n \n \\gamma^T 1= b \n \n \\gamma\\geq 0\n where :\n \n - M is the metric cost matrix\n - a and b are the sample weights\n \n Parameters\n ----------\n a : (ns,) ndarray, float64\n source histogram \n b : (nt,) ndarray, float64\n target histogram\n M : (ns,nt) ndarray, float64\n loss matrix \n \n \n Returns\n -------\n gamma: (ns x nt) ndarray\n Optimal transportation matrix for the given parameters\n \n "; static PyMethodDef __pyx_mdef_2ot_2lp_3emd_1emd_c = {"emd_c", (PyCFunction)__pyx_pw_2ot_2lp_3emd_1emd_c, METH_VARARGS|METH_KEYWORDS, __pyx_doc_2ot_2lp_3emd_emd_c}; static PyObject *__pyx_pw_2ot_2lp_3emd_1emd_c(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { PyArrayObject *__pyx_v_a = 0; diff --git a/ot/optim.py b/ot/optim.py index e6373ce..d1bf672 100644 --- a/ot/optim.py +++ b/ot/optim.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- """ -Created on Wed Oct 26 15:08:19 2016 - -@author: rflamary +Optimization algorithms for OT """ import numpy as np @@ -12,6 +10,42 @@ from lp import emd # The corresponding scipy function does not work for matrices def line_search_armijo(f,xk,pk,gfk,old_fval,args=(),c1=1e-4,alpha0=0.99): + """ + Armijo linesearch function that works with matrices + + find an approximate minimum of f(xk+alpha*pk) that satifies the + armijo conditions. + + Parameters + ---------- + + f : function + loss function + xk : np.ndarray + initial position + pk : np.ndarray + descent direction + gfk : np.ndarray + gradient of f at xk + old_fval: float + loss value at xk + args : tuple, optional + arguments given to f + c1 : float, optional + c1 const in armijo rule (>0) + alpha0 : float, optional + initial step (>0) + + Returns + ------- + alpha : float + step that satisfy armijo conditions + fc : int + nb of function call + fa : float + loss value at step alpha + + """ xk = np.atleast_1d(xk) fc = [0] @@ -61,14 +95,26 @@ def cg(a,b,M,reg,f,df,G0=None,numItermax = 200,stopThr=1e-9,verbose=False,log=Fa samples in the target domain M : np.ndarray (ns,nt) loss matrix - reg: float() + reg : float Regularization term >0 - + G0 : np.ndarray (ns,nt), optional + initial guess (default is indep joint density) + numItermax : int, optional + Max number of iterations + stopThr : float, optional + Stop threshol on error (>0) + verbose : int, optional + Print information along iterations + log : int, optional + record log if True Returns ------- gamma: (ns x nt) ndarray Optimal transportation matrix for the given parameters + log: dict + log dictionary return only if log==True in parameters + References ---------- @@ -77,7 +123,7 @@ def cg(a,b,M,reg,f,df,G0=None,numItermax = 200,stopThr=1e-9,verbose=False,log=Fa See Also -------- - ot.emd.emd : Unregularized optimal ransport + ot.lp.emd : Unregularized optimal ransport ot.bregman.sinkhorn : Entropic regularized optimal transport """ -- cgit v1.2.3 From 3067c8873bf325808985453f0ac968d13435e032 Mon Sep 17 00:00:00 2001 From: Rémi Flamary Date: Fri, 28 Oct 2016 11:22:23 +0200 Subject: update bregman with doc --- examples/demo_barycenter_1D.py | 2 +- ot/bregman.py | 91 +++++++++++++++++++++++++++++++++++------- ot/lp/__init__.py | 5 ++- ot/optim.py | 4 +- 4 files changed, 83 insertions(+), 19 deletions(-) (limited to 'ot/bregman.py') diff --git a/examples/demo_barycenter_1D.py b/examples/demo_barycenter_1D.py index c9f63a2..2376f7b 100644 --- a/examples/demo_barycenter_1D.py +++ b/examples/demo_barycenter_1D.py @@ -43,7 +43,7 @@ bary_l2=A.mean(1) # wasserstein reg=1e-3 -bary_wass,log=ot.bregman.barycenter(A,M,reg) +bary_wass=ot.bregman.barycenter(A,M,reg) pl.figure(2) pl.clf() diff --git a/ot/bregman.py b/ot/bregman.py index 08f965b..b6cdf80 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -43,9 +43,9 @@ def sinkhorn(a,b, M, reg,numItermax = 1000,stopThr=1e-9,verbose=False,log=False) Max number of iterations stopThr: float, optional Stop threshol on error (>0) - verbose : int, optional + verbose : bool, optional Print information along iterations - log : int, optional + log : bool, optional record log if True @@ -96,7 +96,7 @@ def sinkhorn(a,b, M, reg,numItermax = 1000,stopThr=1e-9,verbose=False,log=False) cpt = 0 if log: - log={'loss':[]} + log={'err':[]} # we assume that no distances are null except those of the diagonal of distances u = np.ones(Nini)/Nini @@ -131,7 +131,7 @@ def sinkhorn(a,b, M, reg,numItermax = 1000,stopThr=1e-9,verbose=False,log=False) transp = np.dot(np.diag(u),np.dot(K,np.diag(v))) err = np.linalg.norm((np.sum(transp,axis=0)-b))**2 if log: - log['loss'].append(err) + log['err'].append(err) if verbose: if cpt%200 ==0: @@ -146,10 +146,12 @@ def sinkhorn(a,b, M, reg,numItermax = 1000,stopThr=1e-9,verbose=False,log=False) def geometricBar(weights,alldistribT): + """return the weighted geometric mean of distributions""" assert(len(weights)==alldistribT.shape[1]) return np.exp(np.dot(np.log(alldistribT),weights.T)) def geometricMean(alldistribT): + """return the geometric mean of distributions""" return np.exp(np.mean(np.log(alldistribT),axis=1)) def projR(gamma,p): @@ -161,16 +163,66 @@ def projC(gamma,q): return np.multiply(gamma,q/np.maximum(np.sum(gamma,axis=0),1e-10)) -def barycenter(A,M,reg, weights=None, numItermax = 1000, tol_error=1e-4,log=dict()): - """Compute the Regularizzed wassersteien barycenter of distributions A""" +def barycenter(A,M,reg, weights=None, numItermax = 1000, stopThr=1e-4,verbose=False,log=False): + """Compute the entropic regularized wasserstein barycenter of distributions A + + The function solves the following optimization problem: + + .. math:: + \mathbf{a} = arg\min_\mathbf{a} \sum_i W_{reg}(\mathbf{a},\mathbf{a}_i) + + where : + + - :math:`W_{reg}(\cdot,\cdot)` is the entropic regularized Wasserstein distance (see ot.bregman.sinkhorn) + - :math:`\mathbf{a}_i` are training distributions in the columns of matrix :math:`\mathbf{A}` + - reg and :math:`\mathbf{M}` are respectively the regularization term and the cost matrix for OT + + The algorithm used for solving the problem is the Sinkhorn-Knopp matrix scaling algorithm as proposed in [3]_ + + Parameters + ---------- + A : np.ndarray (d,n) + n training distributions of size d + M : np.ndarray (ns,nt) + loss matrix for OT + reg: float + Regularization term >0 + numItermax: int, optional + Max number of iterations + stopThr: float, optional + Stop threshol on error (>0) + verbose : bool, optional + Print information along iterations + log : bool, optional + record log if True + + + Returns + ------- + a: (d,) ndarray + Wasserstein barycenter + log: dict + log dictionary return only if log==True in parameters + + + References + ---------- + + .. [3] Benamou, J. D., Carlier, G., Cuturi, M., Nenna, L., & Peyré, G. (2015). Iterative Bregman projections for regularized transportation problems. SIAM Journal on Scientific Computing, 37(2), A1111-A1138. + + + + """ if weights is None: weights=np.ones(A.shape[1])/A.shape[1] else: assert(len(weights)==A.shape[1]) + + if log: + log={'err':[]} - #compute Mmax once for all #M = M/np.median(M) # suggested by G. Peyre K = np.exp(-M/reg) @@ -180,19 +232,28 @@ def barycenter(A,M,reg, weights=None, numItermax = 1000, tol_error=1e-4,log=dict UKv=np.dot(K,np.divide(A.T,np.sum(K,axis=0)).T) u = (geometricMean(UKv)/UKv.T).T - log['niter']=0 - log['all_err']=[] - - while (err>tol_error and cptstopThr and cpt0) - verbose : int, optional + verbose : bool, optional Print information along iterations - log : int, optional + log : bool, optional record log if True Returns -- cgit v1.2.3 From 062d6fd23baa27e0334023af320230120b50c828 Mon Sep 17 00:00:00 2001 From: Rémi Flamary Date: Fri, 28 Oct 2016 11:47:22 +0200 Subject: bregman doc finished --- ot/bregman.py | 89 +++++++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 78 insertions(+), 11 deletions(-) (limited to 'ot/bregman.py') diff --git a/ot/bregman.py b/ot/bregman.py index b6cdf80..9183bea 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -183,7 +183,7 @@ def barycenter(A,M,reg, weights=None, numItermax = 1000, stopThr=1e-4,verbose=Fa ---------- A : np.ndarray (d,n) n training distributions of size d - M : np.ndarray (ns,nt) + M : np.ndarray (d,d) loss matrix for OT reg: float Regularization term >0 @@ -256,15 +256,73 @@ def barycenter(A,M,reg, weights=None, numItermax = 1000, stopThr=1e-4,verbose=Fa return geometricBar(weights,UKv) -def unmix(distrib,D,M,M0,h0,reg,reg0,alpha,numItermax = 1000, tol_error=1e-3,log=dict()): +def unmix(a,D,M,M0,h0,reg,reg0,alpha,numItermax = 1000, stopThr=1e-3,verbose=False,log=False): """ - distrib : distribution to unmix + Compute the unmixing of an observation with a given dictionary using Wasserstein distance + + The function solve the following optimization problem: + + .. math:: + \mathbf{h} = arg\min_\mathbf{h} (1- \\alpha) W_{M,reg}(\mathbf{a},\mathbf{Dh})+\\alpha W_{M0,reg0}(\mathbf{h}_0,\mathbf{h}) + + + where : + + - :math:`W_{M,reg}(\cdot,\cdot)` is the entropic regularized Wasserstein distance with M loss matrix (see ot.bregman.sinkhorn) + - :math:`\mathbf{a}` is an observed distribution, :math:`\mathbf{h}_0` is aprior on unmixing + - reg and :math:`\mathbf{M}` are respectively the regularization term and the cost matrix for OT data fitting + - reg0 and :math:`\mathbf{M0}` are respectively the regularization term and the cost matrix for regularization + - :math:`\\alpha`weight data fitting and regularization + + The optimization problem is solved suing the algorithm described in [4] + + + distrib : distribution to unmix D : Dictionnary M : Metric matrix in the space of the distributions to unmix M0 : Metric matrix in the space of the 'abundance values' to solve for h0 : prior on solution (generally uniform distribution) reg,reg0 : transport regularizations alpha : how much should we trust the prior ? ([0,1]) + + Parameters + ---------- + a : np.ndarray (d) + observed distribution + D : np.ndarray (d,n) + dictionary matrix + M : np.ndarray (d,d) + loss matrix + M0 : np.ndarray (n,n) + loss matrix + h0 : np.ndarray (n,) + prior on h + reg: float + Regularization term >0 (Wasserstein data fitting) + reg0: float + Regularization term >0 (Wasserstein reg with h0) + numItermax: int, optional + Max number of iterations + stopThr: float, optional + Stop threshol on error (>0) + verbose : bool, optional + Print information along iterations + log : bool, optional + record log if True + + + Returns + ------- + a: (d,) ndarray + Wasserstein barycenter + log: dict + log dictionary return only if log==True in parameters + + References + ---------- + + .. [4] S. Nakhostin, N. Courty, R. Flamary, D. Tuia, T. Corpetti, Supervised planetary unmixing with optimal transport, Whorkshop on Hyperspectral Image and Signal Processing : Evolution in Remote Sensing (WHISPERS), 2016. + """ #M = M/np.median(M) @@ -277,12 +335,12 @@ def unmix(distrib,D,M,M0,h0,reg,reg0,alpha,numItermax = 1000, tol_error=1e-3,log err=1 cpt=0 #log = {'niter':0, 'all_err':[]} - log['niter']=0 - log['all_err']=[] + if log: + log={'err':[]} - while (err>tol_error and cptstopThr and cpt Date: Fri, 28 Oct 2016 12:50:54 +0200 Subject: doc datasets.py --- ot/bregman.py | 14 ++++-------- ot/datasets.py | 71 +++++++++++++++++++++++++++++++++++++++++++++++++--------- ot/utils.py | 23 +++++++++++++++---- 3 files changed, 83 insertions(+), 25 deletions(-) (limited to 'ot/bregman.py') diff --git a/ot/bregman.py b/ot/bregman.py index 9183bea..ad9a67a 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -276,14 +276,6 @@ def unmix(a,D,M,M0,h0,reg,reg0,alpha,numItermax = 1000, stopThr=1e-3,verbose=Fal The optimization problem is solved suing the algorithm described in [4] - - distrib : distribution to unmix - D : Dictionnary - M : Metric matrix in the space of the distributions to unmix - M0 : Metric matrix in the space of the 'abundance values' to solve for - h0 : prior on solution (generally uniform distribution) - reg,reg0 : transport regularizations - alpha : how much should we trust the prior ? ([0,1]) Parameters ---------- @@ -300,7 +292,9 @@ def unmix(a,D,M,M0,h0,reg,reg0,alpha,numItermax = 1000, stopThr=1e-3,verbose=Fal reg: float Regularization term >0 (Wasserstein data fitting) reg0: float - Regularization term >0 (Wasserstein reg with h0) + Regularization term >0 (Wasserstein reg with h0) + alpha: float + How much should we trust the prior ([0,1]) numItermax: int, optional Max number of iterations stopThr: float, optional @@ -318,7 +312,7 @@ def unmix(a,D,M,M0,h0,reg,reg0,alpha,numItermax = 1000, stopThr=1e-3,verbose=Fal log: dict log dictionary return only if log==True in parameters - References + References ---------- .. [4] S. Nakhostin, N. Courty, R. Flamary, D. Tuia, T. Corpetti, Supervised planetary unmixing with optimal transport, Whorkshop on Hyperspectral Image and Signal Processing : Evolution in Remote Sensing (WHISPERS), 2016. diff --git a/ot/datasets.py b/ot/datasets.py index f22e345..6388d94 100644 --- a/ot/datasets.py +++ b/ot/datasets.py @@ -8,14 +8,50 @@ import scipy as sp def get_1D_gauss(n,m,s): - "return a 1D histogram for a gaussian distribution (n bins, mean m and std s) " + """return a 1D histogram for a gaussian distribution (n bins, mean m and std s) + + Parameters + ---------- + + n : int + number of bins in the histogram + m : float + mean value of the gaussian distribution + s : float + standard deviaton of the gaussian distribution + + + Returns + ------- + h : np.array (n,) + 1D histogram for a gaussian distribution + + """ x=np.arange(n,dtype=np.float64) h=np.exp(-(x-m)**2/(2*s^2)) return h/h.sum() def get_2D_samples_gauss(n,m,sigma): - "return samples from 2D gaussian (n samples, mean m and cov sigma) " + """return n samples drawn from 2D gaussian N(m,sigma) + + Parameters + ---------- + + n : int + number of bins in the histogram + m : np.array (2,) + mean value of the gaussian distribution + sigma : np.array (2,2) + covariance matrix of the gaussian distribution + + + Returns + ------- + X : np.array (n,2) + n samples drawn from N(m,sigma) + + """ if np.isscalar(sigma): sigma=np.array([sigma,]) if len(sigma)>1: @@ -26,8 +62,26 @@ def get_2D_samples_gauss(n,m,sigma): return res def get_data_classif(dataset,n,nz=.5,**kwargs): - """ - dataset generation + """ dataset generation for classification problems + + Parameters + ---------- + + dataset : str + type of classification problem (see code) + n : int + number of training samples + nz : float + noise level (>0) + + + Returns + ------- + X : np.array (n,d) + n observation of size d + y : np.array (n,) + labels of the samples + """ if dataset.lower()=='3gauss': y=np.floor((np.arange(n)*1.0/n*3))+1 @@ -50,15 +104,10 @@ def get_data_classif(dataset,n,nz=.5,**kwargs): x[y==3,0]=2. ; x[y==3,1]=0 x[y!=3,:]+=nz*np.random.randn(sum(y!=3),2) - x[y==3,:]+=2*nz*np.random.randn(sum(y==3),2) -# elif dataset.lower()=='sinreg': -# -# x=np.random.rand(n,1) -# y=4*x+np.sin(2*np.pi*x)+nz*np.random.randn(n,1) - + x[y==3,:]+=2*nz*np.random.randn(sum(y==3),2) else: x=0 y=0 print("unknown dataset") - return x,y \ No newline at end of file + return x,y.astype(int) \ No newline at end of file diff --git a/ot/utils.py b/ot/utils.py index e5ec864..2110c01 100644 --- a/ot/utils.py +++ b/ot/utils.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- """ Various function that can be usefull """ @@ -6,7 +7,21 @@ from scipy.spatial.distance import cdist def unif(n): - """ return a uniform histogram of length n (simplex) """ + """ return a uniform histogram of length n (simplex) + + Parameters + ---------- + + n : int + number of bins in the histogram + + Returns + ------- + h : np.array (n,) + histogram of length n such that h_i=1/n for all i + + + """ return np.ones((n,))/n @@ -22,9 +37,9 @@ def dist(x1,x2=None,metric='sqeuclidean'): matrix with n2 samples of size d (if None then x2=x1) metric : str, fun, optional name of the metric to be computed (full list in the doc of scipy), If a string, - the distance function can be ‘braycurtis’, ‘canberra’, ‘chebyshev’, ‘cityblock’, + the distance function can be ‘braycurtis’, ‘canberra’, ‘chebyshev’, ‘cityblock’, ‘correlation’, ‘cosine’, ‘dice’, ‘euclidean’, ‘hamming’, ‘jaccard’, ‘kulsinski’, - ‘mahalanobis’, ‘matching’, ‘minkowski’, ‘rogerstanimoto’, ‘russellrao’, ‘seuclidean’, + ‘mahalanobis’, ‘matching’, ‘minkowski’, ‘rogerstanimoto’, ‘russellrao’, ‘seuclidean’, ‘sokalmichener’, ‘sokalsneath’, ‘sqeuclidean’, ‘wminkowski’, ‘yule’. @@ -68,5 +83,5 @@ def dist0(n,method='lin_square'): def dots(*args): - """ Stupid but nice dots function for multiple matrix multiply """ + """ dots function for multiple matrix multiply """ return reduce(np.dot,args) \ No newline at end of file -- cgit v1.2.3 From 0d81de9909e8e9eb95858f0a043550b15898f172 Mon Sep 17 00:00:00 2001 From: Rémi Flamary Date: Fri, 28 Oct 2016 15:13:48 +0200 Subject: doc da.py --- docs/source/all.rst | 8 +++-- ot/__init__.py | 15 +++++---- ot/bregman.py | 8 ++--- ot/da.py | 91 ++++++++++++++++++++++++++++++++++++++++++++++++++--- ot/lp/__init__.py | 77 ++++++++++++++++++++++----------------------- 5 files changed, 142 insertions(+), 57 deletions(-) (limited to 'ot/bregman.py') diff --git a/docs/source/all.rst b/docs/source/all.rst index 30f5add..d5733b8 100644 --- a/docs/source/all.rst +++ b/docs/source/all.rst @@ -6,8 +6,6 @@ Python modules ot -- -This module provide easy access to solvers for the most common OT problems - .. automodule:: ot :members: @@ -28,6 +26,12 @@ ot.optim .. automodule:: ot.optim :members: +ot.da +-------- + +.. automodule:: ot.da + :members: + ot.utils -------- diff --git a/ot/__init__.py b/ot/__init__.py index 0602eed..72c820a 100644 --- a/ot/__init__.py +++ b/ot/__init__.py @@ -1,22 +1,21 @@ -# Python Optimal Transport toolbox +"""Python Optimal Transport toolbox""" # All submodules and packages -from . import lp +from . import lp from . import bregman -from . import optim +from . import optim from . import utils from . import datasets from . import plot from . import da - - # OT functions from .lp import emd -from .bregman import sinkhorn,barycenter +from .bregman import sinkhorn, barycenter from .da import sinkhorn_lpl1_mm # utils functions -from .utils import dist,unif +from .utils import dist, unif -__all__ = ["emd","sinkhorn","utils",'datasets','bregman','lp','plot','dist','unif','barycenter','sinkhorn_lpl1_mm','da','optim'] +__all__ = ["emd", "sinkhorn", "utils", 'datasets', 'bregman', 'lp', 'plot', + 'dist', 'unif', 'barycenter', 'sinkhorn_lpl1_mm', 'da', 'optim'] diff --git a/ot/bregman.py b/ot/bregman.py index ad9a67a..2d82ae4 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -6,12 +6,12 @@ Bregman projections for regularized OT import numpy as np -def sinkhorn(a,b, M, reg,numItermax = 1000,stopThr=1e-9,verbose=False,log=False): +def sinkhorn(a,b, M, reg, numItermax = 1000, stopThr=1e-9, verbose=False, log=False): """ - Solve the entropic regularization optimal transport problem and return the OT matrix - + Solve the entropic regularization optimal transport problem + The function solves the following optimization problem: - + .. math:: \gamma = arg\min_\gamma <\gamma,M>_F + reg\cdot\Omega(\gamma) diff --git a/ot/da.py b/ot/da.py index bd20014..300d21a 100644 --- a/ot/da.py +++ b/ot/da.py @@ -1,6 +1,8 @@ +# -*- coding: utf-8 -*- """ -domain adaptation with optimal transport +Domain adaptation with optimal transport """ + import numpy as np from .bregman import sinkhorn @@ -9,7 +11,88 @@ from .bregman import sinkhorn def indices(a, func): return [i for (i, val) in enumerate(a) if func(val)] -def sinkhorn_lpl1_mm(a,labels_a, b, M, reg, eta=0.1): +def sinkhorn_lpl1_mm(a,labels_a, b, M, reg, eta=0.1,numItermax = 10,numInnerItermax = 200,stopInnerThr=1e-9,verbose=False,log=False): + """ + Solve the entropic regularization optimal transport problem with nonconvex group lasso regularization + + The function solves the following optimization problem: + + .. math:: + \gamma = arg\min_\gamma <\gamma,M>_F + reg\cdot\Omega_e(\gamma)+ \eta \Omega_g(\gamma) + + s.t. \gamma 1 = a + + \gamma^T 1= b + + \gamma\geq 0 + where : + + - M is the (ns,nt) metric cost matrix + - :math:`\Omega_e` is the entropic regularization term :math:`\Omega_e(\gamma)=\sum_{i,j} \gamma_{i,j}\log(\gamma_{i,j})` + - :math:`\Omega_g` is the group lasso regulaization term :math:`\Omega_g(\gamma)=\sum_{i,c} \|\gamma_{i,\mathcal{I}_c}\|^{1/2}_1` where :math:`\mathcal{I}_c` are the index of samples from class c in the source domain. + - a and b are source and target weights (sum to 1) + + The algorithm used for solving the problem is the generalised conditional gradient as proposed in [5]_ [7]_ + + + Parameters + ---------- + a : np.ndarray (ns,) + samples weights in the source domain + labels_a : np.ndarray (ns,) + labels of samples in the source domain + b : np.ndarray (nt,) + samples in the target domain + M : np.ndarray (ns,nt) + loss matrix + reg: float + Regularization term for entropic regularization >0 + eta: float, optional + Regularization term for group lasso regularization >0 + numItermax: int, optional + Max number of iterations + numInnerItermax: int, optional + Max number of iterations (inner sinkhorn solver) + stopInnerThr: float, optional + Stop threshold on error (inner sinkhorn solver) (>0) + verbose : bool, optional + Print information along iterations + log : bool, optional + record log if True + + + Returns + ------- + gamma: (ns x nt) ndarray + Optimal transportation matrix for the given parameters + log: dict + log dictionary return only if log==True in parameters + + Examples + -------- + + >>> a=[.5,.5] + >>> b=[.5,.5] + >>> M=[[0.,1.],[1.,0.]] + >>> ot.sinkhorn(a,b,M,1) + array([[ 0.36552929, 0.13447071], + [ 0.13447071, 0.36552929]]) + + + References + ---------- + + .. [5] N. Courty; R. Flamary; D. Tuia; A. Rakotomamonjy, "Optimal Transport for Domain Adaptation," in IEEE Transactions on Pattern Analysis and Machine Intelligence , vol.PP, no.99, pp.1-1 + + .. [7] Rakotomamonjy, A., Flamary, R., & Courty, N. (2015). Generalized conditional gradient: analysis of convergence and applications. arXiv preprint arXiv:1510.06567. + + See Also + -------- + ot.lp.emd : Unregularized OT + ot.bregman.sinkhorn : Entropic regularized OT + ot.optim.cg : General regularized OT + + """ p=0.5 epsilon = 1e-3 @@ -25,9 +108,9 @@ def sinkhorn_lpl1_mm(a,labels_a, b, M, reg, eta=0.1): W=np.zeros(M.shape) - for cpt in range(10): + for cpt in range(numItermax): Mreg = M + eta*W - transp=sinkhorn(a,b,Mreg,reg,numItermax = 200) + transp=sinkhorn(a,b,Mreg,reg,numItermax=numInnerItermax, stopThr=stopInnerThr) # the transport has been computed. Check if classes are really separated W = np.ones((Nini,Nfin)) for t in range(Nfin): diff --git a/ot/lp/__init__.py b/ot/lp/__init__.py index 6c7822a..2adf937 100644 --- a/ot/lp/__init__.py +++ b/ot/lp/__init__.py @@ -1,31 +1,30 @@ +# -*- coding: utf-8 -*- """ Solvers for the original linear program OT problem """ +import numpy as np # import compiled emd from .emd import emd_c -import numpy as np -def emd(a,b,M): - """ - Solves the Earth Movers distance problem and returns the optimal transport matrix - - + +def emd(a, b, M): + """Solves the Earth Movers distance problem and returns the OT matrix + + .. math:: - \gamma = arg\min_\gamma <\gamma,M>_F - + \gamma = arg\min_\gamma <\gamma,M>_F + s.t. \gamma 1 = a - - \gamma^T 1= b - + \gamma^T 1= b \gamma\geq 0 where : - + - M is the metric cost matrix - a and b are the sample weights - + Uses the algorithm proposed in [1]_ - + Parameters ---------- a : (ns,) ndarray, float64 @@ -33,47 +32,47 @@ def emd(a,b,M): b : (nt,) ndarray, float64 Target histogram (uniform weigth if empty list) M : (ns,nt) ndarray, float64 - loss matrix - + loss matrix + Returns ------- gamma: (ns x nt) ndarray Optimal transportation matrix for the given parameters - - + + Examples -------- - + Simple example with obvious solution. The function emd accepts lists and - perform automatic conversion to numpy arrays - + perform automatic conversion to numpy arrays + >>> a=[.5,.5] >>> b=[.5,.5] >>> M=[[0.,1.],[1.,0.]] >>> ot.emd(a,b,M) array([[ 0.5, 0. ], [ 0. , 0.5]]) - + References ---------- - - .. [1] Bonneel, N., Van De Panne, M., Paris, S., & Heidrich, W. (2011, December). Displacement interpolation using Lagrangian mass transport. In ACM Transactions on Graphics (TOG) (Vol. 30, No. 6, p. 158). ACM. - + + .. [1] Bonneel, N., Van De Panne, M., Paris, S., & Heidrich, W. + (2011, December). Displacement interpolation using Lagrangian mass + transport. In ACM Transactions on Graphics (TOG) (Vol. 30, No. 6, p. + 158). ACM. + See Also -------- ot.bregman.sinkhorn : Entropic regularized OT - ot.optim.cg : General regularized OT - - - """ - a=np.asarray(a,dtype=np.float64) - b=np.asarray(b,dtype=np.float64) - M=np.asarray(M,dtype=np.float64) - - if len(a)==0: - a=np.ones((M.shape[0],),dtype=np.float64)/M.shape[0] - if len(b)==0: - b=np.ones((M.shape[1],),dtype=np.float64)/M.shape[1] - - return emd_c(a,b,M) + ot.optim.cg : General regularized OT""" + + a = np.asarray(a, dtype=np.float64) + b = np.asarray(b, dtype=np.float64) + M = np.asarray(M, dtype=np.float64) + + if len(a) == 0: + a = np.ones((M.shape[0], ), dtype=np.float64)/M.shape[0] + if len(b) == 0: + b = np.ones((M.shape[1], ), dtype=np.float64)/M.shape[1] + return emd_c(a, b, M) -- cgit v1.2.3 From a5f2569859424a00100f7ad29ae4d715ee90c29f Mon Sep 17 00:00:00 2001 From: Rémi Flamary Date: Fri, 4 Nov 2016 12:09:36 +0100 Subject: doc nonlinear mapping estimation --- README.md | 2 + docs/source/readme.rst | 3 + ot/bregman.py | 232 ++++++++++++++++++++++++------------------------- ot/da.py | 123 +++++++++++++++++++++----- ot/optim.py | 6 +- 5 files changed, 226 insertions(+), 140 deletions(-) (limited to 'ot/bregman.py') diff --git a/README.md b/README.md index cd9e281..f73cbe5 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,8 @@ The examples folder contain several examples and use case for the library. The f * [Color transfer in images](https://github.com/rflamary/POT/blob/master/examples/Demo_Image_ColorAdaptation.ipynb) * [OT mapping estimation for domain adaptation](https://github.com/rflamary/POT/blob/master/examples/Demo_2D_OTmapping_DomainAdaptation.ipynb) +You can also see the notebooks with [Jupyter nbviewer](https://nbviewer.jupyter.org/github/rflamary/POT/tree/master/examples/). + ## Acknowledgements The contributors to this library are: diff --git a/docs/source/readme.rst b/docs/source/readme.rst index 7b88cad..653adaf 100644 --- a/docs/source/readme.rst +++ b/docs/source/readme.rst @@ -86,6 +86,9 @@ Here is a list of the Python notebooks if you want a quick look: - `OT mapping estimation for domain adaptation `__ +You can also see the notebooks with `Jupyter +nbviewer `__. + Acknowledgements ---------------- diff --git a/ot/bregman.py b/ot/bregman.py index 2d82ae4..a770c5a 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -14,21 +14,21 @@ def sinkhorn(a,b, M, reg, numItermax = 1000, stopThr=1e-9, verbose=False, log=Fa .. math:: \gamma = arg\min_\gamma <\gamma,M>_F + reg\cdot\Omega(\gamma) - + s.t. \gamma 1 = a - - \gamma^T 1= b - + + \gamma^T 1= b + \gamma\geq 0 where : - + - M is the (ns,nt) metric cost matrix - :math:`\Omega` is the entropic regularization term :math:`\Omega(\gamma)=\sum_{i,j} \gamma_{i,j}\log(\gamma_{i,j})` - a and b are source and target weights (sum to 1) - + The algorithm used for solving the problem is the Sinkhorn-Knopp matrix scaling algorithm as proposed in [2]_ - - + + Parameters ---------- a : np.ndarray (ns,) @@ -36,79 +36,79 @@ def sinkhorn(a,b, M, reg, numItermax = 1000, stopThr=1e-9, verbose=False, log=Fa b : np.ndarray (nt,) samples in the target domain M : np.ndarray (ns,nt) - loss matrix - reg: float + loss matrix + reg : float Regularization term >0 - numItermax: int, optional + numItermax : int, optional Max number of iterations - stopThr: float, optional + stopThr : float, optional Stop threshol on error (>0) verbose : bool, optional Print information along iterations log : bool, optional - record log if True - - + record log if True + + Returns ------- - gamma: (ns x nt) ndarray + gamma : (ns x nt) ndarray Optimal transportation matrix for the given parameters - log: dict - log dictionary return only if log==True in parameters + log : dict + log dictionary return only if log==True in parameters Examples -------- - + >>> a=[.5,.5] >>> b=[.5,.5] >>> M=[[0.,1.],[1.,0.]] >>> ot.sinkhorn(a,b,M,1) array([[ 0.36552929, 0.13447071], [ 0.13447071, 0.36552929]]) - - + + References ---------- - + .. [2] M. Cuturi, Sinkhorn Distances : Lightspeed Computation of Optimal Transport, Advances in Neural Information Processing Systems (NIPS) 26, 2013 - - + + See Also -------- ot.lp.emd : Unregularized OT ot.optim.cg : General regularized OT - - """ - + + """ + a=np.asarray(a,dtype=np.float64) b=np.asarray(b,dtype=np.float64) M=np.asarray(M,dtype=np.float64) - + if len(a)==0: a=np.ones((M.shape[0],),dtype=np.float64)/M.shape[0] if len(b)==0: - b=np.ones((M.shape[1],),dtype=np.float64)/M.shape[1] - + b=np.ones((M.shape[1],),dtype=np.float64)/M.shape[1] + # init data Nini = len(a) Nfin = len(b) - - + + cpt = 0 if log: log={'err':[]} - + # we assume that no distances are null except those of the diagonal of distances u = np.ones(Nini)/Nini - v = np.ones(Nfin)/Nfin + v = np.ones(Nfin)/Nfin uprev=np.zeros(Nini) vprev=np.zeros(Nini) #print reg - + K = np.exp(-M/reg) #print np.min(K) - + Kp = np.dot(np.diag(1/a),K) transp = K cpt = 0 @@ -120,10 +120,10 @@ def sinkhorn(a,b, M, reg, numItermax = 1000, stopThr=1e-9, verbose=False, log=Fa print('Warning: numerical errrors') if cpt!=0: u = uprev - v = vprev + v = vprev break uprev = u - vprev = v + vprev = v v = np.divide(b,np.dot(K.T,u)) u = 1./np.dot(Kp,v) if cpt%10==0: @@ -131,14 +131,14 @@ def sinkhorn(a,b, M, reg, numItermax = 1000, stopThr=1e-9, verbose=False, log=Fa transp = np.dot(np.diag(u),np.dot(K,np.diag(v))) err = np.linalg.norm((np.sum(transp,axis=0)-b))**2 if log: - log['err'].append(err) - + log['err'].append(err) + if verbose: if cpt%200 ==0: print('{:5s}|{:12s}'.format('It.','Err')+'\n'+'-'*19) print('{:5d}|{:8e}|'.format(cpt,err)) cpt = cpt +1 - #print 'err=',err,' cpt=',cpt + #print 'err=',err,' cpt=',cpt if log: return np.dot(np.diag(u),np.dot(K,np.diag(v))),log else: @@ -147,12 +147,12 @@ def sinkhorn(a,b, M, reg, numItermax = 1000, stopThr=1e-9, verbose=False, log=Fa def geometricBar(weights,alldistribT): """return the weighted geometric mean of distributions""" - assert(len(weights)==alldistribT.shape[1]) - return np.exp(np.dot(np.log(alldistribT),weights.T)) + assert(len(weights)==alldistribT.shape[1]) + return np.exp(np.dot(np.log(alldistribT),weights.T)) -def geometricMean(alldistribT): +def geometricMean(alldistribT): """return the geometric mean of distributions""" - return np.exp(np.mean(np.log(alldistribT),axis=1)) + return np.exp(np.mean(np.log(alldistribT),axis=1)) def projR(gamma,p): #return np.dot(np.diag(p/np.maximum(np.sum(gamma,axis=1),1e-10)),gamma) @@ -161,65 +161,65 @@ def projR(gamma,p): def projC(gamma,q): #return (np.dot(np.diag(q/np.maximum(np.sum(gamma,axis=0),1e-10)),gamma.T)).T return np.multiply(gamma,q/np.maximum(np.sum(gamma,axis=0),1e-10)) - + def barycenter(A,M,reg, weights=None, numItermax = 1000, stopThr=1e-4,verbose=False,log=False): """Compute the entropic regularized wasserstein barycenter of distributions A The function solves the following optimization problem: - + .. math:: \mathbf{a} = arg\min_\mathbf{a} \sum_i W_{reg}(\mathbf{a},\mathbf{a}_i) - + where : - + - :math:`W_{reg}(\cdot,\cdot)` is the entropic regularized Wasserstein distance (see ot.bregman.sinkhorn) - :math:`\mathbf{a}_i` are training distributions in the columns of matrix :math:`\mathbf{A}` - reg and :math:`\mathbf{M}` are respectively the regularization term and the cost matrix for OT - + The algorithm used for solving the problem is the Sinkhorn-Knopp matrix scaling algorithm as proposed in [3]_ - + Parameters ---------- A : np.ndarray (d,n) - n training distributions of size d + n training distributions of size d M : np.ndarray (d,d) - loss matrix for OT - reg: float + loss matrix for OT + reg : float Regularization term >0 - numItermax: int, optional + numItermax : int, optional Max number of iterations - stopThr: float, optional + stopThr : float, optional Stop threshol on error (>0) verbose : bool, optional Print information along iterations log : bool, optional - record log if True - - + record log if True + + Returns ------- - a: (d,) ndarray + a : (d,) ndarray Wasserstein barycenter - log: dict - log dictionary return only if log==True in parameters + log : dict + log dictionary return only if log==True in parameters + - References ---------- - + .. [3] Benamou, J. D., Carlier, G., Cuturi, M., Nenna, L., & Peyré, G. (2015). Iterative Bregman projections for regularized transportation problems. SIAM Journal on Scientific Computing, 37(2), A1111-A1138. - - - + + + """ - - + + if weights is None: weights=np.ones(A.shape[1])/A.shape[1] else: assert(len(weights)==A.shape[1]) - + if log: log={'err':[]} @@ -231,130 +231,130 @@ def barycenter(A,M,reg, weights=None, numItermax = 1000, stopThr=1e-4,verbose=Fa UKv=np.dot(K,np.divide(A.T,np.sum(K,axis=0)).T) u = (geometricMean(UKv)/UKv.T).T - + while (err>stopThr and cpt0 (Wasserstein data fitting) - reg0: float - Regularization term >0 (Wasserstein reg with h0) - alpha: float + reg0 : float + Regularization term >0 (Wasserstein reg with h0) + alpha : float How much should we trust the prior ([0,1]) - numItermax: int, optional + numItermax : int, optional Max number of iterations - stopThr: float, optional + stopThr : float, optional Stop threshol on error (>0) verbose : bool, optional Print information along iterations log : bool, optional - record log if True - - + record log if True + + Returns ------- - a: (d,) ndarray + a : (d,) ndarray Wasserstein barycenter - log: dict - log dictionary return only if log==True in parameters - + log : dict + log dictionary return only if log==True in parameters + References ---------- - + .. [4] S. Nakhostin, N. Courty, R. Flamary, D. Tuia, T. Corpetti, Supervised planetary unmixing with optimal transport, Whorkshop on Hyperspectral Image and Signal Processing : Evolution in Remote Sensing (WHISPERS), 2016. """ - - #M = M/np.median(M) + + #M = M/np.median(M) K = np.exp(-M/reg) - - #M0 = M0/np.median(M0) + + #M0 = M0/np.median(M0) K0 = np.exp(-M0/reg0) old = h0 err=1 - cpt=0 + cpt=0 #log = {'niter':0, 'all_err':[]} if log: log={'err':[]} - - + + while (err>stopThr and cpt0 - eta: float, optional + eta : float, optional Regularization term for group lasso regularization >0 - numItermax: int, optional + numItermax : int, optional Max number of iterations - numInnerItermax: int, optional + numInnerItermax : int, optional Max number of iterations (inner sinkhorn solver) - stopInnerThr: float, optional + stopInnerThr : float, optional Stop threshold on error (inner sinkhorn solver) (>0) verbose : bool, optional Print information along iterations @@ -67,9 +66,9 @@ def sinkhorn_lpl1_mm(a,labels_a, b, M, reg, eta=0.1,numItermax = 10,numInnerIter Returns ------- - gamma: (ns x nt) ndarray + gamma : (ns x nt) ndarray Optimal transportation matrix for the given parameters - log: dict + log : dict log dictionary return only if log==True in parameters @@ -145,7 +144,10 @@ def joint_OT_mapping_linear(xs,xt,mu=1,eta=0.001,bias=False,verbose=False,verbos The problem consist in solving jointly an optimal transport matrix :math:`\gamma` and a linear mapping that fits the barycentric mapping - :math:`n_s\gamma X_t` + :math:`n_s\gamma X_t`. + + One can also estimate a mapping with constant bias (see supplementary + material of [8]) using the bias optional argument. The algorithm used for solving the problem is the block coordinate descent that alternates between updates of G (using conditionnal gradient) @@ -158,19 +160,19 @@ def joint_OT_mapping_linear(xs,xt,mu=1,eta=0.001,bias=False,verbose=False,verbos samples in the source domain xt : np.ndarray (nt,d) samples in the target domain - mu: float,optional + mu : float,optional Weight for the linear OT loss (>0) - eta: float, optional + eta : float, optional Regularization term for the linear mapping L (>0) - bias: bool,optional + bias : bool,optional Estimate linear mapping with constant bias - numItermax: int, optional + numItermax : int, optional Max number of BCD iterations - stopThr: float, optional + stopThr : float, optional Stop threshold on relative loss decrease (>0) - numInnerItermax: int, optional + numInnerItermax : int, optional Max number of iterations (inner CG solver) - stopInnerThr: float, optional + stopInnerThr : float, optional Stop threshold on error (inner CG solver) (>0) verbose : bool, optional Print information along iterations @@ -180,11 +182,11 @@ def joint_OT_mapping_linear(xs,xt,mu=1,eta=0.001,bias=False,verbose=False,verbos Returns ------- - gamma: (ns x nt) ndarray + gamma : (ns x nt) ndarray Optimal transportation matrix for the given parameters - L: (d x d) ndarray + L : (d x d) ndarray Linear mapping matrix (d+1 x d if bias) - log: dict + log : dict log dictionary return only if log==True in parameters @@ -291,10 +293,89 @@ def joint_OT_mapping_linear(xs,xt,mu=1,eta=0.001,bias=False,verbose=False,verbos def joint_OT_mapping_kernel(xs,xt,mu=1,eta=0.001,kerneltype='gaussian',sigma=1,bias=False,verbose=False,verbose2=False,numItermax = 100,numInnerItermax = 10,stopInnerThr=1e-6,stopThr=1e-5,log=False,**kwargs): - """Joint Ot and mapping estimation (uniform weights and ) + """Joint OT and nonlinear mapping estimation with kernels as proposed in [8] + + The function solves the following optimization problem: + + .. math:: + \min_{\gamma,L\in\mathcal{H}}\quad \|L(X_s) -n_s\gamma X_t\|^2_F + \mu<\gamma,M>_F + \eta \|L\|^2_\mathcal{H} + + s.t. \gamma 1 = a + + \gamma^T 1= b + + \gamma\geq 0 + where : + + - M is the (ns,nt) squared euclidean cost matrix between samples in Xs and Xt (scaled by ns) + - :math:`L` is a ns x d linear operator on a kernel matrix that approximates the barycentric mapping + - a and b are uniform source and target weights + + The problem consist in solving jointly an optimal transport matrix + :math:`\gamma` and the nonlinear mapping that fits the barycentric mapping + :math:`n_s\gamma X_t`. + + One can also estimate a mapping with constant bias (see supplementary + material of [8]) using the bias optional argument. + + The algorithm used for solving the problem is the block coordinate + descent that alternates between updates of G (using conditionnal gradient) + abd the update of L using a classical kernel least square solver. + + + Parameters + ---------- + xs : np.ndarray (ns,d) + samples in the source domain + xt : np.ndarray (nt,d) + samples in the target domain + mu : float,optional + Weight for the linear OT loss (>0) + eta : float, optional + Regularization term for the linear mapping L (>0) + bias : bool,optional + Estimate linear mapping with constant bias + kerneltype : str,optional + kernel used by calling function ot.utils.kernel (gaussian by default) + sigma : float, optional + Gaussian kernel bandwidth. + numItermax : int, optional + Max number of BCD iterations + stopThr : float, optional + Stop threshold on relative loss decrease (>0) + numInnerItermax : int, optional + Max number of iterations (inner CG solver) + stopInnerThr : float, optional + Stop threshold on error (inner CG solver) (>0) + verbose : bool, optional + Print information along iterations + log : bool, optional + record log if True + + + Returns + ------- + gamma : (ns x nt) ndarray + Optimal transportation matrix for the given parameters + L : (ns x d) ndarray + Nonlinear mapping matrix (ns+1 x d if bias) + log : dict + log dictionary return only if log==True in parameters + + + References + ---------- + + .. [8] M. Perrot, N. Courty, R. Flamary, A. Habrard, "Mapping estimation for discrete optimal transport", Neural Information Processing Systems (NIPS), 2016. + + See Also + -------- + ot.lp.emd : Unregularized OT + ot.optim.cg : General regularized OT + """ - ns,nt,d=xs.shape[0],xt.shape[0],xt.shape[1] + ns,nt=xs.shape[0],xt.shape[0] K=kernel(xs,xs,method=kerneltype,sigma=sigma) if bias: diff --git a/ot/optim.py b/ot/optim.py index 2b8f565..7ed658c 100644 --- a/ot/optim.py +++ b/ot/optim.py @@ -27,7 +27,7 @@ def line_search_armijo(f,xk,pk,gfk,old_fval,args=(),c1=1e-4,alpha0=0.99): descent direction gfk : np.ndarray gradient of f at xk - old_fval: float + old_fval : float loss value at xk args : tuple, optional arguments given to f @@ -110,9 +110,9 @@ def cg(a,b,M,reg,f,df,G0=None,numItermax = 200,stopThr=1e-9,verbose=False,log=Fa Returns ------- - gamma: (ns x nt) ndarray + gamma : (ns x nt) ndarray Optimal transportation matrix for the given parameters - log: dict + log : dict log dictionary return only if log==True in parameters -- cgit v1.2.3 From 8b41e141e8ce4ad14a458cb363f46d3176644116 Mon Sep 17 00:00:00 2001 From: Rémi Flamary Date: Mon, 7 Nov 2016 16:51:42 +0100 Subject: add log and epsilon scaling stabilizations --- README.md | 6 +- examples/demo_OT_1D.py | 4 +- examples/demo_optim_OTreg.py | 10 +- ot/bregman.py | 349 +++++++++++++++++++++++++++++++++++++++++++ ot/datasets.py | 2 +- 5 files changed, 362 insertions(+), 9 deletions(-) (limited to 'ot/bregman.py') diff --git a/README.md b/README.md index 4088bc2..c6d480b 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ This open source Python library provide several solvers for optimization problem It provides the following solvers: * OT solver for the linear program/ Earth Movers Distance [1]. -* Entropic regularization OT solver with Sinkhorn Knopp Algorithm [2]. +* Entropic regularization OT solver with Sinkhorn Knopp Algorithm [2] and stabilized version [9][10]. * Bregman projections for Wasserstein barycenter [3] and unmixing [4]. * Optimal transport for domain adaptation with group lasso regularization [5] * Conditional gradient [6] and Generalized conditional gradient for regularized OT [7]. @@ -98,3 +98,7 @@ This toolbox benefit a lot from open source research and we would like to thank [7] Rakotomamonjy, A., Flamary, R., & Courty, N. (2015). Generalized conditional gradient: analysis of convergence and applications. arXiv preprint arXiv:1510.06567. [8] M. Perrot, N. Courty, R. Flamary, A. Habrard, "Mapping estimation for discrete optimal transport", Neural Information Processing Systems (NIPS), 2016. + +[9] Schmitzer, B. (2016). Stabilized Sparse Scaling Algorithms for Entropy Regularized Transport Problems. arXiv preprint arXiv:1610.06519. + +[10] Chizat, L., Peyré, G., Schmitzer, B., & Vialard, F. X. (2016). Scaling algorithms for unbalanced transport problems. arXiv preprint arXiv:1607.05816. diff --git a/examples/demo_OT_1D.py b/examples/demo_OT_1D.py index 6eaa2ff..df65a60 100644 --- a/examples/demo_OT_1D.py +++ b/examples/demo_OT_1D.py @@ -19,8 +19,8 @@ n=100 # nb bins x=np.arange(n,dtype=np.float64) # Gaussian distributions -a=gauss(n,m=20,s=20) # m= mean, s= std -b=gauss(n,m=60,s=60) +a=gauss(n,m=20,s=5) # m= mean, s= std +b=gauss(n,m=60,s=10) # loss matrix M=ot.dist(x.reshape((n,1)),x.reshape((n,1))) diff --git a/examples/demo_optim_OTreg.py b/examples/demo_optim_OTreg.py index 5e19be5..0a8c583 100644 --- a/examples/demo_optim_OTreg.py +++ b/examples/demo_optim_OTreg.py @@ -17,8 +17,8 @@ n=100 # nb bins x=np.arange(n,dtype=np.float64) # Gaussian distributions -a=ot.datasets.get_1D_gauss(n,m=20,s=20) # m= mean, s= std -b=ot.datasets.get_1D_gauss(n,m=60,s=60) +a=ot.datasets.get_1D_gauss(n,m=20,s=5) # m= mean, s= std +b=ot.datasets.get_1D_gauss(n,m=60,s=10) # loss matrix M=ot.dist(x.reshape((n,1)),x.reshape((n,1))) @@ -37,7 +37,7 @@ def f(G): return 0.5*np.sum(G**2) def df(G): return G reg=1e-1 - + Gl2=ot.optim.cg(a,b,M,reg,f,df,verbose=True) pl.figure(3) @@ -47,9 +47,9 @@ ot.plot.plot1D_mat(a,b,Gl2,'OT matrix Frob. reg') def f(G): return np.sum(G*np.log(G)) def df(G): return np.log(G)+1 - + reg=1e-3 - + Ge=ot.optim.cg(a,b,M,reg,f,df,verbose=True) pl.figure(4) diff --git a/ot/bregman.py b/ot/bregman.py index a770c5a..b132225 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -144,6 +144,355 @@ def sinkhorn(a,b, M, reg, numItermax = 1000, stopThr=1e-9, verbose=False, log=Fa else: return np.dot(np.diag(u),np.dot(K,np.diag(v))) +def sinkhorn_stabilized(a,b, M, reg, numItermax = 1000,tau=1e3, stopThr=1e-9,warmstart=None, verbose=False,print_period=20, log=False): + """ + Solve the entropic regularization OT problem with log stabilization + + The function solves the following optimization problem: + + .. math:: + \gamma = arg\min_\gamma <\gamma,M>_F + reg\cdot\Omega(\gamma) + + s.t. \gamma 1 = a + + \gamma^T 1= b + + \gamma\geq 0 + where : + + - M is the (ns,nt) metric cost matrix + - :math:`\Omega` is the entropic regularization term :math:`\Omega(\gamma)=\sum_{i,j} \gamma_{i,j}\log(\gamma_{i,j})` + - a and b are source and target weights (sum to 1) + + The algorithm used for solving the problem is the Sinkhorn-Knopp matrix + scaling algorithm as proposed in [2]_ but with the log stabilization + proposed in [10]_ an defined in [9]_ (Algo 3.1) . + + + Parameters + ---------- + a : np.ndarray (ns,) + samples weights in the source domain + b : np.ndarray (nt,) + samples in the target domain + M : np.ndarray (ns,nt) + loss matrix + reg : float + Regularization term >0 + tau : float + thershold for max value in u or v for log scaling + warmstart : tible of vectors + if given then sarting values for alpha an beta log scalings + numItermax : int, optional + Max number of iterations + stopThr : float, optional + Stop threshol on error (>0) + verbose : bool, optional + Print information along iterations + log : bool, optional + record log if True + + + Returns + ------- + gamma : (ns x nt) ndarray + Optimal transportation matrix for the given parameters + log : dict + log dictionary return only if log==True in parameters + + Examples + -------- + + >>> a=[.5,.5] + >>> b=[.5,.5] + >>> M=[[0.,1.],[1.,0.]] + >>> ot.sinkhorn(a,b,M,1) + array([[ 0.36552929, 0.13447071], + [ 0.13447071, 0.36552929]]) + + + References + ---------- + + .. [2] M. Cuturi, Sinkhorn Distances : Lightspeed Computation of Optimal Transport, Advances in Neural Information Processing Systems (NIPS) 26, 2013 + + .. [9] Schmitzer, B. (2016). Stabilized Sparse Scaling Algorithms for Entropy Regularized Transport Problems. arXiv preprint arXiv:1610.06519. + + .. [10] Chizat, L., Peyré, G., Schmitzer, B., & Vialard, F. X. (2016). Scaling algorithms for unbalanced transport problems. arXiv preprint arXiv:1607.05816. + + + See Also + -------- + ot.lp.emd : Unregularized OT + ot.optim.cg : General regularized OT + + """ + + a=np.asarray(a,dtype=np.float64) + b=np.asarray(b,dtype=np.float64) + M=np.asarray(M,dtype=np.float64) + + if len(a)==0: + a=np.ones((M.shape[0],),dtype=np.float64)/M.shape[0] + if len(b)==0: + b=np.ones((M.shape[1],),dtype=np.float64)/M.shape[1] + + # init data + na = len(a) + nb = len(b) + + + cpt = 0 + if log: + log={'err':[]} + + # we assume that no distances are null except those of the diagonal of distances + if warmstart is None: + alpha,beta=np.zeros(na),np.zeros(nb) + else: + alpha,beta=warmstart + u,v = np.ones(na)/na,np.ones(nb)/nb + uprev,vprev=np.zeros(na),np.zeros(nb) + + + #print reg + + + def get_K(alpha,beta): + """log space computation""" + return np.exp(-(M-alpha.reshape((na,1))-beta.reshape((1,nb)))/reg) + + def get_Gamma(alpha,beta,u,v): + """log space gamma computation""" + return np.exp(-(M-alpha.reshape((na,1))-beta.reshape((1,nb)))/reg+np.log(u.reshape((na,1)))+np.log(v.reshape((1,nb)))) + + #print np.min(K) + + K=get_K(alpha,beta) + transp = K + loop=1 + cpt = 0 + err=1 + while loop: + + if np.abs(u).max()>tau or np.abs(v).max()>tau: + alpha,beta=alpha+reg*np.log(u),beta+reg*np.log(v) + u,v = np.ones(na)/na,np.ones(nb)/nb + K=get_K(alpha,beta) + + uprev = u + vprev = v + v = b/np.dot(K.T,u) + u = a/np.dot(K,v) + + + + if cpt%print_period==0: + # we can speed up the process by checking for the error only all the 10th iterations + transp = get_Gamma(alpha,beta,u,v) + err = np.linalg.norm((np.sum(transp,axis=0)-b))**2 + if log: + log['err'].append(err) + + if verbose: + if cpt%(print_period*20) ==0: + print('{:5s}|{:12s}'.format('It.','Err')+'\n'+'-'*19) + print('{:5d}|{:8e}|'.format(cpt,err)) + + + if err<=stopThr: + loop=False + + if cpt>=numItermax: + loop=False + + + if np.any(np.dot(K.T,u)==0) or np.any(np.isnan(u)) or np.any(np.isnan(v)): + # we have reached the machine precision + # come back to previous solution and quit loop + print('Warning: numerical errrors') + if cpt!=0: + u = uprev + v = vprev + break + + cpt = cpt +1 + #print 'err=',err,' cpt=',cpt + if log: + log['logu']=alpha/reg+np.log(u) + log['logv']=beta/reg+np.log(v) + log['alpha']=alpha+reg*np.log(u) + log['beta']=beta+reg*np.log(v) + log['warmstart']=(log['alpha'],log['beta']) + return get_Gamma(alpha,beta,u,v),log + else: + return get_Gamma(alpha,beta,u,v) + +def sinkhorn_epsilon_scaling(a,b, M, reg, numItermax = 100, epsilon0=1e4, numInnerItermax = 100,tau=1e3, stopThr=1e-9,warmstart=None, verbose=False,print_period=10, log=False): + """ + Solve the entropic regularization optimal transport problem with log + stabilization and epsilon scaling. + + The function solves the following optimization problem: + + .. math:: + \gamma = arg\min_\gamma <\gamma,M>_F + reg\cdot\Omega(\gamma) + + s.t. \gamma 1 = a + + \gamma^T 1= b + + \gamma\geq 0 + where : + + - M is the (ns,nt) metric cost matrix + - :math:`\Omega` is the entropic regularization term :math:`\Omega(\gamma)=\sum_{i,j} \gamma_{i,j}\log(\gamma_{i,j})` + - a and b are source and target weights (sum to 1) + + The algorithm used for solving the problem is the Sinkhorn-Knopp matrix + scaling algorithm as proposed in [2]_ but with the log stabilization + proposed in [10]_ and the log scaling proposed in [9]_ algorithm 3.2 + + + Parameters + ---------- + a : np.ndarray (ns,) + samples weights in the source domain + b : np.ndarray (nt,) + samples in the target domain + M : np.ndarray (ns,nt) + loss matrix + reg : float + Regularization term >0 + tau : float + thershold for max value in u or v for log scaling + tau : float + thershold for max value in u or v for log scaling + warmstart : tible of vectors + if given then sarting values for alpha an beta log scalings + numItermax : int, optional + Max number of iterations + numInnerItermax : int, optional + Max number of iterationsin the inner slog stabilized sinkhorn + epsilon0 : int, optional + first epsilon regularization value (then exponential decrease to reg) + stopThr : float, optional + Stop threshol on error (>0) + verbose : bool, optional + Print information along iterations + log : bool, optional + record log if True + + + Returns + ------- + gamma : (ns x nt) ndarray + Optimal transportation matrix for the given parameters + log : dict + log dictionary return only if log==True in parameters + + Examples + -------- + + >>> a=[.5,.5] + >>> b=[.5,.5] + >>> M=[[0.,1.],[1.,0.]] + >>> ot.sinkhorn(a,b,M,1) + array([[ 0.36552929, 0.13447071], + [ 0.13447071, 0.36552929]]) + + + References + ---------- + + .. [2] M. Cuturi, Sinkhorn Distances : Lightspeed Computation of Optimal Transport, Advances in Neural Information Processing Systems (NIPS) 26, 2013 + + .. [9] Schmitzer, B. (2016). Stabilized Sparse Scaling Algorithms for Entropy Regularized Transport Problems. arXiv preprint arXiv:1610.06519. + + See Also + -------- + ot.lp.emd : Unregularized OT + ot.optim.cg : General regularized OT + + """ + + a=np.asarray(a,dtype=np.float64) + b=np.asarray(b,dtype=np.float64) + M=np.asarray(M,dtype=np.float64) + + if len(a)==0: + a=np.ones((M.shape[0],),dtype=np.float64)/M.shape[0] + if len(b)==0: + b=np.ones((M.shape[1],),dtype=np.float64)/M.shape[1] + + # init data + na = len(a) + nb = len(b) + + # nrelative umerical precision with 64 bits + numItermin = 35 + numItermax=max(numItermin,numItermax) # ensure that last velue is exact + + + cpt = 0 + if log: + log={'err':[]} + + # we assume that no distances are null except those of the diagonal of distances + if warmstart is None: + alpha,beta=np.zeros(na),np.zeros(nb) + else: + alpha,beta=warmstart + + + def get_K(alpha,beta): + """log space computation""" + return np.exp(-(M-alpha.reshape((na,1))-beta.reshape((1,nb)))/reg) + + #print np.min(K) + def get_reg(n): # exponential decreasing + return (epsilon0-reg)*np.exp(-n)+reg + + loop=1 + cpt = 0 + err=1 + while loop: + + regi=get_reg(cpt) + + G,logi=sinkhorn_stabilized(a,b, M, regi, numItermax = numInnerItermax,tau=1e3, stopThr=1e-9,warmstart=(alpha,beta), verbose=False,print_period=20,tau=tau, log=True) + + alpha=logi['alpha'] + beta=logi['beta'] + + if cpt>=numItermax: + loop=False + + if cpt%(print_period)==0: # spsion nearly converged + # we can speed up the process by checking for the error only all the 10th iterations + transp = G + err = np.linalg.norm((np.sum(transp,axis=0)-b))**2+np.linalg.norm((np.sum(transp,axis=1)-a))**2 + if log: + log['err'].append(err) + + if verbose: + if cpt%(print_period*10) ==0: + print('{:5s}|{:12s}'.format('It.','Err')+'\n'+'-'*19) + print('{:5d}|{:8e}|'.format(cpt,err)) + + if err<=stopThr and cpt>numItermin: + loop=False + + cpt = cpt +1 + #print 'err=',err,' cpt=',cpt + if log: + log['alpha']=alpha + log['beta']=beta + log['warmstart']=(log['alpha'],log['beta']) + return G,log + else: + return G + def geometricBar(weights,alldistribT): """return the weighted geometric mean of distributions""" diff --git a/ot/datasets.py b/ot/datasets.py index c750812..8605691 100644 --- a/ot/datasets.py +++ b/ot/datasets.py @@ -28,7 +28,7 @@ def get_1D_gauss(n,m,s): """ x=np.arange(n,dtype=np.float64) - h=np.exp(-(x-m)**2/(2*s^2)) + h=np.exp(-(x-m)**2/(2*s**2)) return h/h.sum() -- cgit v1.2.3 From 8b079cb1735e063709d4421241d23d11e32b55cb Mon Sep 17 00:00:00 2001 From: Rémi Flamary Date: Mon, 7 Nov 2016 17:01:20 +0100 Subject: v0.1.11 --- ot/bregman.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'ot/bregman.py') diff --git a/ot/bregman.py b/ot/bregman.py index b132225..91c526a 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -460,7 +460,7 @@ def sinkhorn_epsilon_scaling(a,b, M, reg, numItermax = 100, epsilon0=1e4, numInn regi=get_reg(cpt) - G,logi=sinkhorn_stabilized(a,b, M, regi, numItermax = numInnerItermax,tau=1e3, stopThr=1e-9,warmstart=(alpha,beta), verbose=False,print_period=20,tau=tau, log=True) + G,logi=sinkhorn_stabilized(a,b, M, regi, numItermax = numInnerItermax, stopThr=1e-9,warmstart=(alpha,beta), verbose=False,print_period=20,tau=tau, log=True) alpha=logi['alpha'] beta=logi['beta'] -- cgit v1.2.3 From 2ccba4bf4bd0a13f82ffeb34c05e97c80466a030 Mon Sep 17 00:00:00 2001 From: Rémi Flamary Date: Tue, 8 Nov 2016 09:19:30 +0100 Subject: travis CI --- Makefile | 3 +++ ot/__init__.py | 1 + ot/bregman.py | 7 +++++-- ot/lp/__init__.py | 2 +- requirements.txt | 5 +++++ test/test_load_module.py | 10 ++++++++++ travis.yml | 16 ++++++++++++++++ 7 files changed, 41 insertions(+), 3 deletions(-) create mode 100644 requirements.txt create mode 100644 test/test_load_module.py create mode 100644 travis.yml (limited to 'ot/bregman.py') diff --git a/Makefile b/Makefile index e3808df..34e9f20 100644 --- a/Makefile +++ b/Makefile @@ -33,6 +33,9 @@ sremove : clean : $(PYTHON) setup.py clean + +test: + pytest uploadpypi: #python setup.py register diff --git a/ot/__init__.py b/ot/__init__.py index 08b57eb..a925a6f 100644 --- a/ot/__init__.py +++ b/ot/__init__.py @@ -1,5 +1,6 @@ """Python Optimal Transport toolbox""" + # All submodules and packages from . import lp from . import bregman diff --git a/ot/bregman.py b/ot/bregman.py index 91c526a..568ead2 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -59,6 +59,7 @@ def sinkhorn(a,b, M, reg, numItermax = 1000, stopThr=1e-9, verbose=False, log=Fa Examples -------- + >>> import ot >>> a=[.5,.5] >>> b=[.5,.5] >>> M=[[0.,1.],[1.,0.]] @@ -203,10 +204,11 @@ def sinkhorn_stabilized(a,b, M, reg, numItermax = 1000,tau=1e3, stopThr=1e-9,war Examples -------- + >>> import ot >>> a=[.5,.5] >>> b=[.5,.5] >>> M=[[0.,1.],[1.,0.]] - >>> ot.sinkhorn(a,b,M,1) + >>> ot.bregman.sinkhorn_stabilized(a,b,M,1) array([[ 0.36552929, 0.13447071], [ 0.13447071, 0.36552929]]) @@ -394,10 +396,11 @@ def sinkhorn_epsilon_scaling(a,b, M, reg, numItermax = 100, epsilon0=1e4, numInn Examples -------- + >>> import ot >>> a=[.5,.5] >>> b=[.5,.5] >>> M=[[0.,1.],[1.,0.]] - >>> ot.sinkhorn(a,b,M,1) + >>> ot.bregman.sinkhorn_epsilon_scaling(a,b,M,1) array([[ 0.36552929, 0.13447071], [ 0.13447071, 0.36552929]]) diff --git a/ot/lp/__init__.py b/ot/lp/__init__.py index 2adf937..1e55f5a 100644 --- a/ot/lp/__init__.py +++ b/ot/lp/__init__.py @@ -45,7 +45,7 @@ def emd(a, b, M): Simple example with obvious solution. The function emd accepts lists and perform automatic conversion to numpy arrays - + >>> import ot >>> a=[.5,.5] >>> b=[.5,.5] >>> M=[[0.,1.],[1.,0.]] diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..cd73c15 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +POT +numpy +scipy +cython +matplotlib diff --git a/test/test_load_module.py b/test/test_load_module.py new file mode 100644 index 0000000..a04c5df --- /dev/null +++ b/test/test_load_module.py @@ -0,0 +1,10 @@ + + +import ot +import doctest + +# test lp solver +doctest.testmod(ot.lp,verbose=True) + +# test bregman solver +doctest.testmod(ot.bregman,verbose=True) diff --git a/travis.yml b/travis.yml new file mode 100644 index 0000000..c73bd8f --- /dev/null +++ b/travis.yml @@ -0,0 +1,16 @@ +language: python +python: + - "2.6" + - "2.7" + - "3.2" + - "3.3" + - "3.4" + # does not have headers provided, please ask https://launchpad.net/~pypy/+archive/ppa + # maintainers to fix their pypy-dev package. + - "pypy" +# command to install dependencies +install: + - pip install . + - pip install -r requirements.txt +# command to run tests +script: pytest -- cgit v1.2.3 From bb6020dd69933735a6cf821eabcbedca92f2ecb4 Mon Sep 17 00:00:00 2001 From: Rémi Flamary Date: Thu, 5 Jan 2017 13:40:38 +0100 Subject: more efficient sinkhorn --- ot/bregman.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'ot/bregman.py') diff --git a/ot/bregman.py b/ot/bregman.py index 568ead2..fd22259 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -141,9 +141,9 @@ def sinkhorn(a,b, M, reg, numItermax = 1000, stopThr=1e-9, verbose=False, log=Fa cpt = cpt +1 #print 'err=',err,' cpt=',cpt if log: - return np.dot(np.diag(u),np.dot(K,np.diag(v))),log + return u.reshape((-1,1))*K*v.reshape((1,-1)),log else: - return np.dot(np.diag(u),np.dot(K,np.diag(v))) + return u.reshape((-1,1))*K*v.reshape((1,-1)) def sinkhorn_stabilized(a,b, M, reg, numItermax = 1000,tau=1e3, stopThr=1e-9,warmstart=None, verbose=False,print_period=20, log=False): """ -- cgit v1.2.3 From 0b806374d33ae83d39846096a1838b096c0c0b8e Mon Sep 17 00:00:00 2001 From: Rémi Flamary Date: Tue, 7 Feb 2017 17:17:34 +0100 Subject: log return for sinkhorn --- ot/bregman.py | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'ot/bregman.py') diff --git a/ot/bregman.py b/ot/bregman.py index fd22259..27f8ff5 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -139,6 +139,10 @@ def sinkhorn(a,b, M, reg, numItermax = 1000, stopThr=1e-9, verbose=False, log=Fa print('{:5s}|{:12s}'.format('It.','Err')+'\n'+'-'*19) print('{:5d}|{:8e}|'.format(cpt,err)) cpt = cpt +1 + if log: + log['u']=u + log['v']=v + #print 'err=',err,' cpt=',cpt if log: return u.reshape((-1,1))*K*v.reshape((1,-1)),log -- cgit v1.2.3 From 7bcae0fa807d67b36c44d5e0abeb45df8c65c3c6 Mon Sep 17 00:00:00 2001 From: Leo gautheron Date: Thu, 13 Apr 2017 10:53:29 +0200 Subject: update bregman file - change commented prints to python3 compatible syntax - Correct issue that could cause the sinkhorn algo to stop with u and v containing nan/infinite numbers: - Assign uprev and vprev before changing u and v. - Then update u and v. - Then check if u and v contain nan, but ALSO infinite values. - if there are issues, then display error (with 2 r, not 3 :p) along with the iteration number (there may have errors at iteration 0) --- ot/bregman.py | 50 ++++++++++++++++++++++++-------------------------- 1 file changed, 24 insertions(+), 26 deletions(-) (limited to 'ot/bregman.py') diff --git a/ot/bregman.py b/ot/bregman.py index 27f8ff5..b06eaeb 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -102,31 +102,30 @@ def sinkhorn(a,b, M, reg, numItermax = 1000, stopThr=1e-9, verbose=False, log=Fa # we assume that no distances are null except those of the diagonal of distances u = np.ones(Nini)/Nini v = np.ones(Nfin)/Nfin - uprev=np.zeros(Nini) - vprev=np.zeros(Nini) - #print reg + #print(reg) K = np.exp(-M/reg) - #print np.min(K) + #print(np.min(K)) Kp = np.dot(np.diag(1/a),K) transp = K cpt = 0 err=1 while (err>stopThr and cpt Date: Tue, 18 Apr 2017 15:05:39 +0200 Subject: Performance improvement sinkhorn Doing the computation this way is equivalent and allows to reduce the space complexity required from O(max(a, b)^2) to O(a*b) (especially usefull to transport a small number of sources example to a lot of target) This also allows to decrease the computation time. --- ot/bregman.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'ot/bregman.py') diff --git a/ot/bregman.py b/ot/bregman.py index b06eaeb..1deccce 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -108,7 +108,8 @@ def sinkhorn(a,b, M, reg, numItermax = 1000, stopThr=1e-9, verbose=False, log=Fa K = np.exp(-M/reg) #print(np.min(K)) - Kp = np.dot(np.diag(1/a),K) + Kp = (1/a).reshape(-1, 1) * K + transp = K cpt = 0 err=1 @@ -128,7 +129,7 @@ def sinkhorn(a,b, M, reg, numItermax = 1000, stopThr=1e-9, verbose=False, log=Fa break if cpt%10==0: # we can speed up the process by checking for the error only all the 10th iterations - transp = np.dot(np.diag(u),np.dot(K,np.diag(v))) + transp = u.reshape(-1, 1) * (K * v) err = np.linalg.norm((np.sum(transp,axis=0)-b))**2 if log: log['err'].append(err) -- cgit v1.2.3 From 691c97a033e54359e8c933e3bdd34bf5cf40151d Mon Sep 17 00:00:00 2001 From: Leo gautheron Date: Tue, 18 Apr 2017 16:04:03 +0200 Subject: little cleanup sinkhorn --- ot/bregman.py | 4 ---- 1 file changed, 4 deletions(-) (limited to 'ot/bregman.py') diff --git a/ot/bregman.py b/ot/bregman.py index 1deccce..0453f14 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -94,8 +94,6 @@ def sinkhorn(a,b, M, reg, numItermax = 1000, stopThr=1e-9, verbose=False, log=Fa Nini = len(a) Nfin = len(b) - - cpt = 0 if log: log={'err':[]} @@ -109,8 +107,6 @@ def sinkhorn(a,b, M, reg, numItermax = 1000, stopThr=1e-9, verbose=False, log=Fa #print(np.min(K)) Kp = (1/a).reshape(-1, 1) * K - - transp = K cpt = 0 err=1 while (err>stopThr and cpt Date: Thu, 20 Apr 2017 12:12:15 +0200 Subject: sinkhorn GPU implementation --- ot/__init__.py | 6 +- ot/bregman.py | 6 +- ot/da.py | 23 +- ot/gpu/__init__.py | 6 + ot/gpu/bregman.py | 83 + ot/gpu/cudamat/.gitignore | 38 + ot/gpu/cudamat/CHANGELOG | 41 + ot/gpu/cudamat/CONTRIBUTE.md | 74 + ot/gpu/cudamat/INSTALL.md | 53 + ot/gpu/cudamat/LICENSE | 21 + ot/gpu/cudamat/README.md | 65 + ot/gpu/cudamat/cudamat/__init__.py | 2 + ot/gpu/cudamat/cudamat/cudamat.cu | 1633 +++ ot/gpu/cudamat/cudamat/cudamat.cuh | 35 + ot/gpu/cudamat/cudamat/cudamat.py | 1575 ++ ot/gpu/cudamat/cudamat/cudamat_kernels.cu | 804 ++ ot/gpu/cudamat/cudamat/cudamat_kernels.cuh | 92 + ot/gpu/cudamat/cudamat/learn.cu | 34 + ot/gpu/cudamat/cudamat/learn.py | 21 + ot/gpu/cudamat/cudamat/learn_kernels.cu | 10 + ot/gpu/cudamat/cudamat/learn_kernels.cuh | 12 + ot/gpu/cudamat/cudamat/rnd_multipliers_32bit.txt | 16028 +++++++++++++++++++++ ot/gpu/cudamat/examples/bench_cudamat.py | 97 + ot/gpu/cudamat/examples/nn_cudamat.py | 133 + ot/gpu/cudamat/examples/rbm_cudamat.py | 98 + ot/gpu/cudamat/examples/rbm_numpy.py | 72 + ot/gpu/cudamat/examples/util.py | 22 + ot/gpu/cudamat/setup.py | 121 + ot/gpu/cudamat/test/test_cudamat.py | 1217 ++ ot/gpu/cudamat/test/test_learn.py | 27 + ot/gpu/da.py | 81 + 31 files changed, 22514 insertions(+), 16 deletions(-) create mode 100644 ot/gpu/__init__.py create mode 100644 ot/gpu/bregman.py create mode 100644 ot/gpu/cudamat/.gitignore create mode 100644 ot/gpu/cudamat/CHANGELOG create mode 100644 ot/gpu/cudamat/CONTRIBUTE.md create mode 100644 ot/gpu/cudamat/INSTALL.md create mode 100644 ot/gpu/cudamat/LICENSE create mode 100644 ot/gpu/cudamat/README.md create mode 100644 ot/gpu/cudamat/cudamat/__init__.py create mode 100644 ot/gpu/cudamat/cudamat/cudamat.cu create mode 100644 ot/gpu/cudamat/cudamat/cudamat.cuh create mode 100644 ot/gpu/cudamat/cudamat/cudamat.py create mode 100644 ot/gpu/cudamat/cudamat/cudamat_kernels.cu create mode 100644 ot/gpu/cudamat/cudamat/cudamat_kernels.cuh create mode 100644 ot/gpu/cudamat/cudamat/learn.cu create mode 100644 ot/gpu/cudamat/cudamat/learn.py create mode 100644 ot/gpu/cudamat/cudamat/learn_kernels.cu create mode 100644 ot/gpu/cudamat/cudamat/learn_kernels.cuh create mode 100644 ot/gpu/cudamat/cudamat/rnd_multipliers_32bit.txt create mode 100644 ot/gpu/cudamat/examples/bench_cudamat.py create mode 100644 ot/gpu/cudamat/examples/nn_cudamat.py create mode 100644 ot/gpu/cudamat/examples/rbm_cudamat.py create mode 100644 ot/gpu/cudamat/examples/rbm_numpy.py create mode 100644 ot/gpu/cudamat/examples/util.py create mode 100755 ot/gpu/cudamat/setup.py create mode 100644 ot/gpu/cudamat/test/test_cudamat.py create mode 100644 ot/gpu/cudamat/test/test_learn.py create mode 100644 ot/gpu/da.py (limited to 'ot/bregman.py') diff --git a/ot/__init__.py b/ot/__init__.py index d89b3a3..702269c 100644 --- a/ot/__init__.py +++ b/ot/__init__.py @@ -9,6 +9,7 @@ from . import utils from . import datasets from . import plot from . import da +from . import gpu # OT functions from .lp import emd, emd2 @@ -20,6 +21,7 @@ from .utils import dist, unif, tic, toc, toq __version__ = "0.2" -__all__ = ["emd", "emd2", "sinkhorn", "utils", 'datasets', 'bregman', 'lp', +__all__ = ["emd", "emd2", "sinkhorn", "utils", 'datasets', 'bregman', 'lp', 'plot', 'tic', 'toc', 'toq', - 'dist', 'unif', 'barycenter', 'sinkhorn_lpl1_mm', 'da', 'optim'] + 'dist', 'unif', 'barycenter', 'sinkhorn_lpl1_mm', 'da', 'optim', + 'gpu'] diff --git a/ot/bregman.py b/ot/bregman.py index 0453f14..c46e5dc 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -112,9 +112,11 @@ def sinkhorn(a,b, M, reg, numItermax = 1000, stopThr=1e-9, verbose=False, log=Fa while (err>stopThr and cpt stopThr and cpt < numItermax): + uprev_GPU = u_GPU.copy() + vprev_GPU = v_GPU.copy() + + KtransposeU_GPU = K_GPU.transpose().dot(u_GPU) + b_GPU.divide(KtransposeU_GPU, target=v_GPU) + ones_GPU.divide(Kp_GPU.dot(v_GPU), target=u_GPU) + + if (np.any(KtransposeU_GPU.asarray() == 0) or + not u_GPU.allfinite() or not v_GPU.allfinite()): + # we have reached the machine precision + # come back to previous solution and quit loop + print('Warning: numerical errors at iteration', cpt) + u_GPU = uprev_GPU.copy() + v_GPU = vprev_GPU.copy() + break + if cpt % 10 == 0: + # we can speed up the process by checking for the error only all + # the 10th iterations + K_GPU.mult_by_col(u_GPU, target=tmp_GPU) + tmp_GPU.mult_by_row(v_GPU.transpose(), target=tmp_GPU) + + bcopy_GPU = b_GPU.copy().transpose() + bcopy_GPU.add_sums(tmp_GPU, axis=0, beta=-1) + err = bcopy_GPU.euclid_norm()**2 + if log: + log['err'].append(err) + + if verbose: + if cpt % 200 == 0: + print('{:5s}|{:12s}'.format('It.', 'Err')+'\n'+'-'*19) + print('{:5d}|{:8e}|'.format(cpt, err)) + cpt += 1 + if log: + log['u'] = u_GPU.asarray() + log['v'] = v_GPU.asarray() + + # print('err=',err,' cpt=',cpt) + K_GPU.mult_by_col(u_GPU, target=K_GPU) + K_GPU.mult_by_row(v_GPU.transpose(), target=K_GPU) + if log: + return K_GPU.asarray(), log + else: + return K_GPU.asarray() diff --git a/ot/gpu/cudamat/.gitignore b/ot/gpu/cudamat/.gitignore new file mode 100644 index 0000000..7a10978 --- /dev/null +++ b/ot/gpu/cudamat/.gitignore @@ -0,0 +1,38 @@ +*.py[cod] + +# C extensions +*.so + +# Packages +*.egg +*.egg-info +dist +build +eggs +parts +bin +var +sdist +develop-eggs +.installed.cfg +lib +lib64 + +# Installer logs +pip-log.txt + +# Unit test / coverage reports +.coverage +.tox +nosetests.xml + +# Translations +*.mo + +# Mr Developer +.mr.developer.cfg +.project +.pydevproject + +# Vim +*.swp diff --git a/ot/gpu/cudamat/CHANGELOG b/ot/gpu/cudamat/CHANGELOG new file mode 100644 index 0000000..446df0f --- /dev/null +++ b/ot/gpu/cudamat/CHANGELOG @@ -0,0 +1,41 @@ +Version 0.3 +- Added equality testing. Contributed by Ryan P. Adams. +- Added flag that determines whether threads are synced after each call. +- Added direct blas calls for two elementwise operations. Contributed by Vincent Vanhoucke. +- Added the set_selected_columns. Contributed by Tijmen Tieleman. +- Added tanh, abs, and log_1_plus_exp. Contributed by Ilya Sutskever. +- Some bug fixes. Contributed by Jonathan Taylor. +- Added the select_columns method. Code contributed by Tijmen Tieleman. +- on_device should now work as intended. +- allocate_device_memory now returns an error when cublasAlloc fails. +- Fixed bug in max that showed up when an entire column was negative. +- Fixed bug in activation computations in examples/rbm_numpy.py. +- Added get_col_slice and set_col_slice methods. +- Added init and shutdown methods to shorten cublas_init and cublas_shutdown. +- Added bound checking to the various slicing methods. +- Fixed problem with pow and negative numbers. +- Added support for matrix powers in pow. + +Version 0.2 +- Methods add, subtract, mult, divide can now take scalars as well as instances of CUDAMatrix. +- Deprecated add_scalar, mult_by_scalar, div_by_scalar. +- Methods now return target or self to make chaining operations easier. +- Added asarray method. +- Added transpose method. +- Added sqrt and pow functions. +- Added the sigmoid method to the module level. +- Added add_row_vec. +- Added empty. Now when you don't provide a target or pre-allocated temporary storage cudamat methods will not take up CPU RAM or transfer anything between the CPU and GPU. +- Added get_row_slice and set_row_slice. +- Added less_than_scalar, greater_than, greater_than_scalar. +- Added max (axis=1 is currently not supported.) + +Version 0.1.5 +- Added shape attribute and reshape method. + +Version 0.1 +- Most methods now throw python exceptions instead of exiting after encountering an error. +- The CUDAMatrix constructor now automatically converts ndarray objects to float32 in FORTRAN order. +- Renamed scalar_mult to mult_by_scalar and scalar_div to div_by_scalar. +- Added log and exp functions. +- Removed add_row_sums and sum_rows. diff --git a/ot/gpu/cudamat/CONTRIBUTE.md b/ot/gpu/cudamat/CONTRIBUTE.md new file mode 100644 index 0000000..a98a39f --- /dev/null +++ b/ot/gpu/cudamat/CONTRIBUTE.md @@ -0,0 +1,74 @@ +Development installation +------------------------ + +If you want to develop cudamat, you should clone the github repository instead +of downloading a release. Furthermore, it is useful to install it in editable +mode. Instead of copying the files somewhere Python can find them, this will +point Python directly to the directory you install it from. Either of the +following commands will do: + +```bash +# a) Install for your user in editable mode: +python setup.py develop --prefix=~/.local +# b) Install for your user in editable mode, but with pip: +pip install --user --editable . +``` + +As for the [standard installation](INSTALL.md), you can set the `NVCC_FLAGS` +environment variable to compile for a specific architecture. + +Update after local changes +-------------------------- + +Your changes to `.py` files will show up immediately the next time you import +cudamat. Changes to `.cu` and `.cuh` files require a recompilation triggered +by just running the above installation command again. + +Update after remote changes +--------------------------- + +To obtain the latest version, just pull in the remote changes: + +```bash +git checkout master +git fetch origin +git merge origin/master +``` + +Then recompile as per the instructions in the previous section. + +Contribute back +--------------- + +If you created a great new feature that is useful to the rest of the world, +and it even comes with docstrings and updated tests, we will gladly incorporate +it into cudamat. To do that, you will need to send us a pull request from your +fork. + +If you haven't forked cudamat yet, log in to your github account, go to +https://github.com/cudamat/cudamat and hit the "Fork" button. +Now instead of having `origin` point to `cudamat/cudamat`, you will want to have +it point to your fork, and have `upstream` point to the official project: + +```bash +git remote rename origin upstream +git remote add origin git@github.com:yourusername/cudamat +git fetch origin +``` + +Create a branch to house your changes: + +```bash +git checkout -b my-new-feature +``` + +Hack away, then add your changes, commit and push: + +```bash +git add all.py my.py changes.py +git commit -m 'Added a new feature that does this and that' +git push origin my-new-feature +``` + +Now send us a pull request asking to merge `yourusername:my-new-feature` into +`cudamat:master` and we will come back to you! diff --git a/ot/gpu/cudamat/INSTALL.md b/ot/gpu/cudamat/INSTALL.md new file mode 100644 index 0000000..95cfef3 --- /dev/null +++ b/ot/gpu/cudamat/INSTALL.md @@ -0,0 +1,53 @@ +Prerequisites +------------- + +cudamat needs the following to be installed first: + +* Python 2.x or 3.x and numpy +* The CUDA SDK +* nose for running the tests (optional) + +Installation +------------ + +Once you have installed the prerequisites and downloaded cudamat, switch to the +cudamat directory and run either of the following commands to install it: + +```bash +# a) Install for your user: +python setup.py install --user +# b) Install for your user, but with pip: +pip install --user . +# c) Install system-wide: +sudo python setup.py install +# d) Install system-wide, but with pip: +sudo pip install . +``` + +If your Nvidia GPU supports a higher Compute Capability than the default one of +your CUDA toolkit, you can set the `NVCCFLAGS` environment variable when +installing cudamat to compile it for your architecture. For example, to install +for your user for a GTX 780 Ti (Compute Capability 3.5), you would run: + +```bash +NVCCFLAGS=-arch=sm_35 python setup.py install --user +``` + +To compile for both Compute Capability 2.0 and 3.5, you would run: + +```bash +NVCCFLAGS="-gencode arch=compute_20,code=sm_20 -gencode arch=compute_35,code=sm_35" ... +``` + +Testing +------- + +To test your setup, run the included unit tests and optionally the benchmark: + +```bash +cd test # so it doesn't try importing cudamat from the source directory +# Run tests +nosetests +# Run benchmark +python ../examples/bench_cudamat.py +``` diff --git a/ot/gpu/cudamat/LICENSE b/ot/gpu/cudamat/LICENSE new file mode 100644 index 0000000..43b4816 --- /dev/null +++ b/ot/gpu/cudamat/LICENSE @@ -0,0 +1,21 @@ +Copyright (c) 2009-2013, Volodymyr Mnih +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that +the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this list of conditions and the +following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and +the following disclaimer in the documentation and/or other materials provided with the distribution. + * Neither the name of the nor the names of its contributors may be used to endorse or +promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED +WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR +OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH +DAMAGE. diff --git a/ot/gpu/cudamat/README.md b/ot/gpu/cudamat/README.md new file mode 100644 index 0000000..9d2dd6f --- /dev/null +++ b/ot/gpu/cudamat/README.md @@ -0,0 +1,65 @@ +CUDAMat +======= + +The aim of the cudamat project is to make it easy to perform basic matrix calculations on CUDA-enabled GPUs from Python. cudamat provides a Python matrix class that performs calculations on a GPU. At present, some of the operations our GPU matrix class supports include: + +* Easy conversion to and from instances of `numpy.ndarray`. +* Limited slicing support. +* Matrix multiplication and transpose. +* Elementwise addition, subtraction, multiplication, and division. +* Elementwise application of exp, log, pow, sqrt. +* Summation, maximum and minimum along rows or columns. +* Conversion of CUDA errors into Python exceptions. + +The current feature set of cudamat is biased towards features needed for implementing some common machine learning algorithms. We have included implementations of feedforward neural networks and restricted Boltzmann machines in the examples that come with cudamat. + +Example: + +```python +import numpy as np +import cudamat as cm + +cm.cublas_init() + +# create two random matrices and copy them to the GPU +a = cm.CUDAMatrix(np.random.rand(32, 256)) +b = cm.CUDAMatrix(np.random.rand(256, 32)) + +# perform calculations on the GPU +c = cm.dot(a, b) +d = c.sum(axis = 0) + +# copy d back to the host (CPU) and print +print(d.asarray()) +``` + +Documentation +------------- + +An overview of the main features of cudamat can be found in the technical report: + +[CUDAMat: A CUDA-based matrix class for Python](http://www.cs.toronto.edu/~vmnih/docs/cudamat_tr.pdf), Volodymyr Mnih, UTML TR 2009-004. + +Download +-------- + +You can obtain the latest release from the repository by typing: + +```bash +git clone https://github.com/cudamat/cudamat.git +``` + +You can also download one of the releases from the [releases](https://github.com/cudamat/cudamat/releases) section. + +Installation +------------ + +cudamat uses setuptools and can be installed via pip. +For details, please see [INSTALL.md](INSTALL.md). + +Development +----------- + +If you want to contribute new features or improvements, you're welcome to fork +cudamat on github and send us your pull requests! +Please see [CONTRIBUTE.md](CONTRIBUTE.md) if you need any help with that. diff --git a/ot/gpu/cudamat/cudamat/__init__.py b/ot/gpu/cudamat/cudamat/__init__.py new file mode 100644 index 0000000..13b072d --- /dev/null +++ b/ot/gpu/cudamat/cudamat/__init__.py @@ -0,0 +1,2 @@ +from .cudamat import * +from . import learn diff --git a/ot/gpu/cudamat/cudamat/cudamat.cu b/ot/gpu/cudamat/cudamat/cudamat.cu new file mode 100644 index 0000000..522f9cc --- /dev/null +++ b/ot/gpu/cudamat/cudamat/cudamat.cu @@ -0,0 +1,1633 @@ +#include +#include +#include +#include +#include +#include "cudamat_kernels.cuh" +#include "cudamat.cuh" + +extern "C" { + +/* ------------------------------ CUBLAS init/shutdown ------------------------------ */ + +inline bool check_cublas_error() { + cublasStatus status = cublasGetError(); + + return status != CUBLAS_STATUS_SUCCESS; +} + +inline bool checkCUDAError() { + cudaError_t err = cudaGetLastError(); + + if (cudaSuccess != err) + printf("%s\n", cudaGetErrorString( err)); + return cudaSuccess != err; +} + +EXPORT const char* get_last_cuda_error() { + cudaError_t err = cudaGetLastError(); + + return cudaGetErrorString( err); +} + +EXPORT const char* get_last_clib_error() { + return strerror(errno); +} + +EXPORT int cublas_init() { + cublasInit(); + if (check_cublas_error()) + return CUBLAS_ERROR; + else + return 0; +} + +EXPORT int cublas_shutdown() { + cublasShutdown(); + cudaThreadExit(); + + return 0; +} + + +EXPORT int cuda_set_device(int deviceId) { + cudaSetDevice(deviceId); + + if (checkCUDAError()) + return CUDA_ERROR; + else + return 0; +} + +EXPORT int init_random(rnd_struct* rnd_state, int seed, char* cudamatpath) { + unsigned int * host_mults; + host_mults = (unsigned int*)malloc(NUM_RND_STREAMS * sizeof(unsigned int)); + FILE * pFile; + + pFile = fopen (cudamatpath,"r"); + if (pFile == NULL) { + return ERROR_FILE_OPEN; + } + + for (int i = 0; i < NUM_RND_STREAMS; i++) { + if (fscanf (pFile, "%u", &host_mults[i]) != 1) { + return ERROR_FILE_SCAN; + } + } + fclose (pFile); + + cublasAlloc(NUM_RND_STREAMS, sizeof(unsigned int), (void**)&rnd_state->dev_mults); + cublasAlloc(NUM_RND_STREAMS, sizeof(unsigned long long), (void**)&rnd_state->dev_words); + cublasSetVector(NUM_RND_STREAMS, sizeof(unsigned int), host_mults, 1, rnd_state->dev_mults, 1); + //cudaMalloc((void **)&rnd_state->dev_mults, NUM_RND_STREAMS * sizeof(unsigned int)); + //cudaMalloc((void **)&rnd_state->dev_words, NUM_RND_STREAMS * sizeof(unsigned long long)); + //cudaMemcpy(rnd_state->dev_mults, host_mults, NUM_RND_STREAMS * sizeof(unsigned int), cudaMemcpyHostToDevice); + cudaThreadSynchronize(); + + kSeedRandom<<>>(rnd_state->dev_mults, rnd_state->dev_words, seed); + + cudaThreadSynchronize(); + + if (checkCUDAError()) + return CUDA_ERROR; + else + return 0; +} + +/* ------------------------------ Utility routines ------------------------------ */ + +EXPORT int get_leading_dimension(cudamat* mat) { + return mat->is_trans ? mat->size[1] : mat->size[0]; +} + +EXPORT int get_nonleading_dimension(cudamat* mat) { + return mat->is_trans ? mat->size[0] : mat->size[1]; +} + +EXPORT void set_transpose(cudamat* mat, int is_trans) { + mat->is_trans = is_trans; +} + +inline char get_transpose_char(cudamat* mat) { + return mat->is_trans ? 't' : 'n'; +} + +EXPORT void cuda_sync_threads() { + cudaThreadSynchronize(); +} + +/* ------------------------------ Allocating/moving data ------------------------------ */ + +EXPORT int allocate_device_memory(cudamat* mat) { + int len = mat->size[0]*mat->size[1]; + + cublasStatus stat; + + stat = cublasAlloc(len, sizeof(mat->data_device[0]), (void**)&mat->data_device); + + if (stat != CUBLAS_STATUS_SUCCESS || check_cublas_error()) { + checkCUDAError(); + return CUBLAS_ERROR; + } + + mat->on_device = 1; + return 0; +} + +EXPORT int copy_to_host(cudamat* mat) { + int len = mat->size[0]*mat->size[1]; + + if (mat->on_device) { + cublasGetVector(len, sizeof(mat->data_host[0]), mat->data_device, 1, mat->data_host, 1); + + if (check_cublas_error()) + return CUBLAS_ERROR; + } else + return ERROR_NOT_ON_DEVICE; + + return 0; +} + +EXPORT int copy_to_device(cudamat* mat) { + int len = mat->size[0]*mat->size[1]; + int err_code = 0; + + //if (!mat->owns_data) + // return VIEW_ERROR; + + if (!mat->on_device) { + err_code = allocate_device_memory(mat); + if (err_code) + return err_code; + } + + cublasSetVector(len, sizeof(mat->data_host[0]), mat->data_host, 1, mat->data_device, 1); + + if (check_cublas_error()) + return CUBLAS_ERROR; + + return 0; +} + +EXPORT int copy_on_device(cudamat* mat1, cudamat* mat2) { + int len = mat1->size[0]*mat1->size[1]; + + if (mat1->size[0] != mat2->size[0] || mat1->size[1] != mat2->size[1]) + return ERROR_INCOMPATIBLE_DIMENSIONS; + + cublasDcopy(len, mat1->data_device, 1, mat2->data_device, 1); + + if (check_cublas_error()) + return CUBLAS_ERROR; + else + return 0; +} + +EXPORT int get_row_slice(cudamat* source, cudamat* target, unsigned int start, unsigned int end) { + int height = source->size[0]; + int width = source->size[1]; + + if ((end - start) != target->size[0] || source->size[1] != target->size[1] || start >= end || end > height) + return ERROR_INCOMPATIBLE_DIMENSIONS; + + dim3 kernelBlockGrid((int)ceil((end - start)/32.), (int)ceil(width/32.), 1); + dim3 kernelBlockDim(32, 1, 1); + + kGetRowSlice<<>>(source->data_device, target->data_device, start, end, width, height); + + if (SYNC_THREADS) + cudaThreadSynchronize(); + + if (checkCUDAError()) + return CUDA_ERROR; + else + return 0; +} + +EXPORT int set_row_slice(cudamat* source, cudamat* target, unsigned int start, unsigned int end) { + int height = target->size[0]; + int width = target->size[1]; + + if ((end - start) != source->size[0] || source->size[1] != target->size[1] || start >= end || end > height) + return ERROR_INCOMPATIBLE_DIMENSIONS; + + dim3 kernelBlockGrid((int)ceil((end - start)/32.), (int)ceil(width/32.), 1); + dim3 kernelBlockDim(32, 1, 1); + + kSetRowSlice<<>>(source->data_device, target->data_device, start, end, width, height); + + if (SYNC_THREADS) + cudaThreadSynchronize(); + + if (checkCUDAError()) + return CUDA_ERROR; + else + return 0; +} + +EXPORT int copy_transpose(cudamat* source, cudamat* target) { + unsigned int height = source->size[0]; + unsigned int width = source->size[1]; + + if (source->size[0] != target->size[1] || source->size[1] != target->size[0]) + return ERROR_INCOMPATIBLE_DIMENSIONS; + + // setup execution parameters + unsigned int grid_x = height / COPY_BLOCK_SIZE; + if (height % COPY_BLOCK_SIZE) + grid_x++; + + unsigned int grid_y = width / COPY_BLOCK_SIZE; + if (width % COPY_BLOCK_SIZE) + grid_y++; + + dim3 grid(grid_x, grid_y, 1); + dim3 threads(COPY_BLOCK_SIZE, COPY_BLOCK_SIZE, 1); + + kTranspose<<< grid, threads >>>(target->data_device, source->data_device, height, width); + + if (checkCUDAError()) + return CUDA_ERROR; + else + return 0; +} + +EXPORT int free_device_memory(cudamat* mat) { + if (mat->owns_data && mat->on_device) { + cublasStatus stat; + + stat = cublasFree(mat->data_device); + mat->on_device = 0; + + if (stat != CUBLAS_STATUS_SUCCESS || check_cublas_error()) + return CUBLAS_ERROR; + } + + return 0; +} + +EXPORT int reshape(cudamat* mat, unsigned int m, unsigned int n) { + if (mat->size[0] * mat->size[1] != m * n) + return ERROR_INCOMPATIBLE_DIMENSIONS; + + mat->size[0] = m; + mat->size[1] = n; + + return 0; +} + +EXPORT int get_slice(cudamat* source, cudamat* target, unsigned int first_col, unsigned int last_col) { + if (source->is_trans) + return ERROR_TRANSPOSED; + + if (!source->on_device) + return ERROR_NOT_ON_DEVICE; + + if (last_col > source->size[1] || (first_col >= last_col)) + return ERROR_INCOMPATIBLE_DIMENSIONS; + + int num_rows = source->size[0]; + + target->data_host = 0; + target->data_device = source->data_device + first_col * num_rows; + target->on_device = 1; + target->on_host = 0; + target->size[0] = source->size[0]; + target->size[1] = last_col - first_col; + target->is_trans = 0; + target->owns_data = 0; + + return 0; +} + +EXPORT int get_vector_slice(cudamat* source, cudamat* target, unsigned int first_ind, unsigned int last_ind) { + // source must be a vector + if (source->size[0] > 1 && source->size[1] > 1) + return ERROR_GENERIC; + + if (source->is_trans) + return ERROR_TRANSPOSED; + + if (!source->on_device) + return ERROR_NOT_ON_DEVICE; + + if (first_ind >= last_ind) + return ERROR_INCOMPATIBLE_DIMENSIONS; + + int num_rows = source->size[0]; + + target->data_host = 0; + target->data_device = source->data_device + first_ind * num_rows; + target->on_device = 1; + target->on_host = 0; + target->is_trans = 0; + target->owns_data = 0; + + if (source->size[0] > 1) { + if (last_ind > source->size[0]) + return ERROR_INCOMPATIBLE_DIMENSIONS; + + target->size[0] = last_ind - first_ind; + target->size[1] = 1; + } else { + if (last_ind > source->size[1]) + return ERROR_INCOMPATIBLE_DIMENSIONS; + + target->size[0] = 1; + target->size[1] = last_ind - first_ind; + } + + return 0; +} + +/* ------------------------------ Initialization routines ------------------------------ */ + +EXPORT void init_from_array(cudamat* mat, double* data, int m, int n) { + mat->data_host = data; + mat->size[0] = m; + mat->size[1] = n; + mat->on_device = 0; + mat->on_host = 1; + mat->is_trans = 0; + mat->owns_data = 1; +} + +EXPORT int init_empty(cudamat* mat, int m, int n) { + mat->size[0] = m; + mat->size[1] = n; + mat->on_device = 0; + mat->on_host = 0; + mat->is_trans = 0; + mat->owns_data = 1; + + return allocate_device_memory(mat); +} + +/* ------------------------------ Random number generation ------------------------------ */ +EXPORT int fill_with_rand(rnd_struct* rnd_state, cudamat* mat) { + int len = mat->size[0] * mat->size[1]; + + if (!mat->on_device) + return ERROR_NOT_ON_DEVICE; + + kRandomUniform<<>>(rnd_state->dev_mults, rnd_state->dev_words, mat->data_device, len); + + if (SYNC_THREADS) + cudaThreadSynchronize(); + + if (checkCUDAError()) + return CUDA_ERROR; + else + return 0; +} + +EXPORT int fill_with_randn(rnd_struct* rnd_state, cudamat* mat) { + int len = mat->size[0] * mat->size[1]; + + if (!mat->on_device) + return ERROR_NOT_ON_DEVICE; + + kRandomGaussian<<>>(rnd_state->dev_mults, rnd_state->dev_words, mat->data_device, len); + + if (SYNC_THREADS) + cudaThreadSynchronize(); + + if (checkCUDAError()) + return CUDA_ERROR; + else + return 0; +} +/* ------------------------------ Algebraic operations ------------------------------ */ + +EXPORT int add_col_vec(cudamat* mat, cudamat* vec, cudamat* target) { + unsigned int h = mat->size[0], + w = mat->size[1]; + + if (!mat->on_device || !vec->on_device) + return ERROR_NOT_ON_DEVICE; + + if (mat->is_trans) + return ERROR_TRANSPOSED; + + if (mat->size[0] != vec->size[0] || vec->size[1] != 1 || + mat->size[0] != target->size[0] || mat->size[1] != target->size[1]) + return ERROR_INCOMPATIBLE_DIMENSIONS; + + kAddColVector<<>>(mat->data_device, vec->data_device, target->data_device, w, h); + + if (SYNC_THREADS) + cudaThreadSynchronize(); + + if (checkCUDAError()) { + return CUDA_ERROR; + } + + return 0; +} + +EXPORT int add_col_mult(cudamat* mat, cudamat* vec, cudamat* target, double mult) { + unsigned int h = mat->size[0], + w = mat->size[1]; + + if (!mat->on_device || !vec->on_device) + return ERROR_NOT_ON_DEVICE; + + if (mat->is_trans) + return ERROR_TRANSPOSED; + + if (mat->size[0] != vec->size[0] || vec->size[1] != 1 || + mat->size[0] != target->size[0] || mat->size[1] != target->size[1]) + return ERROR_INCOMPATIBLE_DIMENSIONS; + + kAddColMult<<>>(mat->data_device, vec->data_device, target->data_device, mult, w, h); + + if (SYNC_THREADS) + cudaThreadSynchronize(); + + if (checkCUDAError()) + return CUDA_ERROR; + + return 0; +} + +EXPORT int add_row_vec(cudamat* mat, cudamat* vec, cudamat* target) { + unsigned int h = mat->size[0], + w = mat->size[1]; + + if (!mat->on_device || !vec->on_device) + return ERROR_NOT_ON_DEVICE; + + if (mat->is_trans) + return ERROR_TRANSPOSED; + + if (mat->size[1] != vec->size[1] || vec->size[0] != 1 || + mat->size[0] != target->size[0] || mat->size[1] != target->size[1]) + return ERROR_INCOMPATIBLE_DIMENSIONS; + + kAddRowVector<<>>(mat->data_device, vec->data_device, target->data_device, w, h); + + if (SYNC_THREADS) + cudaThreadSynchronize(); + + if (checkCUDAError()) + return CUDA_ERROR; + + return 0; +} + +EXPORT int mult_by_col_vec(cudamat* mat, cudamat* vec, cudamat* target) { + unsigned int h = mat->size[0], + w = mat->size[1]; + + if (!mat->on_device || !vec->on_device) + return ERROR_NOT_ON_DEVICE; + + if (mat->is_trans) + return ERROR_TRANSPOSED; + + if (mat->size[0] != vec->size[0] || vec->size[1] != 1 || + mat->size[0] != target->size[0] || mat->size[1] != target->size[1]) + return ERROR_INCOMPATIBLE_DIMENSIONS; + + kMultByColVector<<>>(mat->data_device, vec->data_device, target->data_device, w, h); + + if (SYNC_THREADS) + cudaThreadSynchronize(); + + if (checkCUDAError()) + return CUDA_ERROR; + + return 0; +} + +EXPORT int mult_by_row_vec(cudamat* mat, cudamat* vec, cudamat* target) { + unsigned int h = mat->size[0], + w = mat->size[1]; + + if (!mat->on_device || !vec->on_device) + return ERROR_NOT_ON_DEVICE; + + if (mat->is_trans) + return ERROR_TRANSPOSED; + + if (mat->size[1] != vec->size[1] || vec->size[0] != 1 || + mat->size[0] != target->size[0] || mat->size[1] != target->size[1]) + return ERROR_INCOMPATIBLE_DIMENSIONS; + + kMultByRowVector<<>>(mat->data_device, vec->data_device, target->data_device, w, h); + + if (SYNC_THREADS) + cudaThreadSynchronize(); + + if (checkCUDAError()) + return CUDA_ERROR; + + return 0; +} + +EXPORT int divide_by_col_vec(cudamat* mat, cudamat* vec, cudamat* target) { + unsigned int h = mat->size[0], + w = mat->size[1]; + + if (!mat->on_device || !vec->on_device) + return ERROR_NOT_ON_DEVICE; + + if (mat->is_trans) + return ERROR_TRANSPOSED; + + if (mat->size[0] != vec->size[0] || vec->size[1] != 1 || + mat->size[0] != target->size[0] || mat->size[1] != target->size[1]) + return ERROR_INCOMPATIBLE_DIMENSIONS; + + kDivByColVector<<>>(mat->data_device, vec->data_device, target->data_device, w, h); + + if (SYNC_THREADS) + cudaThreadSynchronize(); + + if (checkCUDAError()) + return CUDA_ERROR; + + return 0; +} + +EXPORT int divide_by_row_vec(cudamat* mat, cudamat* vec, cudamat* target) { + unsigned int h = mat->size[0], + w = mat->size[1]; + + if (!mat->on_device || !vec->on_device) + return ERROR_NOT_ON_DEVICE; + + if (mat->is_trans) + return ERROR_TRANSPOSED; + + if (mat->size[1] != vec->size[1] || vec->size[0] != 1 || + mat->size[0] != target->size[0] || mat->size[1] != target->size[1]) + return ERROR_INCOMPATIBLE_DIMENSIONS; + + kDivByRowVector<<>>(mat->data_device, vec->data_device, target->data_device, w, h); + + if (SYNC_THREADS) + cudaThreadSynchronize(); + + if (checkCUDAError()) + return CUDA_ERROR; + + return 0; +} + +EXPORT int less_than(cudamat* mat1, cudamat* mat2, cudamat* target) { + int len = mat1->size[0]*mat1->size[1]; + + if (!mat1->on_device || !mat2->on_device) + return ERROR_NOT_ON_DEVICE; + + if (mat1->is_trans != mat2->is_trans) + return ERROR_TRANSPOSEDNESS; + + if (mat1->size[0] != mat2->size[0] || mat1->size[1] != mat2->size[1] || + mat1->size[0] != target->size[0] || mat1->size[1] != target->size[1]) + return ERROR_INCOMPATIBLE_DIMENSIONS; + + kLessThan<<>>(mat1->data_device, mat2->data_device, target->data_device, len); + + if (SYNC_THREADS) + cudaThreadSynchronize(); + + if (checkCUDAError()) + return CUDA_ERROR; + + return 0; +} + +EXPORT int less_than_scalar(cudamat* mat, double val, cudamat* target) { + int len = mat->size[0]*mat->size[1]; + + if (!mat->on_device || !target->on_device) + return ERROR_NOT_ON_DEVICE; + + if (mat->is_trans != target->is_trans) + return ERROR_TRANSPOSEDNESS; + + if (mat->size[0] != target->size[0] || mat->size[1] != target->size[1]) + return ERROR_INCOMPATIBLE_DIMENSIONS; + + kLessThanScalar<<>>(mat->data_device, val, target->data_device, len); + + if (SYNC_THREADS) + cudaThreadSynchronize(); + + if (checkCUDAError()) + return CUDA_ERROR; + + return 0; +} + +EXPORT int greater_than(cudamat* mat1, cudamat* mat2, cudamat* target) { + int len = mat1->size[0]*mat1->size[1]; + + if (!mat1->on_device || !mat2->on_device || !target->on_device) + return ERROR_NOT_ON_DEVICE; + + if (mat1->is_trans != mat2->is_trans) + return ERROR_TRANSPOSEDNESS; + + if (mat1->size[0] != mat2->size[0] || mat1->size[1] != mat2->size[1] || + mat1->size[0] != target->size[0] || mat1->size[1] != target->size[1]) + return ERROR_INCOMPATIBLE_DIMENSIONS; + + kGreaterThan<<>>(mat1->data_device, mat2->data_device, target->data_device, len); + + if (SYNC_THREADS) + cudaThreadSynchronize(); + + if (checkCUDAError()) + return CUDA_ERROR; + + return 0; +} + +EXPORT int greater_than_scalar(cudamat* mat, double val, cudamat* target) { + int len = mat->size[0]*mat->size[1]; + + if (!mat->on_device || !target->on_device) + return ERROR_NOT_ON_DEVICE; + + if (mat->is_trans != target->is_trans) + return ERROR_TRANSPOSEDNESS; + + if (mat->size[0] != target->size[0] || mat->size[1] != target->size[1]) + return ERROR_INCOMPATIBLE_DIMENSIONS; + + kGreaterThanScalar<<>>(mat->data_device, val, target->data_device, len); + + if (SYNC_THREADS) + cudaThreadSynchronize(); + + if (checkCUDAError()) + return CUDA_ERROR; + + return 0; +} + +EXPORT int equals(cudamat* mat1, cudamat* mat2, cudamat* target) { + int len = mat1->size[0]*mat1->size[1]; + + if (!mat1->on_device || !mat2->on_device || !target->on_device) + return ERROR_NOT_ON_DEVICE; + + if (mat1->is_trans != mat2->is_trans) + return ERROR_TRANSPOSEDNESS; + + if (mat1->size[0] != mat2->size[0] || mat1->size[1] != mat2->size[1] || + mat1->size[0] != target->size[0] || mat1->size[1] != target->size[1]) + return ERROR_INCOMPATIBLE_DIMENSIONS; + + kEquals<<>>(mat1->data_device, mat2->data_device, target->data_device, len); + + if (SYNC_THREADS) + cudaThreadSynchronize(); + + if (checkCUDAError()) + return CUDA_ERROR; + + return 0; +} + +EXPORT int equals_scalar(cudamat* mat, double val, cudamat* target) { + int len = mat->size[0]*mat->size[1]; + + if (!mat->on_device || !target->on_device) + return ERROR_NOT_ON_DEVICE; + + if (mat->is_trans != target->is_trans) + return ERROR_TRANSPOSEDNESS; + + if (mat->size[0] != target->size[0] || mat->size[1] != target->size[1]) + return ERROR_INCOMPATIBLE_DIMENSIONS; + + kEqualsScalar<<>>(mat->data_device, val, target->data_device, len); + + if (SYNC_THREADS) + cudaThreadSynchronize(); + + if (checkCUDAError()) + return CUDA_ERROR; + + return 0; +} + +EXPORT int minimum(cudamat* mat1, cudamat* mat2, cudamat* target) { + int len = mat1->size[0]*mat1->size[1]; + + if (!mat1->on_device || !mat2->on_device || !target->on_device) + return ERROR_NOT_ON_DEVICE; + + if (mat1->is_trans != mat2->is_trans) + return ERROR_TRANSPOSEDNESS; + + if (mat1->size[0] != mat2->size[0] || mat1->size[1] != mat2->size[1] || + mat1->size[0] != target->size[0] || mat1->size[1] != target->size[1]) + return ERROR_INCOMPATIBLE_DIMENSIONS; + + kMinimum<<>>(mat1->data_device, mat2->data_device, target->data_device, len); + + cudaThreadSynchronize(); + + if (checkCUDAError()) + return CUDA_ERROR; + + return 0; +} + +EXPORT int minimum_scalar(cudamat* mat, double val, cudamat* target) { + int len = mat->size[0]*mat->size[1]; + + if (!mat->on_device || !target->on_device) + return ERROR_NOT_ON_DEVICE; + + if (mat->is_trans != target->is_trans) + return ERROR_TRANSPOSEDNESS; + + if (mat->size[0] != target->size[0] || mat->size[1] != target->size[1]) + return ERROR_INCOMPATIBLE_DIMENSIONS; + + kMinimumScalar<<>>(mat->data_device, val, target->data_device, len); + + cudaThreadSynchronize(); + + if (checkCUDAError()) + return CUDA_ERROR; + + return 0; +} + +EXPORT int maximum(cudamat* mat1, cudamat* mat2, cudamat* target) { + int len = mat1->size[0]*mat1->size[1]; + + if (!mat1->on_device || !mat2->on_device || !target->on_device) + return ERROR_NOT_ON_DEVICE; + + if (mat1->is_trans != mat2->is_trans) + return ERROR_TRANSPOSEDNESS; + + if (mat1->size[0] != mat2->size[0] || mat1->size[1] != mat2->size[1] || + mat1->size[0] != target->size[0] || mat1->size[1] != target->size[1]) + return ERROR_INCOMPATIBLE_DIMENSIONS; + + kMaximum<<>>(mat1->data_device, mat2->data_device, target->data_device, len); + + cudaThreadSynchronize(); + + if (checkCUDAError()) + return CUDA_ERROR; + + return 0; +} + +EXPORT int maximum_scalar(cudamat* mat, double val, cudamat* target) { + int len = mat->size[0]*mat->size[1]; + + if (!mat->on_device || !target->on_device) + return ERROR_NOT_ON_DEVICE; + + if (mat->is_trans != target->is_trans) + return ERROR_TRANSPOSEDNESS; + + if (mat->size[0] != target->size[0] || mat->size[1] != target->size[1]) + return ERROR_INCOMPATIBLE_DIMENSIONS; + + kMaximumScalar<<>>(mat->data_device, val, target->data_device, len); + + cudaThreadSynchronize(); + + if (checkCUDAError()) + return CUDA_ERROR; + + return 0; +} + +EXPORT int min_by_axis(cudamat* mat, cudamat* target, int axis) { + unsigned int h = mat->size[0], + w = mat->size[1]; + + if (!mat->on_device || !target->on_device) + return ERROR_NOT_ON_DEVICE; + + if (mat->is_trans) + return ERROR_TRANSPOSED; + + if (axis == 0) { + if (target->size[0] != 1 || target->size[1] != mat->size[1]) + return ERROR_INCOMPATIBLE_DIMENSIONS; + + kMinColumnwise<<>>(mat->data_device, target->data_device, w, h); + } else { + if (target->size[1] != 1 || target->size[0] != mat->size[0]) + return ERROR_INCOMPATIBLE_DIMENSIONS; + + kMinRowwise<<>>(mat->data_device, target->data_device, w, h); + } + + if (SYNC_THREADS) + cudaThreadSynchronize(); + + if (checkCUDAError()) + return CUDA_ERROR; + + return 0; +} + +EXPORT int max_by_axis(cudamat* mat, cudamat* target, int axis) { + unsigned int h = mat->size[0], + w = mat->size[1]; + + if (!mat->on_device || !target->on_device) + return ERROR_NOT_ON_DEVICE; + + if (mat->is_trans) + return ERROR_TRANSPOSED; + + if (axis == 0) { + if (target->size[0] != 1 || target->size[1] != mat->size[1]) + return ERROR_INCOMPATIBLE_DIMENSIONS; + + kMaxColumnwise<<>>(mat->data_device, target->data_device, w, h); + } else { + if (target->size[1] != 1 || target->size[0] != mat->size[0]) + return ERROR_INCOMPATIBLE_DIMENSIONS; + + kMaxRowwise<<>>(mat->data_device, target->data_device, w, h); + } + + if (SYNC_THREADS) + cudaThreadSynchronize(); + + if (checkCUDAError()) + return CUDA_ERROR; + + return 0; +} + +EXPORT int argmin_by_axis(cudamat* mat, cudamat* target, int axis) { + unsigned int h = mat->size[0], + w = mat->size[1]; + + if (!mat->on_device || !target->on_device) + return ERROR_NOT_ON_DEVICE; + + if (mat->is_trans) + return ERROR_TRANSPOSED; + + if (axis == 0) { + if (target->size[0] != 1 || target->size[1] != mat->size[1]) + return ERROR_INCOMPATIBLE_DIMENSIONS; + + kArgMinColumnwise<<>>(mat->data_device, target->data_device, w, h); + } else { + if (target->size[1] != 1 || target->size[0] != mat->size[0]) + return ERROR_INCOMPATIBLE_DIMENSIONS; + + kArgMinRowwise<<>>(mat->data_device, target->data_device, w, h); + } + + if (SYNC_THREADS) + cudaThreadSynchronize(); + + if (checkCUDAError()) + return CUDA_ERROR; + + return 0; +} + +EXPORT int argmax_by_axis(cudamat* mat, cudamat* target, int axis) { + unsigned int h = mat->size[0], + w = mat->size[1]; + + if (!mat->on_device || !target->on_device) + return ERROR_NOT_ON_DEVICE; + + if (mat->is_trans) + return ERROR_TRANSPOSED; + + if (axis == 0) { + if (target->size[0] != 1 || target->size[1] != mat->size[1]) + return ERROR_INCOMPATIBLE_DIMENSIONS; + + kArgMaxColumnwise<<>>(mat->data_device, target->data_device, w, h); + } else { + if (target->size[1] != 1 || target->size[0] != mat->size[0]) + return ERROR_INCOMPATIBLE_DIMENSIONS; + + kArgMaxRowwise<<>>(mat->data_device, target->data_device, w, h); + } + + if (SYNC_THREADS) + cudaThreadSynchronize(); + + if (checkCUDAError()) + return CUDA_ERROR; + + return 0; +} + +EXPORT int sign(cudamat* mat, cudamat* target) { + int len = mat->size[0]*mat->size[1]; + + if (!mat->on_device || !target->on_device) + return ERROR_NOT_ON_DEVICE; + + if (mat->is_trans != target->is_trans) + return ERROR_TRANSPOSEDNESS; + + if (mat->size[0] != target->size[0] || mat->size[1] != target->size[1]) + return ERROR_INCOMPATIBLE_DIMENSIONS; + + kSign<<>>(mat->data_device, target->data_device, len); + + if (SYNC_THREADS) + cudaThreadSynchronize(); + + if (checkCUDAError()) + return CUDA_ERROR; + + return 0; +} + +EXPORT int apply_sigmoid(cudamat* mat, cudamat* target) { + unsigned int len = mat->size[0] * mat->size[1]; + + if (!mat->on_device || !target->on_device) + return ERROR_NOT_ON_DEVICE; + + if (mat->size[0] != target->size[0] || mat->size[1] != target->size[1]) + return ERROR_INCOMPATIBLE_DIMENSIONS; + + kApplySigmoid<<>>(mat->data_device, target->data_device, len); + + if (SYNC_THREADS) + cudaThreadSynchronize(); + + if (checkCUDAError()) + return CUDA_ERROR; + + return 0; +} + +EXPORT int apply_tanh(cudamat* mat, cudamat* target) { + unsigned int len = mat->size[0] * mat->size[1]; + + if (!mat->on_device || !target->on_device) + return ERROR_NOT_ON_DEVICE; + + if (mat->size[0] != target->size[0] || mat->size[1] != target->size[1]) + return ERROR_INCOMPATIBLE_DIMENSIONS; + + kApplyTanh<<>>(mat->data_device, target->data_device, len); + + if (SYNC_THREADS) + cudaThreadSynchronize(); + + if (checkCUDAError()) + return CUDA_ERROR; + + return 0; +} + +EXPORT int apply_soft_threshold(cudamat* mat, double alpha, cudamat* target) { + unsigned int len = mat->size[0] * mat->size[1]; + + if (!mat->on_device || !target->on_device) + return ERROR_NOT_ON_DEVICE; + + if (mat->size[0] != target->size[0] || mat->size[1] != target->size[1]) + return ERROR_INCOMPATIBLE_DIMENSIONS; + + kApplySoftThreshold<<>>(mat->data_device, alpha, target->data_device, len); + + cudaThreadSynchronize(); + + if (checkCUDAError()) + return CUDA_ERROR; + + return 0; +} + +EXPORT int apply_abs(cudamat* mat, cudamat* target) { + unsigned int len = mat->size[0] * mat->size[1]; + + if (!mat->on_device || !target->on_device) + return ERROR_NOT_ON_DEVICE; + + if (mat->size[0] != target->size[0] || mat->size[1] != target->size[1]) + return ERROR_INCOMPATIBLE_DIMENSIONS; + + kApplyAbs<<>>(mat->data_device, target->data_device, len); + + if (SYNC_THREADS) + cudaThreadSynchronize(); + + if (checkCUDAError()) + return CUDA_ERROR; + + return 0; +} + +EXPORT int apply_log_1_plus_exp(cudamat* mat, cudamat* target) { + unsigned int len = mat->size[0] * mat->size[1]; + + if (!mat->on_device || !target->on_device) + return ERROR_NOT_ON_DEVICE; + + if (mat->size[0] != target->size[0] || mat->size[1] != target->size[1]) + return ERROR_INCOMPATIBLE_DIMENSIONS; + + kApplyLog1PlusExp<<>>(mat->data_device, target->data_device, len); + + if (SYNC_THREADS) + cudaThreadSynchronize(); + + if (checkCUDAError()) + return CUDA_ERROR; + + return 0; +} + +EXPORT int apply_log(cudamat* mat, cudamat* target) { + unsigned int len = mat->size[0] * mat->size[1]; + + if (!mat->on_device || !target->on_device) + return ERROR_NOT_ON_DEVICE; + + if (mat->size[0] != target->size[0] || mat->size[1] != target->size[1]) + return ERROR_INCOMPATIBLE_DIMENSIONS; + + kLog<<>>(mat->data_device, target->data_device, len); + + if (SYNC_THREADS) + cudaThreadSynchronize(); + + if (checkCUDAError()) + return CUDA_ERROR; + + return 0; +} + +EXPORT int apply_exp(cudamat* mat, cudamat* target) { + unsigned int len = mat->size[0] * mat->size[1]; + + if (!mat->on_device || !target->on_device) + return ERROR_NOT_ON_DEVICE; + + if (mat->size[0] != target->size[0] || mat->size[1] != target->size[1]) + return ERROR_INCOMPATIBLE_DIMENSIONS; + + kExp<<>>(mat->data_device, target->data_device, len); + + if (SYNC_THREADS) + cudaThreadSynchronize(); + + if (checkCUDAError()) + return CUDA_ERROR; + + return 0; +} + +EXPORT int apply_gamma(cudamat* mat, cudamat* target) { + unsigned int len = mat->size[0] * mat->size[1]; + + if (!mat->on_device || !target->on_device) + return ERROR_NOT_ON_DEVICE; + + if (mat->size[0] != target->size[0] || mat->size[1] != target->size[1]) + return ERROR_INCOMPATIBLE_DIMENSIONS; + + kGamma<<>>(mat->data_device, target->data_device, len); + + if (SYNC_THREADS) + cudaThreadSynchronize(); + + if (checkCUDAError()) + return CUDA_ERROR; + + return 0; +} + +EXPORT int apply_lgamma(cudamat* mat, cudamat* target) { + unsigned int len = mat->size[0] * mat->size[1]; + + if (!mat->on_device || !target->on_device) + return ERROR_NOT_ON_DEVICE; + + if (mat->size[0] != target->size[0] || mat->size[1] != target->size[1]) + return ERROR_INCOMPATIBLE_DIMENSIONS; + + kLogGamma<<>>(mat->data_device, target->data_device, len); + + if (SYNC_THREADS) + cudaThreadSynchronize(); + + if (checkCUDAError()) + return CUDA_ERROR; + + return 0; +} + +EXPORT int apply_sqrt(cudamat* mat, cudamat* target) { + unsigned int len = mat->size[0] * mat->size[1]; + + if (!mat->on_device || !target->on_device) + return ERROR_NOT_ON_DEVICE; + + if (mat->size[0] != target->size[0] || mat->size[1] != target->size[1]) + return ERROR_INCOMPATIBLE_DIMENSIONS; + + kSqrt<<>>(mat->data_device, target->data_device, len); + + if (SYNC_THREADS) + cudaThreadSynchronize(); + + if (checkCUDAError()) + return CUDA_ERROR; + + return 0; +} + +EXPORT int apply_pow(cudamat* mat, double pow, cudamat* target) { + unsigned int len = mat->size[0] * mat->size[1]; + + if (!mat->on_device || !target->on_device) + return ERROR_NOT_ON_DEVICE; + + if (mat->size[0] != target->size[0] || mat->size[1] != target->size[1]) + return ERROR_INCOMPATIBLE_DIMENSIONS; + + kPow<<>>(mat->data_device, pow, target->data_device, len); + + if (SYNC_THREADS) + cudaThreadSynchronize(); + + if (checkCUDAError()) + return CUDA_ERROR; + + return 0; +} + +EXPORT int apply_pow_matrix(cudamat* mat, cudamat* pow, cudamat* target) { + unsigned int len = mat->size[0] * mat->size[1]; + + if (!mat->on_device || !target->on_device) + return ERROR_NOT_ON_DEVICE; + + if (mat->size[0] != target->size[0] || mat->size[1] != target->size[1]) + return ERROR_INCOMPATIBLE_DIMENSIONS; + + if (mat->size[0] != pow->size[0] || mat->size[1] != pow->size[1]) + return ERROR_INCOMPATIBLE_DIMENSIONS; + + kPowMatrix<<>>(mat->data_device, pow->data_device, target->data_device, len); + + if (SYNC_THREADS) + cudaThreadSynchronize(); + + if (checkCUDAError()) + return CUDA_ERROR; + + return 0; +} + +EXPORT int reciprocal(cudamat* mat, cudamat* target) { + unsigned int len = mat->size[0] * mat->size[1]; + + if (!mat->on_device || !target->on_device) + return ERROR_NOT_ON_DEVICE; + + if (mat->size[0] != target->size[0] || mat->size[1] != target->size[1]) + return ERROR_INCOMPATIBLE_DIMENSIONS; + + kReciprocal<<>>(mat->data_device, target->data_device, len); + + if (SYNC_THREADS) + cudaThreadSynchronize(); + + if (checkCUDAError()) + return CUDA_ERROR; + + return 0; +} + +EXPORT int dot(cudamat* mat1, cudamat* mat2, cudamat* target, double beta, double alpha) { + if (!mat1->on_device || !mat2->on_device || !target->on_device) + return ERROR_NOT_ON_DEVICE; + + if (get_leading_dimension(mat1) != get_leading_dimension(target) || + get_nonleading_dimension(mat2) != get_nonleading_dimension(target) || + get_nonleading_dimension(mat1) != get_leading_dimension(mat2)) { + return ERROR_INCOMPATIBLE_DIMENSIONS; + } + int m = get_leading_dimension(mat1), + k = get_leading_dimension(mat2), + n = get_nonleading_dimension(mat2); + + // gemv if second matrix is a (column) vector + if (n == 1) { + cublasDgemv(get_transpose_char(mat1), mat1->size[0], mat1->size[1], + alpha, mat1->data_device, mat1->size[0], + mat2->data_device, 1, + beta, target->data_device, 1); + } + // gemv if first matrix is a (row) vector + else if (m == 1) { + cublasDgemv(mat2->is_trans ? 'n' : 't', mat2->size[0], mat2->size[1], + alpha, mat2->data_device, mat2->size[0], + mat1->data_device, 1, + beta, target->data_device, 1); + } + // gemm otherwise + else { + cublasDgemm(get_transpose_char(mat1), get_transpose_char(mat2), + m, n, k, + alpha, mat1->data_device, mat1->size[0], + mat2->data_device, mat2->size[0], + beta, target->data_device, target->size[0]); + } + + if (check_cublas_error()) + return CUBLAS_ERROR; + + if (SYNC_THREADS) + cudaThreadSynchronize(); + + return 0; +} + +EXPORT double vdot(cudamat* mat1, cudamat* mat2, int* err_code) { + int len = mat1->size[0]*mat1->size[1]; + double res; + + if (!mat1->on_device || !mat2->on_device) + return ERROR_NOT_ON_DEVICE; + + if (mat1->is_trans != mat2->is_trans) { + *err_code = ERROR_TRANSPOSEDNESS; + return 0; + } + + if (mat1->size[0] != mat2->size[0] || mat1->size[1] != mat2->size[1]) { + *err_code = ERROR_INCOMPATIBLE_DIMENSIONS; + return 0; + } + + res = cublasDdot(len, mat1->data_device, 1, mat2->data_device, 1); + + if (check_cublas_error()) { + *err_code = CUBLAS_ERROR; + return -1.; + } else { + *err_code = 0; + return res; + } +} + +/* Perform the operation mat1 = mat1 + alpha * mat2. mat1 and mat2 must + have the same transposedness. */ +EXPORT int add_mult(cudamat* mat1, cudamat* mat2, double alpha) { + int len = mat1->size[0]*mat1->size[1]; + + if (!mat1->on_device || !mat2->on_device) + return ERROR_NOT_ON_DEVICE; + + if (mat1->is_trans != mat2->is_trans) + return ERROR_TRANSPOSEDNESS; + + if (mat1->size[0] != mat2->size[0] || mat1->size[1] != mat2->size[1]) + return ERROR_INCOMPATIBLE_DIMENSIONS; + + cublasDaxpy(len, alpha, mat2->data_device, 1, mat1->data_device, 1); + + if (check_cublas_error()) + return CUBLAS_ERROR; + + return 0; +} + +EXPORT int add_elementwise(cudamat* mat1, cudamat* mat2, cudamat* target) { + int len = mat1->size[0]*mat1->size[1]; + + if (!mat1->on_device || !mat2->on_device || !target->on_device) + return ERROR_NOT_ON_DEVICE; + + if (mat1->is_trans != mat2->is_trans) + return ERROR_TRANSPOSEDNESS; + + if (mat1->size[0] != mat2->size[0] || mat1->size[1] != mat2->size[1] || + mat1->size[0] != target->size[0] || mat1->size[1] != target->size[1]) + return ERROR_INCOMPATIBLE_DIMENSIONS; + + if (mat1 == target) { + cublasDaxpy(len, 1, mat2->data_device, 1, mat1->data_device, 1); + + if (check_cublas_error()) + return CUBLAS_ERROR; + + } else { + kAdd<<>>(mat1->data_device, mat2->data_device, target->data_device, len); + + if (SYNC_THREADS) + cudaThreadSynchronize(); + + if (checkCUDAError()) + return CUDA_ERROR; + } + + return 0; +} + +EXPORT int subtract_elementwise(cudamat* mat1, cudamat* mat2, cudamat* target) { + int len = mat1->size[0]*mat1->size[1]; + + if (!mat1->on_device || !mat2->on_device || !target->on_device) + return ERROR_NOT_ON_DEVICE; + + if (mat1->is_trans != mat2->is_trans) + return ERROR_TRANSPOSEDNESS; + + if (mat1->size[0] != mat2->size[0] || mat1->size[1] != mat2->size[1] || + mat1->size[0] != target->size[0] || mat1->size[1] != target->size[1]) + return ERROR_INCOMPATIBLE_DIMENSIONS; + + kSubtract<<>>(mat1->data_device, mat2->data_device, target->data_device, len); + + if (SYNC_THREADS) + cudaThreadSynchronize(); + + if (checkCUDAError()) + return CUDA_ERROR; + + return 0; +} + +EXPORT int divide_elementwise(cudamat* mat1, cudamat* mat2, cudamat* target) { + int len = mat1->size[0]*mat1->size[1]; + + if (!mat1->on_device || !mat2->on_device || !target->on_device) + return ERROR_NOT_ON_DEVICE; + + if (mat1->is_trans != mat2->is_trans) + return ERROR_TRANSPOSEDNESS; + + if (mat1->size[0] != mat2->size[0] || mat1->size[1] != mat2->size[1] || + mat1->size[0] != target->size[0] || mat1->size[1] != target->size[1]) + return ERROR_INCOMPATIBLE_DIMENSIONS; + + kDivide<<>>(mat1->data_device, mat2->data_device, target->data_device, len); + + if (SYNC_THREADS) + cudaThreadSynchronize(); + + if (checkCUDAError()) + return CUDA_ERROR; + + return 0; +} + +/* Elementwise multiplication of 2 matrices */ +EXPORT int mult_elementwise(cudamat* mat1, cudamat* mat2, cudamat* target) { + int len = mat1->size[0]*mat1->size[1]; + + if (!mat1->on_device || !mat2->on_device || !target->on_device) + return ERROR_NOT_ON_DEVICE; + + if (mat1->is_trans != mat2->is_trans) + return ERROR_TRANSPOSEDNESS; + + if (mat1->size[0] != mat2->size[0] || mat1->size[1] != mat2->size[1] || + mat1->size[0] != target->size[0] || mat1->size[1] != target->size[1]) + return ERROR_INCOMPATIBLE_DIMENSIONS; + + kMult<<>>(mat1->data_device, mat2->data_device, target->data_device, len); + + if (SYNC_THREADS) + cudaThreadSynchronize(); + + if (checkCUDAError()) + return CUDA_ERROR; + + return 0; +} + +EXPORT int assign_scalar(cudamat* mat, double alpha) { + int len = mat->size[0]*mat->size[1]; + + if (!mat->on_device) + return ERROR_NOT_ON_DEVICE; + + kAssignScalar<<>>(mat->data_device, alpha, len); + + if (SYNC_THREADS) + cudaThreadSynchronize(); + + if (checkCUDAError()) + return CUDA_ERROR; + + return 0; +} + +EXPORT int mult_by_scalar(cudamat* mat, double alpha, cudamat* target) { + int len = mat->size[0]*mat->size[1]; + + if (!mat->on_device || !target->on_device) + return ERROR_NOT_ON_DEVICE; + + if (mat->size[0] != target->size[0] || mat->size[1] != target->size[1]) + return ERROR_INCOMPATIBLE_DIMENSIONS; + + if (mat == target) { + cublasDscal(len, alpha, mat->data_device, 1); + + if (check_cublas_error()) + return CUBLAS_ERROR; + + } else { + kMultScalar<<>>(mat->data_device, alpha, target->data_device, len); + + if (SYNC_THREADS) + cudaThreadSynchronize(); + + if (checkCUDAError()) + return CUDA_ERROR; + } + + return 0; +} + +EXPORT int divide_by_scalar(cudamat* mat, double alpha, cudamat* target) { + int len = mat->size[0]*mat->size[1]; + + if (!mat->on_device || !target->on_device) + return ERROR_NOT_ON_DEVICE; + + if (mat->size[0] != target->size[0] || mat->size[1] != target->size[1]) + return ERROR_INCOMPATIBLE_DIMENSIONS; + + kDivideScalar<<>>(mat->data_device, alpha, target->data_device, len); + + if (SYNC_THREADS) + cudaThreadSynchronize(); + + if (checkCUDAError()) + return CUDA_ERROR; + + return 0; +} + +EXPORT int add_scalar(cudamat* mat, double alpha, cudamat* target) { + int len = mat->size[0]*mat->size[1]; + + if (!mat->on_device || !target->on_device) + return ERROR_NOT_ON_DEVICE; + + if (mat->size[0] != target->size[0] || mat->size[1] != target->size[1]) + return ERROR_INCOMPATIBLE_DIMENSIONS; + + kAddScalar<<>>(mat->data_device, alpha, target->data_device, len); + + if (SYNC_THREADS) + cudaThreadSynchronize(); + + if (checkCUDAError()) + return CUDA_ERROR; + + return 0; +} + +EXPORT double euclid_norm(cudamat* mat, int* err_code) { + int len = mat->size[0]*mat->size[1]; + + if (!mat->on_device) { + *err_code = ERROR_NOT_ON_DEVICE; + return -1.; + } + + double res = cublasDnrm2(len, mat->data_device, 1); + + if (check_cublas_error()) { + *err_code = CUBLAS_ERROR; + return -1.; + } else { + *err_code = 0; + return res; + } +} + +EXPORT double manhattan_norm(cudamat* mat, int* err_code) { + int len = mat->size[0]*mat->size[1]; + + if (!mat->on_device) { + *err_code = ERROR_NOT_ON_DEVICE; + return -1.; + } + + double res = cublasDasum(len, mat->data_device, 1); + + if (check_cublas_error()) { + *err_code = CUBLAS_ERROR; + return -1.; + } else { + *err_code = 0; + return res; + } +} + +EXPORT int selectRows(cudamat* source, cudamat* target, cudamat* indices){ + const int nRetRows = indices->size[1]; + + if (nRetRows==0) return 0; + + dim3 gridDim((nRetRows+31)/32); + dim3 blockDim(32); + + kSelectRows<<>>(source->data_device, target->data_device, indices->data_device, nRetRows, source->size[0], source->size[1]); + + if (SYNC_THREADS) + cudaThreadSynchronize(); + + if (checkCUDAError()) + return CUDA_ERROR; + else + return 0; +} + +EXPORT int setSelectedRows(cudamat* target, cudamat* source, cudamat* indices){ + const int nSetRows = indices->size[1]; + + if (nSetRows==0) + return 0; + + dim3 gridDim((nSetRows+31)/32); + dim3 blockDim(32); + + kSetSelectedRows<<>>(target->data_device, source->data_device, indices->data_device, nSetRows, target->size[0], target->size[1]); + + if (SYNC_THREADS) + cudaThreadSynchronize(); + + if (checkCUDAError()) + return CUDA_ERROR; + else + return 0; +} + +EXPORT int where(cudamat* condition_mat, cudamat* if_mat, cudamat* else_mat, cudamat* target) { + unsigned int len = condition_mat->size[0] * condition_mat->size[1]; + + if (!condition_mat->on_device || !target->on_device) + return ERROR_NOT_ON_DEVICE; + + if (condition_mat->size[0] != target->size[0] || condition_mat->size[1] != target->size[1]) + return ERROR_INCOMPATIBLE_DIMENSIONS; + + if (condition_mat->size[0] != if_mat->size[0] || condition_mat->size[1] != if_mat->size[1]) + return ERROR_INCOMPATIBLE_DIMENSIONS; + + if (condition_mat->size[0] != else_mat->size[0] || condition_mat->size[1] != else_mat->size[1]) + return ERROR_INCOMPATIBLE_DIMENSIONS; + + kWhere<<>>(condition_mat->data_device, + if_mat->data_device, else_mat->data_device, target->data_device, len); + + if (SYNC_THREADS) + cudaThreadSynchronize(); + + if (checkCUDAError()) + return CUDA_ERROR; + + return 0; +} + +EXPORT int correlate(cudamat* source, cudamat* kernel, cudamat* dest) { + int len = source->size[0] * source->size[1]; + + if (!source->on_device || !kernel->on_device || !dest->on_device) + return ERROR_NOT_ON_DEVICE; + + if (source->size[0] != dest->size[0] || source->size[1] != dest->size[1]) + return ERROR_INCOMPATIBLE_DIMENSIONS; + + if (kernel->size[0] % 2 == 0 || kernel->size[1] % 2 == 0 || + kernel->size[0] > source->size[0] || kernel->size[1] > source->size[1]) + return ERROR_INCOMPATIBLE_DIMENSIONS; + + kCorrelate<<>>(source->data_device, + kernel->data_device, dest->data_device, source->size[1], source->size[0], + kernel->size[1], kernel->size[0]); + + if (SYNC_THREADS) + cudaThreadSynchronize(); + + if (checkCUDAError()) + return CUDA_ERROR; + + return 0; +} + +} diff --git a/ot/gpu/cudamat/cudamat/cudamat.cuh b/ot/gpu/cudamat/cudamat/cudamat.cuh new file mode 100644 index 0000000..e4e2e34 --- /dev/null +++ b/ot/gpu/cudamat/cudamat/cudamat.cuh @@ -0,0 +1,35 @@ +#if defined(_WIN32) || defined(__CYGWIN__) + #define EXPORT __declspec(dllexport) +#else + #define EXPORT __attribute__ ((visibility("default"))) +#endif + +#define SYNC_THREADS 1 + +#define ERROR_INCOMPATIBLE_DIMENSIONS -1 +#define CUBLAS_ERROR -2 +#define CUDA_ERROR -3 +#define VIEW_ERROR -4 +#define ERROR_TRANSPOSED -5 +#define ERROR_GENERIC -6 +#define ERROR_TRANSPOSEDNESS -7 +#define ERROR_NOT_ON_DEVICE -8 +#define ERROR_UNSUPPORTED -9 +#define ERROR_FILE_OPEN -10 +#define ERROR_FILE_SCAN -11 + +struct cudamat { + double* data_host; + double* data_device; + int on_device; + int on_host; + int size[2]; + int is_trans; // 0 or 1 + int owns_data; +}; + +struct rnd_struct { + unsigned int* dev_mults; + unsigned long long* dev_words; +}; + diff --git a/ot/gpu/cudamat/cudamat/cudamat.py b/ot/gpu/cudamat/cudamat/cudamat.py new file mode 100644 index 0000000..f855ed2 --- /dev/null +++ b/ot/gpu/cudamat/cudamat/cudamat.py @@ -0,0 +1,1575 @@ +import os +import platform +import warnings +import sysconfig +import sys + +import ctypes as ct +import numpy as np + +def load_library(basename): + if platform.system() == 'Windows': + ext = '.dll' + else: + ext = sysconfig.get_config_var('SO') + return ct.cdll.LoadLibrary(os.path.join( + os.path.dirname(__file__) or os.path.curdir, + basename + ext)) + +_cudamat = load_library('libcudamat') + +_cudamat.get_last_cuda_error.restype = ct.c_char_p +_cudamat.get_last_clib_error.restype = ct.c_char_p +_cudamat.cublas_init.restype = ct.c_int +_cudamat.cublas_shutdown.restype = ct.c_int +_cudamat.cuda_set_device.restype = ct.c_int +_cudamat.init_random.restype = ct.c_int + +_cudamat.init_empty.restype = ct.c_int +_cudamat.reshape.restype = ct.c_int +_cudamat.copy_to_host.restype = ct.c_int +_cudamat.allocate_device_memory = ct.c_int +_cudamat.copy_to_device.restype = ct.c_int +_cudamat.copy_on_device.restype = ct.c_int +_cudamat.free_device_memory.restype = ct.c_int + +_cudamat.get_slice.restype = ct.c_int +_cudamat.get_row_slice.restype = ct.c_int +_cudamat.set_row_slice.restype = ct.c_int +_cudamat.copy_transpose.restype = ct.c_int +_cudamat.get_vector_slice.restype = ct.c_int +_cudamat.fill_with_rand.restype = ct.c_int +_cudamat.fill_with_randn.restype = ct.c_int + +_cudamat.add_col_vec.restype = ct.c_int +_cudamat.add_col_mult.restype = ct.c_int +_cudamat.add_row_vec.restype = ct.c_int +_cudamat.mult_by_col_vec.restype = ct.c_int +_cudamat.mult_by_row_vec.restype = ct.c_int +_cudamat.divide_by_col_vec.restype = ct.c_int +_cudamat.divide_by_row_vec.restype = ct.c_int + +_cudamat.less_than.restype = ct.c_int +_cudamat.less_than_scalar.restype = ct.c_int +_cudamat.greater_than.restype = ct.c_int +_cudamat.greater_than_scalar.restype = ct.c_int +_cudamat.equals.restype = ct.c_int +_cudamat.equals_scalar.restype = ct.c_int +_cudamat.minimum.restype = ct.c_int +_cudamat.minimum_scalar.restype = ct.c_int +_cudamat.maximum.restype = ct.c_int +_cudamat.maximum_scalar.restype = ct.c_int +_cudamat.min_by_axis.restype = ct.c_int +_cudamat.max_by_axis.restype = ct.c_int +_cudamat.argmin_by_axis.restype = ct.c_int +_cudamat.argmax_by_axis.restype = ct.c_int +_cudamat.sign.restype = ct.c_int +_cudamat.apply_sigmoid.restype = ct.c_int +_cudamat.apply_tanh.restype = ct.c_int +_cudamat.apply_soft_threshold.restype = ct.c_int +_cudamat.apply_abs.restype = ct.c_int +_cudamat.apply_log_1_plus_exp.restype = ct.c_int +_cudamat.apply_log.restype = ct.c_int +_cudamat.apply_exp.restype = ct.c_int +_cudamat.apply_gamma.restype = ct.c_int +_cudamat.apply_lgamma.restype = ct.c_int +_cudamat.apply_sqrt.restype = ct.c_int +_cudamat.apply_pow.restype = ct.c_int +_cudamat.apply_pow_matrix.restype = ct.c_int +_cudamat.reciprocal.restype = ct.c_int + +_cudamat.add_elementwise.restype = ct.c_int +_cudamat.subtract_elementwise.restype = ct.c_int +_cudamat.divide_elementwise.restype = ct.c_int +_cudamat.mult_elementwise.restype = ct.c_int +_cudamat.assign_scalar.restype = ct.c_int +_cudamat.mult_by_scalar.restype = ct.c_int +_cudamat.divide_by_scalar.restype = ct.c_int +_cudamat.add_scalar.restype = ct.c_int + +_cudamat.euclid_norm.restype = ct.c_double +_cudamat.manhattan_norm.restype = ct.c_double +_cudamat.selectRows.restype = ct.c_int +_cudamat.setSelectedRows.restype = ct.c_int +_cudamat.vdot.restype = ct.c_double +_cudamat.dot.restype = ct.c_int + +_cudamat.where.restype = ct.c_int + +_cudamat.correlate.restype = ct.c_int + + +def deprecated(func): + """This is a decorator which can be used to mark functions + as deprecated. It will result in a warning being emmitted + when the function is used.""" + + def newFunc(*args, **kwargs): + warnings.warn("Call to deprecated function %s." % func.__name__, + category=DeprecationWarning) + return func(*args, **kwargs) + newFunc.__name__ = func.__name__ + newFunc.__doc__ = func.__doc__ + newFunc.__dict__.update(func.__dict__) + return newFunc + + +class CUDAMatException(Exception): + pass + + +def get_last_cuda_error(): + errmsg = _cudamat.get_last_cuda_error() + if sys.version_info >= (3,): + return bytes(errmsg).decode() + else: + return str(errmsg) + + +def get_last_clib_error(): + errmsg = _cudamat.get_last_clib_error() + if sys.version_info >= (3,): + return bytes(errmsg).decode() + else: + return str(errmsg) + + +def generate_exception(err_code, **kwargs): + """ + Return a CUDAMatException object based on the error code err_code. + Additional arguments are error-specific and optional. + """ + + if err_code == -1: + return CUDAMatException("Incompatible matrix dimensions.") + elif err_code == -2: + return CUDAMatException("CUBLAS error.") + elif err_code == -3: + return CUDAMatException("CUDA error: " + get_last_cuda_error()) + elif err_code == -4: + return CUDAMatException("Operation not supported on views.") + elif err_code == -5: + return CUDAMatException("Operation not supported on " + "transposed matrices.") + elif err_code == -6: + return CUDAMatException("") + elif err_code == -7: + return CUDAMatException("Incompatible transposedness.") + elif err_code == -8: + return CUDAMatException("Matrix is not in device memory.") + elif err_code == -9: + return CUDAMatException("Operation not supported.") + elif err_code == -10: + filepath = kwargs.get("filepath",""); + if filepath: + filepath = ": '%s'" % filepath + return CUDAMatException("Cannot open file%s: %s" % (filepath,get_last_clib_error())) + elif err_code == -11: + filepath = kwargs.get("filepath",""); + if filepath: + filepath = ": '%s'" % filepath + return CUDAMatException("Cannot parse file%s." % filepath) + else: + return CUDAMatException("") + + +class cudamat(ct.Structure): + _fields_ = [('data_host', ct.POINTER(ct.c_double)), + ('data_device', ct.POINTER(ct.c_double)), + ('on_device', ct.c_int), + ('on_host', ct.c_int), + ('size', ct.c_int * 2), + ('is_trans', ct.c_int), + ('owns_data', ct.c_int)] + + +class rnd_struct(ct.Structure): + _fields_ = [('dev_rnd_mults', ct.POINTER(ct.c_uint)), + ('dev_rnd_words', ct.POINTER(ct.c_longlong))] + + +class TransposedCUDAMatrix(object): + def __init__(self, mat): + self.mat = cudamat() + ct.memmove(ct.pointer(self.mat), ct.pointer(mat), ct.sizeof(self.mat)) + self.mat.is_trans = 1 + self.p_mat = ct.pointer(self.mat) + + +class CUDAMatrix(object): + """ + A CUDAMatrix object represents a matrix of single precision floating point + numbers on a GPU. + """ + + def __init__(self, array, copy_to_device=True, copy_on_host=True): + """ + Initializes a new matrix object in one of two ways. If array is a numpy + ndarray, memory for a matrix with the same dimensions is allocated on + the GPU. If the copy_to_device flag is set to True, the GPU matrix is + initialized with the given ndarray. If the copy_on_host flag is set to + True, a copy of the matrix will be created in host memory even if the + matrix is of the correct type (float64, Fortran-contiguous order). + If array is not an ndarray, it must be a cudamat structure (typically + the user will never use this way of calling __init__). + """ + + if type(array) in [np.ndarray, np.memmap]: + # Convert array to float64 in FORTRAN order + array = reformat(array, copy=copy_on_host) + + # Initialize as a ndarray-tied matrix. + self.mat = cudamat() + self.size = self.mat.size + self.p_mat = ct.pointer(self.mat) + self.numpy_array = array + + _cudamat.init_from_array( + self.p_mat, + array.ctypes.data_as(ct.POINTER(ct.c_double)), + ct.c_int(array.shape[0]), + ct.c_int(array.shape[1])) + + if copy_to_device: + err_code = _cudamat.copy_to_device(self.p_mat) + if err_code: + raise generate_exception(err_code) + + else: + # Initialize based on existing cudamat structure. + mat = array + self.mat = mat + self.p_mat = ct.pointer(self.mat) + + self.T = TransposedCUDAMatrix(self.mat) + + # Keep a reference to free device memory in case of a crash. + self.__free_device_memory = _cudamat.free_device_memory + + def __del__(self): + try: + if 'p_mat' in self.__dict__: + err_code = self.__free_device_memory(self.p_mat) + if err_code: + raise generate_exception(err_code) + except AttributeError: + pass + + @staticmethod + def init_random(seed=0): + """ + Initialize and seed the random number generator. + """ + + CUDAMatrix.rndInitialized = 1 + CUDAMatrix.rnd_state = rnd_struct() + CUDAMatrix.rnd_state_p = ct.pointer(CUDAMatrix.rnd_state) + + cudamat_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), + 'rnd_multipliers_32bit.txt') + + if sys.version_info >= (3,): + cudamat_path = cudamat_path.encode(sys.getfilesystemencoding()) + + err_code = _cudamat.init_random(CUDAMatrix.rnd_state_p, + ct.c_int(seed), + cudamat_path) + if err_code: + if sys.version_info >= (3,): + cudamat_path = cudamat_path.decode(sys.getfilesystemencoding()) + raise generate_exception(err_code, filepath=cudamat_path) + + @property + def shape(self): + return (self.mat.size[0], self.mat.size[1]) + + def reshape(self, shape): + """ + Reshapes self to have the given shape. The number of elements cannot + change as this only changes how the contents are interpreted. + """ + + m = ct.c_uint(shape[0]) + n = ct.c_uint(shape[1]) + + # Reshape the default matrix + err_code = _cudamat.reshape(self.p_mat, m, n) + if err_code: + raise generate_exception(err_code) + # Reshape the transposed matrix + err_code = _cudamat.reshape(self.T.p_mat, m, n) + if err_code: + raise generate_exception(err_code) + # Reshape the CPU matrix + if self.mat.on_host: + self.numpy_array = np.reshape(self.numpy_array, shape, order='F') + + return self + + def asarray(self): + """ + Copies the matrix to an ndarray on the CPU and returns it. + """ + + self.copy_to_host() + + return self.numpy_array + + def copy_to_device(self): + """ + Copy the matrix to the GPU. + """ + + err_code = _cudamat.copy_to_device(self.p_mat) + if err_code: + raise generate_exception(err_code) + + def copy_to_host(self): + """ + Copy the matrix to the CPU. + """ + + if not self.mat.on_host: + # allocate host storage if necessary + m = self.mat.size[0] + n = self.mat.size[1] + + self.numpy_array = np.empty((m, n), dtype=np.float64, order='F') + self.mat.data_host = \ + self.numpy_array.ctypes.data_as(ct.POINTER(ct.c_double)) + + self.mat.on_host = 1 + + err_code = _cudamat.copy_to_host(self.p_mat) + if err_code: + raise generate_exception(err_code) + + def copy(self, include_host=False): + """ + Create a copy of the matrix on GPU. If include_host is True, also + creates a copy of the matrix on CPU if there was any. + """ + + new_mat = empty(self.shape).assign(self) + + if include_host and self.mat.on_host: + new_mat.numpy_array = self.numpy_array.copy() + new_mat.mat.data_host = \ + new_mat.numpy_array.ctypes.data_as(ct.POINTER(ct.c_double)) + new_mat.mat.on_host = 1 + + return new_mat + + def assign(self, val): + """Assign val to self, where val can be a scalar or a CUDAMatrix + with the same dimensions as self. """ + + if isinstance(val, CUDAMatrix): + err_code = _cudamat.copy_on_device(val.p_mat, self.p_mat) + elif isinstance(val, (int, float)): + err_code = _cudamat.assign_scalar(self.p_mat, ct.c_double(val)) + else: + raise ValueError("Assigned value must be of type" + "CUDAMatrix, int, or float.") + + if err_code: + raise generate_exception(err_code) + + return self + + def free_device_memory(self): + """ + Free memory used up by the matrix on the GPU. + """ + + err_code = _cudamat.free_device_memory(self.p_mat) + if err_code: + raise generate_exception(err_code) + + def set_trans(self, is_trans): + """ + Set the transposedness flag to is_trans. + """ + + _cudamat.set_transpose(self.p_mat, ct.c_int(1 * is_trans)) + + def slice(self, first_col, last_col, include_host=False): + """ + Creates a view into a consecutive range of columns of an existing + matrix on GPU. If include_host is set to True, also creates a view + into the CPU copy of the matrix (i.e., the numpy_array). + """ + mat = cudamat() + + if self.mat.size[0] == 1 or self.mat.size[1] == 1: + err_code = _cudamat.get_vector_slice(self.p_mat, + ct.pointer(mat), + ct.c_int(first_col), + ct.c_int(last_col)) + else: + err_code = _cudamat.get_slice(self.p_mat, + ct.pointer(mat), + ct.c_int(first_col), + ct.c_int(last_col)) + + if err_code: + raise generate_exception(err_code) + + new_mat = CUDAMatrix(mat) + + try: + new_mat.sliceof = self.sliceof + except: + new_mat.sliceof = self + + # reproduce the slice on the host as well (if requested) + if include_host and self.mat.on_host: + new_mat.numpy_array = self.numpy_array[:, first_col:last_col] + new_mat.mat.data_host = \ + new_mat.numpy_array.ctypes.data_as(ct.POINTER(ct.c_double)) + new_mat.mat.on_host = 1 + + return new_mat + + def get_col_slice(self, first_col, last_col, target=None): + """ + Get the columns with indices first_col through last_col. If a target + is provided, columns are copied into the target. Otherwise, returns a + view into the existing memory on GPU. + """ + col_slice = self.slice(first_col, last_col) + + if target: + target.assign(col_slice) + return target + else: + return col_slice + + def set_col_slice(self, first_col, last_col, mat): + """ + Assign the contents of mat to the columns with indices first_col + through last_col. + """ + self.slice(first_col, last_col).assign(mat) + + return self + + def get_row_slice(self, start, end, target=None): + """ + Get the rows with indices start through end. If target is not provided + memory for a new matrix will be allocated. + """ + + width = self.shape[1] + + if not target: + target = empty((end-start, width)) + + err_code = _cudamat.get_row_slice(self.p_mat, + target.p_mat, + ct.c_int(start), + ct.c_int(end)) + if err_code: + raise generate_exception(err_code) + + return target + + def set_row_slice(self, start, end, mat): + """ + Assign the contents of mat to the rows with indices start through end. + """ + + err_code = _cudamat.set_row_slice(mat.p_mat, self.p_mat, + ct.c_int(start), ct.c_int(end)) + if err_code: + raise generate_exception(err_code) + + return self + + def transpose(self, target=None): + """ + Return a transposed copy of the matrix. + """ + if not target: + target = empty((self.shape[1], self.shape[0])) + + err_code = _cudamat.copy_transpose(self.p_mat, target.p_mat) + if err_code: + raise generate_exception(err_code) + + return target + + def fill_with_rand(self): + """ + Fill matrix on the GPU with random numbers drawn from the uniform + distribution over the (0,1) interval. + """ + + err_code = _cudamat.fill_with_rand(CUDAMatrix.rnd_state_p, self.p_mat) + if err_code: + raise generate_exception(err_code) + + return self + + def fill_with_randn(self): + """ + Fill matrix on the GPU with random numbers drawn from + the standard normal distribution. + """ + + err_code = _cudamat.fill_with_randn(CUDAMatrix.rnd_state_p, self.p_mat) + if err_code: + raise generate_exception(err_code) + + return self + + def add_col_vec(self, vec, target=None): + """ + Add vector vec to every column of the matrix. If a target is provided, + it is used to store the result instead of self. + """ + + if not target: + target = self + + err_code = _cudamat.add_col_vec(self.p_mat, vec.p_mat, target.p_mat) + if err_code: + raise generate_exception(err_code) + + return target + + def add_col_mult(self, vec, mult, target=None): + """ + Add a multiple of vector vec to every column of the matrix. If a target + is provided, it is used to store the result instead of self. + """ + + if not target: + target = self + + err_code = _cudamat.add_col_mult(self.p_mat, vec.p_mat, + target.p_mat, ct.c_double(mult)) + if err_code: + raise generate_exception(err_code) + + return target + + def add_row_vec(self, vec, target=None): + """ + Add vector vec to every row of the matrix. If a target is provided, + it is used to store the result instead of self. + """ + + if not target: + target = self + + err_code = _cudamat.add_row_vec(self.p_mat, vec.p_mat, target.p_mat) + if err_code: + raise generate_exception(err_code) + + return target + + def mult_by_col(self, vec, target=None): + """ + Multiply vector vec into every column of the matrix. If a target is + provided, it is used to store the result instead of self. + """ + + if not target: + target = self + + err_code = _cudamat.mult_by_col_vec(self.p_mat, vec.p_mat, target.p_mat) + if err_code: + raise generate_exception(err_code) + + return target + + def mult_by_row(self, vec, target=None): + """ + Multiply vector vec into every row of the matrix. If a target is + provided, it is used to store the result instead of self. + """ + + if not target: + target = self + + err_code = _cudamat.mult_by_row_vec(self.p_mat, vec.p_mat, target.p_mat) + if err_code: + raise generate_exception(err_code) + + return target + + def div_by_col(self, vec, target=None): + """ + Divide every column of the matrix by vector vec. If a target is + provided, it is used to store the result instead of self. + """ + + if not target: + target = self + + err_code = _cudamat.divide_by_col_vec(self.p_mat, vec.p_mat, + target.p_mat) + if err_code: + raise generate_exception(err_code) + + return target + + def div_by_row(self, vec, target=None): + """ + Divide every row of the matrix by vector vec. If a target is + provided, it is used to store the result instead of self. + """ + + if not target: + target = self + + err_code = _cudamat.divide_by_row_vec(self.p_mat, vec.p_mat, + target.p_mat) + if err_code: + raise generate_exception(err_code) + + return target + + def sum(self, axis, target=None, mult=1.): + """ + Sum the matrix along the given dimension, where 0 represents the leading + dimension and 1 represents the non-leading dimension. If a target is + not provided, a new vector is created for storing the result. The result + is multiplied by the given factor mult (defaults to 1). + """ + + return sum(self, axis, target, mult) + + def mean(self, axis, target=None): + """ + Compute the mean of the matrix along the given dimension, where 0 + represents the leading dimension and 1 represents the non-leading + dimension. If a target is not provided, a new vector is created for + storing the result. + """ + + return mean(self, axis, target) + + def add_sums(self, mat, axis, mult=1., beta=1.): + """ + Add a multiple of the sums of the matrix mat along the given dimension + to self. Self is scaled by beta before adding anything. + """ + + m = _cudamat.get_leading_dimension(mat.p_mat) + n = _cudamat.get_nonleading_dimension(mat.p_mat) + + if axis == 0: + # sum along leading dimension + check_ones_matrix(m) + left = CUDAMatrix.ones.slice(0, m) + left.set_trans(True) + right = mat + + elif axis == 1: + # sum along non-leading dimension + left = mat + check_ones_matrix(n) + right = CUDAMatrix.ones.slice(0, n) + + err_code = _cudamat.dot(left.p_mat, right.p_mat, self.p_mat, + ct.c_double(beta), ct.c_double(mult)) + if err_code: + raise generate_exception(err_code) + + return self + + def less_than(self, val, target=None): + """ + Perform the operation target = 1. * (self < val), + where val can be a matrix or a scalar. + """ + + if not target: + target = self + + if isinstance(val, (int, float)): + err_code = _cudamat.less_than_scalar(self.p_mat, ct.c_double(val), + target.p_mat) + else: + err_code = _cudamat.less_than(self.p_mat, val.p_mat, target.p_mat) + + if err_code: + raise generate_exception(err_code) + + return target + + def greater_than(self, val, target=None): + """ + Perform the operation target = 1. * (self > val), + where val can be a matrix or a scalar. + """ + + if not target: + target = self + + if isinstance(val, (int, float)): + err_code = _cudamat.greater_than_scalar(self.p_mat, + ct.c_double(val), + target.p_mat) + else: + err_code = _cudamat.greater_than(self.p_mat, val.p_mat, + target.p_mat) + + if err_code: + raise generate_exception(err_code) + + return target + + def equals(self, val, target=None): + """ + Perform the operation target = 1. * (self == val), + where val can be a matrix or a scalar. + """ + + if not target: + target = self + + if isinstance(val, (int, float)): + err_code = _cudamat.equals_scalar(self.p_mat, ct.c_double(val), + target.p_mat) + else: + err_code = _cudamat.equals(self.p_mat, val.p_mat, target.p_mat) + + if err_code: + raise generate_exception(err_code) + + return target + + def minimum(self, val, target=None): + """ + Perform the element-wise operation target = min(self, val), where + val can be a matrix or a scalar. + """ + + if not target: + target = self + + if isinstance(val, (int, float)): + err_code = _cudamat.minimum_scalar(self.p_mat, ct.c_double(val), + target.p_mat) + else: + err_code = _cudamat.minimum(self.p_mat, val.p_mat, target.p_mat) + + if err_code: + raise generate_exception(err_code) + + return target + + def maximum(self, val, target=None): + """ + Perform the element-wise operation target = max(self, val), where + val can be a matrix or a scalar. + """ + + if not target: + target = self + + if isinstance(val, (int, float)): + err_code = _cudamat.maximum_scalar(self.p_mat, ct.c_double(val), + target.p_mat) + else: + err_code = _cudamat.maximum(self.p_mat, val.p_mat, target.p_mat) + + if err_code: + raise generate_exception(err_code) + + return target + + def min(self, axis, target=None): + """ + Find the minimum value along the given dimension, where 0 represents the + leading dimension and 1 represents the non-leading dimension. If a + target is not prvided, a new vector is created for storing the result. + """ + + m, n = self.shape + + if axis == 0: + if not target: + target = empty((1, n)) + + elif axis == 1: + if not target: + target = empty((m, 1)) + + err_code = _cudamat.min_by_axis(self.p_mat, target.p_mat, + ct.c_int(axis)) + if err_code: + raise generate_exception(err_code) + + return target + + def max(self, axis, target=None): + """ + Find the maximum value along the given dimension, where 0 represents the + leading dimension and 1 represents the non-leading dimension. If a + target is not prvided, a new vector is created for storing the result. + """ + + m, n = self.shape + + if axis == 0: + if not target: + target = empty((1, n)) + + elif axis == 1: + if not target: + target = empty((m, 1)) + + err_code = _cudamat.max_by_axis(self.p_mat, target.p_mat, + ct.c_int(axis)) + if err_code: + raise generate_exception(err_code) + + return target + + def argmin(self, axis, target=None): + """ + Find the index of the minimum value along the given dimension, where 0 + represents the leading dimension and 1 represents the non-leading + dimension. If a target is not provided, a new vector is created for + storing the result. + """ + + m, n = self.shape + + if axis == 0: + if not target: + target = empty((1, n)) + + elif axis == 1: + if not target: + target = empty((m, 1)) + + err_code = _cudamat.argmin_by_axis(self.p_mat, target.p_mat, + ct.c_int(axis)) + if err_code: + raise generate_exception(err_code) + + return target + + def argmax(self, axis, target=None): + """ + Find the index of the maximum value along the given dimension, where 0 + represents the leading dimension and 1 represents the non-leading + dimension. If a target is not provided, a new vector is created for + storing the result. + """ + + m, n = self.shape + + if axis == 0: + if not target: + target = empty((1, n)) + + elif axis == 1: + if not target: + target = empty((m, 1)) + + err_code = _cudamat.argmax_by_axis(self.p_mat, target.p_mat, + ct.c_int(axis)) + if err_code: + raise generate_exception(err_code) + + return target + + def sign(self, target=None): + """ + Find the sign of each element of the matrix. + """ + + if not target: + target = empty((self.mat.size[0], self.mat.size[1])) + + err_code = _cudamat.sign(self.p_mat, target.p_mat) + if err_code: + raise generate_exception(err_code) + + return target + + def apply_sigmoid(self, target=None): + """ + Apply the logistic sigmoid to each element of the matrix. + """ + + return sigmoid(self, target) + + def apply_tanh(self, target=None): + """ + Apply the tanh to each element of the matrix. + """ + + return tanh(self, target) + + def apply_soft_threshold(self, alpha, target=None): + """ + Apply the soft threshold function to each element of the matrix: + + x = sign(x) * max(0, abs(x) - alpha) + """ + + return soft_threshold(self, alpha, target) + + def reciprocal(self, target=None): + """ + Find the reciprocal of each element of the matrix. + """ + + if not target: + target = self + + err_code = _cudamat.reciprocal(self.p_mat, target.p_mat) + if err_code: + raise generate_exception(err_code) + + return target + + def dot(self, mat2, target=None): + """ + Multiply the matrix by mat2 from the right. + """ + + return dot(self, mat2, target) + + def add_dot(self, m1, m2, mult=1., beta=1.): + """ + Add the dot product of m1 and m2 to the matrix, scaled by mult. + Self is scaled by beta before adding anything. + """ + + err_code = _cudamat.dot(m1.p_mat, m2.p_mat, self.p_mat, + ct.c_double(beta), ct.c_double(mult)) + if err_code: + raise generate_exception(err_code) + + return self + + def subtract_dot(self, m1, m2, mult=1., beta=1.): + """ + Subtract the dot product of m1 and m2 from the matrix, scaled by mult. + Self is scaled by beta before subtracting anything. + """ + + return self.add_dot(m1, m2, mult=-1. * mult, beta=beta) + + def add_mult(self, mat2, alpha=1.): + """ + Add multiple of mat2 to the matrix. + """ + + err_code = _cudamat.add_mult(self.p_mat, mat2.p_mat, ct.c_double(alpha)) + if err_code: + raise generate_exception(err_code) + + return self + + def subtract_mult(self, mat2, alpha=1.): + """ + Subtract a multiple of mat2 from the matrix. + """ + + err_code = _cudamat.add_mult(self.p_mat, mat2.p_mat, + ct.c_double(-1. * alpha)) + if err_code: + raise generate_exception(err_code) + + return self + + def add(self, val, target=None): + """Add val to self, where val can be a scalar or a CUDAMatrix with the + same dimensions as self. """ + + if not target: + target = self + + if isinstance(val, CUDAMatrix): + err_code = _cudamat.add_elementwise(self.p_mat, val.p_mat, + target.p_mat) + elif isinstance(val, (int, float)): + err_code = _cudamat.add_scalar(self.p_mat, ct.c_double(val), + target.p_mat) + else: + raise ValueError("Value must be of type CUDAMatrix, int, or float.") + + if err_code: + raise generate_exception(err_code) + + return target + + def subtract(self, val, target=None): + """Subtract val from self, where val can be a scalar or a CUDAMatrix with + the same dimensions as self. """ + + if not target: + target = self + + if isinstance(val, CUDAMatrix): + err_code = _cudamat.subtract_elementwise(self.p_mat, val.p_mat, + target.p_mat) + elif isinstance(val, (int, float)): + err_code = _cudamat.add_scalar(self.p_mat, ct.c_double(-1*val), + target.p_mat) + else: + raise ValueError("Value must be of type CUDAMatrix, int, or float.") + + if err_code: + raise generate_exception(err_code) + + return target + + def divide(self, val, target=None): + """Divide self by val, where val can be a scalar or + a CUDAMatrix with the same dimensions as self. """ + + if not target: + target = self + + if isinstance(val, CUDAMatrix): + err_code = _cudamat.divide_elementwise(self.p_mat, val.p_mat, + target.p_mat) + elif isinstance(val, (int, float)): + err_code = _cudamat.divide_by_scalar(self.p_mat, ct.c_double(val), + target.p_mat) + else: + raise ValueError("Value must be of type CUDAMatrix, int, or float.") + + if err_code: + raise generate_exception(err_code) + + return target + + def mult(self, val, target=None): + """Multiply self by val, where val can be a scalar or a CUDAMatrix with + the same dimensions as self. """ + + if not target: + target = self + + if isinstance(val, CUDAMatrix): + err_code = _cudamat.mult_elementwise(self.p_mat, val.p_mat, + target.p_mat) + elif isinstance(val, (int, float)): + err_code = _cudamat.mult_by_scalar(self.p_mat, ct.c_double(val), + target.p_mat) + else: + raise ValueError("Value must be of type CUDAMatrix, int, or float.") + + if err_code: + raise generate_exception(err_code) + + return target + + @deprecated + def assign_scalar(self, alpha): + """ + Assign scalar alpha to every element of the matrix. + """ + + err_code = _cudamat.assign_scalar(self.p_mat, ct.c_double(alpha)) + if err_code: + raise generate_exception(err_code) + + return self + + @deprecated + def mult_by_scalar(self, alpha, target=None): + """ + Multiply the matrix by a scalar. + """ + + if not target: + target = self + + err_code = _cudamat.mult_by_scalar(self.p_mat, ct.c_double(alpha), + target.p_mat) + if err_code: + raise generate_exception(err_code) + + return target + + @deprecated + def div_by_scalar(self, alpha, target=None): + """ + Divide the matrix by a scalar. + """ + + if not target: + target = self + + err_code = _cudamat.divide_by_scalar(self.p_mat, ct.c_double(alpha), + target.p_mat) + if err_code: + raise generate_exception(err_code) + + return target + + @deprecated + def add_scalar(self, alpha, target=None): + """ + Increment the matrix by a scalar. + """ + + if not target: + target = self + + err_code = _cudamat.add_scalar(self.p_mat, ct.c_double(alpha), + target.p_mat) + if err_code: + raise generate_exception(err_code) + + return target + + def euclid_norm(self): + """ + Returns the L2 norm of the matrix flattened to a vector. + """ + err_code = ct.c_int(0) + res = _cudamat.euclid_norm(self.p_mat, ct.byref(err_code)) + + if err_code: + raise generate_exception(err_code.value) + + return res + + def manhattan_norm(self): + """ + Returns the L1 norm of the matrix flattened to a vector. + """ + err_code = ct.c_int(0) + res = _cudamat.manhattan_norm(self.p_mat, ct.byref(err_code)) + + if err_code: + raise generate_exception(err_code.value) + + return res + + def allfinite(self): + """ + Checks if all entries in this matrix are finite, i.e., there is no + NaN and no positive or negative infinity. + """ + # Caveat: For a very large matrix of very large finite numbers, the + # manhattan norm may overflow and allfinite() may return False. + return np.isfinite(self.manhattan_norm()) + + def select_columns(self, indices, target): + """ + Copies some columns of self into target. + must be a row vector. Its elements are float64's representing + integers, e.g. "34.0" means the integer "34". + After this call, for all r,c, target[r,c]=self[r,indices[c]]. + This returns target. + Negative indices are interpreted in the usual Python way: all + elements of had better be in the range + [-self.shape[1], self.shape[1]-1]. + This does bounds checking, but out of bounds indices do not raise an + exception (because the programmer was lazy). Instead, they result + in NaN values in . + """ + + err_code = _cudamat.selectRows(self.p_mat, target.p_mat, indices.p_mat) + + if err_code: + raise generate_exception(err_code) + + return target + + def set_selected_columns(self, indices, source): + """ + copies all columns of source into some columns of self. + must be a row vector. Its elements are float64's representing + integers, e.g. "34.0" means the integer "34". after this call, for all + r,c, self[r,indices[c]]=source[r,c]. This returns self. + Negative indices are interpreted in the usual Python way: all elements + of had better be in the range + [-self.shape[1], self.shape[1]-1]. + This does bounds checking, but out of bounds indices do not raise an + exception (because the programmer was lazy). Instead, they result in NaN + values in . + """ + + err_code = _cudamat.setSelectedRows(self.p_mat, source.p_mat, + indices.p_mat) + if err_code: + raise generate_exception(err_code) + + return self + + +def empty(shape): + """ + Creates and returns a new CUDAMatrix with the given shape. + """ + + mat = cudamat() + err_code = _cudamat.init_empty(ct.pointer(mat), ct.c_int(shape[0]), + ct.c_int(shape[1])) + + if err_code: + raise generate_exception(err_code) + + return CUDAMatrix(mat) + + +def check_ones_matrix(min_size): + if min_size > CUDAMatrix.ones.shape[0]: + raise CUDAMatException( + 'Not enough memory allocated for reduction. ' + '({} needed, {} actual), use cudamat.init() ' + 'to allocate more'.format(min_size, CUDAMatrix.ones.shape[0])) + + +def sum(mat, axis, target=None, mult=1.): + """ + Sum the matrix along the given dimension, where 0 represents the leading + dimension and 1 represents the non-leading dimension. If a target is + not provided, a new vector is created for storing the result. The result + is multiplied by the given factor mult (defaults to 1). + """ + + m = _cudamat.get_leading_dimension(mat.p_mat) + n = _cudamat.get_nonleading_dimension(mat.p_mat) + + if axis == 0: + # sum along leading dimension + check_ones_matrix(m) + left = CUDAMatrix.ones.slice(0, m) + left.set_trans(True) + right = mat + + if not target: + target = empty((1, n)) + + elif axis == 1: + # sum along non-leading dimension + left = mat + check_ones_matrix(n) + right = CUDAMatrix.ones.slice(0, n) + + if not target: + target = empty((m, 1)) + + err_code = _cudamat.dot(left.p_mat, right.p_mat, target.p_mat, + ct.c_double(0.), ct.c_double(mult)) + if err_code: + raise generate_exception(err_code) + + return target + + +def mean(mat, axis, target=None): + """ + Compute the mean of the matrix along the given dimension, where 0 represents + the leading dimension and 1 represents the non-leading dimension. If a + target is not provided, a new vector is created for storing the result. + """ + + return sum(mat, axis, target=target, mult=1. / mat.shape[axis]) + + +def dot(m1, m2, target=None, beta=0., alpha=1.): + """ + Find the dot product between m1 and m2 and store in target: + target = beta*target + alpha*(m1 m2) + If no target is given, it will be created automatically, but not + initialized -- so beta should be left at its default value zero. + """ + + if not target: + m = _cudamat.get_leading_dimension(m1.p_mat) + n = _cudamat.get_nonleading_dimension(m2.p_mat) + + target = empty((m, n)) + + err_code = _cudamat.dot(m1.p_mat, m2.p_mat, + target.p_mat, ct.c_double(beta), + ct.c_double(alpha)) + if err_code: + raise generate_exception(err_code) + + return target + + +def vdot(m1, m2): + """ + Compute the vector dot product of matrices m1 and m2. + """ + + err_code = ct.c_int(0) + res = _cudamat.vdot(m1.p_mat, m2.p_mat, ct.byref(err_code)) + + if err_code: + raise generate_exception(err_code.value) + + return res + + +def sigmoid(mat, target=None): + """ + Apply the logistic sigmoid to each element of the matrix mat. + """ + + if not target: + target = mat + + err_code = _cudamat.apply_sigmoid(mat.p_mat, target.p_mat) + if err_code: + raise generate_exception(err_code) + + return target + + +def tanh(mat, target=None): + """ + Apply the tanh to each element of the matrix mat. + """ + + if not target: + target = mat + + err_code = _cudamat.apply_tanh(mat.p_mat, target.p_mat) + if err_code: + raise generate_exception(err_code) + + return target + + +def soft_threshold(mat, alpha, target=None): + """ + Apply the soft threshold function to each element of the matrix: + + mat = sign(mat) * max(0, abs(mat) - alpha) + """ + + if not target: + target = mat + + err_code = _cudamat.apply_soft_threshold(mat.p_mat, ct.c_double(alpha), + target.p_mat) + if err_code: + raise generate_exception(err_code) + + return target + + +def abs(mat, target=None): + """ + Apply abs to each element of the matrix mat. + """ + + if not target: + target = mat + + err_code = _cudamat.apply_abs(mat.p_mat, target.p_mat) + if err_code: + raise generate_exception(err_code) + + return target + + +def log_1_plus_exp(mat, target=None): + """ + Apply log(1+exp(x)) to each element of the matrix mat. + """ + + if not target: + target = mat + + err_code = _cudamat.apply_log_1_plus_exp(mat.p_mat, target.p_mat) + if err_code: + raise generate_exception(err_code) + + return target + + +def log(mat, target=None): + """ + Find the natural logarithm of each element of the matrix mat. + """ + + if not target: + target = mat + + err_code = _cudamat.apply_log(mat.p_mat, target.p_mat) + if err_code: + raise generate_exception(err_code) + + return target + + +def exp(mat, target=None): + """ + Apply the exponential function to each element of the matrix mat. + """ + + if not target: + target = mat + + err_code = _cudamat.apply_exp(mat.p_mat, target.p_mat) + if err_code: + raise generate_exception(err_code) + + return target + + +def gamma(mat, target=None): + """ + Apply the gamma function to each element of the matrix mat. + """ + + if not target: + target = mat + + err_code = _cudamat.apply_gamma(mat.p_mat, target.p_mat) + if err_code: + raise generate_exception(err_code) + + return target + + +def lgamma(mat, target=None): + """ + Apply the log gamma function to each element of the matrix mat. + """ + + if not target: + target = mat + + err_code = _cudamat.apply_lgamma(mat.p_mat, target.p_mat) + if err_code: + raise generate_exception(err_code) + + return target + + +def sqrt(mat, target=None): + """ + Compute the square root of each element of the matrix mat. + """ + + if not target: + target = mat + + err_code = _cudamat.apply_sqrt(mat.p_mat, target.p_mat) + if err_code: + raise generate_exception(err_code) + + return target + + +def pow(mat, p, target=None): + """ + If p is a scalar, compute the 'p'th power of each element of the matrix mat, + otherwise raise each element of the matrix mat to the power given by the + corresponding element of the matrix p. + """ + + if not target: + target = mat + + if isinstance(p, CUDAMatrix): + err_code = _cudamat.apply_pow_matrix(mat.p_mat, p.p_mat, target.p_mat) + elif isinstance(p, (int, float)): + err_code = _cudamat.apply_pow(mat.p_mat, ct.c_double(p), target.p_mat) + else: + raise ValueError("Value must be of type CUDAMatrix, int, or float.") + + if err_code: + raise generate_exception(err_code) + + return target + + +def where(condition_mat, if_mat, else_mat, target=None): + """ + For each element i, j, store if_math[i, j] in target[i,j] if + condition_mat[i, j] is True, and else_mat[i, j] otherwise. + """ + if not target: + target = condition_mat + + err_code = _cudamat.where(condition_mat.p_mat, if_mat.p_mat, + else_mat.p_mat, target.p_mat) + if err_code: + raise generate_exception(err_code) + + return target + + +def correlate(mat, kernel, target=None): + """ + Cross-correlate a matrix with a kernel matrix. + The kernel matrix is centered over each element of the matrix mat. + Width and height of the kernel matrix must be an odd integer. + If a target is not provided, a new matrix is created for storing the result. + Note that this function cannot operate in-place. + """ + if not target: + m = _cudamat.get_leading_dimension(mat.p_mat) + n = _cudamat.get_nonleading_dimension(mat.p_mat) + + target = empty((m, n)) + + err_code = _cudamat.correlate(mat.p_mat, kernel.p_mat, target.p_mat) + if err_code: + raise generate_exception(err_code) + + return target + + +def cuda_sync_threads(): + _cudamat.cuda_sync_threads() + + +def reformat(array, copy=True): + """ + Returns array as a float64 array in FORTRAN order. + If copy is set to False, the array will only be copied if it is not already + in the correct format. + """ + return np.array(array, dtype=np.float64, order='F', copy=copy) + + +def cuda_set_device(dev_id): + """ + Selects the CUDA device with the given ID. + """ + + err_code = _cudamat.cuda_set_device(ct.c_int(dev_id)) + if err_code: + raise generate_exception(err_code) + + +def cublas_init(max_ones=(1024*256)): + """ + Initialize Cublas. + + 'max_ones' is an optional argument that determines the length of + the largest sum that can be computed using Cublas matrix multiply. + A larger value causes more memory to be allocated for this purpose. + """ + + err = _cudamat.cublas_init() + if err: + raise CUDAMatException('error initializing CUBLAS: (err=%u)' % err) + CUDAMatrix.ones = empty((max_ones, 1)).assign(1.0) + +init = cublas_init + + +def cublas_shutdown(): + """ + Shut down Cublas. + """ + + CUDAMatrix.ones = 0 + _cudamat.cublas_shutdown() + +shutdown = cublas_shutdown diff --git a/ot/gpu/cudamat/cudamat/cudamat_kernels.cu b/ot/gpu/cudamat/cudamat/cudamat_kernels.cu new file mode 100644 index 0000000..707b387 --- /dev/null +++ b/ot/gpu/cudamat/cudamat/cudamat_kernels.cu @@ -0,0 +1,804 @@ +#include "cudamat_kernels.cuh" +#include "float.h" + +/* ------------------------- Random number generation ------------------------- */ + +__global__ void kSeedRandom(unsigned int* rndMults, unsigned long long* rndWords, unsigned int seed) { + const unsigned int idx = blockIdx.x * blockDim.x + threadIdx.x; + + // The initial x is the seed and the initial carry is 1 + unsigned long long rndWord = ((unsigned long long)seed << 32) + 1; + const unsigned int rndMult = rndMults[idx]; + /* + * Run the chain for a few steps so that all the streams have a chance + * to differentiate. They start out generating similar random numbers + * because all the multipliers are similar. + */ + for(unsigned int i = 0; i < NUM_RND_BURNIN; i++) { + rndWord = rndMult * LOW_BITS(rndWord) + HIGH_BITS(rndWord); + } + rndWords[idx] = rndWord; +} + +__global__ void kRandomUniform(unsigned int* rndMults, unsigned long long* rndWords, double* gData, unsigned int numElements) { + const unsigned int idx = blockIdx.x * blockDim.x + threadIdx.x; + unsigned long long rndWord = rndWords[idx]; + const unsigned int rndMult = rndMults[idx]; + + for(unsigned int i = idx; i < numElements; i += NUM_RND_STREAMS) { + rndWord = rndMult * LOW_BITS(rndWord) + HIGH_BITS(rndWord); + gData[i] = (__uint2double_rn(LOW_BITS(rndWord)) + 1.0f) / 4294967296.0f; + } + rndWords[idx] = rndWord; +} + +__global__ void kRandomGaussian(unsigned int* rndMults, unsigned long long* rndWords, double* gData, unsigned int numElements) { + const unsigned int idx = blockIdx.x * blockDim.x + threadIdx.x; + unsigned long long rndWord = rndWords[idx]; + const unsigned int rndMult = rndMults[idx]; + + double rnd1, rnd2, R, T; + for(unsigned int i = idx; i < numElements; i += 2*NUM_RND_STREAMS) { + rndWord = rndMult * LOW_BITS(rndWord) + HIGH_BITS(rndWord); + rnd1 = (__uint2double_rn(LOW_BITS(rndWord)) + 1.0f) / 4294967296.0f; + rndWord = rndMult * LOW_BITS(rndWord) + HIGH_BITS(rndWord); + rnd2 = (__uint2double_rn(LOW_BITS(rndWord)) + 1.0f) / 4294967296.0f; + T = 2 * PI * rnd2; + R = sqrtf(-2 * __logf(rnd1)); + gData[i] = R * __cosf(T); + if (i + NUM_RND_STREAMS < numElements) + gData[i + NUM_RND_STREAMS] = R * __sinf(T); + } + rndWords[idx] = rndWord; +} + +/* ------------------------- Data copying ------------------------- */ + +/* +Copy row slice from source to target. There is a block for every 32x32 chunk being copied. +*/ +__global__ void kGetRowSlice(double* source, double* target, int start, int end, int width, int height) { + const int row = start + blockIdx.x * 32 + threadIdx.x; + const int start_col = blockIdx.y * 32; + + const int end_col = (start_col + 32 < width) ? start_col + 32: width; + + const int target_height = end - start; + + if (row < end) { + for (int cur_col = start_col; cur_col < end_col; cur_col++) + target[cur_col * target_height + row - start] = source[cur_col * height + row]; + } +} + +__global__ void kSetRowSlice(double* source, double* target, int start, int end, int width, int height) { + const int row = start + blockIdx.x * 32 + threadIdx.x; + const int start_col = blockIdx.y * 32; + + const int end_col = (start_col + 32 < width) ? start_col + 32: width; + + const int source_height = end - start; + + if (row < end) { + for (int cur_col = start_col; cur_col < end_col; cur_col++) + target[cur_col * height + row] = source[cur_col * source_height + row - start]; + //source[cur_col * height + row - start] = target[cur_col * target_height + row]; + } +} + +__global__ void kTranspose(double *odata, double *idata, int width, int height) { + __shared__ double block[COPY_BLOCK_SIZE][COPY_BLOCK_SIZE+1]; + + // read the matrix tile into shared memory + unsigned int xIndex = blockIdx.x * COPY_BLOCK_SIZE + threadIdx.x; + unsigned int yIndex = blockIdx.y * COPY_BLOCK_SIZE + threadIdx.y; + + if((xIndex < width) && (yIndex < height)) { + unsigned int index_in = yIndex * width + xIndex; + + block[threadIdx.y][threadIdx.x] = idata[index_in]; + } + + __syncthreads(); + + // write the transposed matrix tile to global memory + xIndex = blockIdx.y * COPY_BLOCK_SIZE + threadIdx.x; + yIndex = blockIdx.x * COPY_BLOCK_SIZE + threadIdx.y; + + if((xIndex < height) && (yIndex < width)) { + unsigned int index_out = yIndex * height + xIndex; + + odata[index_out] = block[threadIdx.x][threadIdx.y]; + } +} + +/* ------------------------- Mathematical operations ------------------------- */ + +__global__ void kLessThan(double* mat1, double* mat2, double* target, unsigned int len) { + const unsigned int idx = blockIdx.x * blockDim.x + threadIdx.x; + const unsigned int numThreads = blockDim.x * gridDim.x; + + for (unsigned int i = idx; i < len; i += numThreads) { + target[i] = mat1[i] < mat2[i]; + } +} + +__global__ void kLessThanScalar(double* mat, double val, double* target, unsigned int len) { + const unsigned int idx = blockIdx.x * blockDim.x + threadIdx.x; + const unsigned int numThreads = blockDim.x * gridDim.x; + + for (unsigned int i = idx; i < len; i += numThreads) { + target[i] = mat[i] < val; + } +} + +__global__ void kGreaterThan(double* mat1, double* mat2, double* target, unsigned int len) { + const unsigned int idx = blockIdx.x * blockDim.x + threadIdx.x; + const unsigned int numThreads = blockDim.x * gridDim.x; + + for (unsigned int i = idx; i < len; i += numThreads) { + target[i] = mat1[i] > mat2[i]; + } +} + +__global__ void kGreaterThanScalar(double* mat, double val, double* target, unsigned int len) { + const unsigned int idx = blockIdx.x * blockDim.x + threadIdx.x; + const unsigned int numThreads = blockDim.x * gridDim.x; + + for (unsigned int i = idx; i < len; i += numThreads) { + target[i] = mat[i] > val; + } +} + +__global__ void kEquals(double* mat1, double* mat2, double* target, unsigned int len) { + const unsigned int idx = blockIdx.x * blockDim.x + threadIdx.x; + const unsigned int numThreads = blockDim.x * gridDim.x; + + for (unsigned int i = idx; i < len; i += numThreads) { + target[i] = mat1[i] == mat2[i]; + } +} + +__global__ void kEqualsScalar(double* mat, double val, double* target, unsigned int len) { + const unsigned int idx = blockIdx.x * blockDim.x + threadIdx.x; + const unsigned int numThreads = blockDim.x * gridDim.x; + + for (unsigned int i = idx; i < len; i += numThreads) { + target[i] = mat[i] == val; + } +} + +__global__ void kMinimum(double* mat1, double* mat2, double* target, unsigned int len) { + const unsigned int idx = blockIdx.x * blockDim.x + threadIdx.x; + const unsigned int numThreads = blockDim.x * gridDim.x; + + for (unsigned int i = idx; i < len; i += numThreads) { + target[i] = fminf(mat1[i], mat2[i]); + } +} + +__global__ void kMinimumScalar(double* mat, double val, double* target, unsigned int len) { + const unsigned int idx = blockIdx.x * blockDim.x + threadIdx.x; + const unsigned int numThreads = blockDim.x * gridDim.x; + + for (unsigned int i = idx; i < len; i += numThreads) { + target[i] = fminf(mat[i], val); + } +} + +__global__ void kMaximum(double* mat1, double* mat2, double* target, unsigned int len) { + const unsigned int idx = blockIdx.x * blockDim.x + threadIdx.x; + const unsigned int numThreads = blockDim.x * gridDim.x; + + for (unsigned int i = idx; i < len; i += numThreads) { + target[i] = fmaxf(mat1[i], mat2[i]); + } +} + +__global__ void kMaximumScalar(double* mat, double val, double* target, unsigned int len) { + const unsigned int idx = blockIdx.x * blockDim.x + threadIdx.x; + const unsigned int numThreads = blockDim.x * gridDim.x; + + for (unsigned int i = idx; i < len; i += numThreads) { + target[i] = fmaxf(mat[i], val); + } +} + +__global__ void kMinColumnwise(double* mat, double* target, unsigned int width, unsigned int height) { + __shared__ double min_vals[32]; + double cur_min = FLT_MAX; + double val = 0; + + for (unsigned int i = threadIdx.x; i < height; i += 32) { + val = mat[blockIdx.x * height + i]; + + if (val < cur_min) + cur_min = val; + } + + min_vals[threadIdx.x] = cur_min; + + __syncthreads(); + + if (threadIdx.x == 0) { + cur_min = FLT_MAX; + + for (unsigned int i = 0; i < 32; i++) + if (min_vals[i] < cur_min) + cur_min = min_vals[i]; + + target[blockIdx.x] = cur_min; + } +} + +__global__ void kMinRowwise(double* mat, double* target, unsigned int width, unsigned int height) { + __shared__ double min_vals[32]; + double cur_min = FLT_MAX; + double val = 0; + + for (unsigned int i = threadIdx.x; i < width; i += 32) { + val = mat[i * height + blockIdx.x]; + + if (val < cur_min) + cur_min = val; + } + + min_vals[threadIdx.x] = cur_min; + + __syncthreads(); + + if (threadIdx.x == 0) { + cur_min = FLT_MAX; + + for (unsigned int i = 0; i < 32; i++) + if (min_vals[i] < cur_min) + cur_min = min_vals[i]; + + target[blockIdx.x] = cur_min; + } +} + +__global__ void kMaxColumnwise(double* mat, double* target, unsigned int width, unsigned int height) { + __shared__ double max_vals[32]; + double cur_max = -FLT_MAX; + double val = 0; + + for (unsigned int i = threadIdx.x; i < height; i += 32) { + val = mat[blockIdx.x * height + i]; + + if (val > cur_max) + cur_max = val; + } + + max_vals[threadIdx.x] = cur_max; + + __syncthreads(); + + if (threadIdx.x == 0) { + cur_max = -FLT_MAX; + + for (unsigned int i = 0; i < 32; i++) + if (max_vals[i] > cur_max) + cur_max = max_vals[i]; + + target[blockIdx.x] = cur_max; + } +} + +__global__ void kMaxRowwise(double* mat, double* target, unsigned int width, unsigned int height) { + __shared__ double max_vals[32]; + double cur_max = -FLT_MAX; + double val = 0; + + for (unsigned int i = threadIdx.x; i < width; i += 32) { + val = mat[i * height + blockIdx.x]; + + if (val > cur_max) + cur_max = val; + } + + max_vals[threadIdx.x] = cur_max; + + __syncthreads(); + + if (threadIdx.x == 0) { + cur_max = -FLT_MAX; + + for (unsigned int i = 0; i < 32; i++) + if (max_vals[i] > cur_max) + cur_max = max_vals[i]; + + target[blockIdx.x] = cur_max; + } +} + +__global__ void kArgMinColumnwise(double* mat, double* target, unsigned int width, unsigned int height) { + __shared__ double min_vals[32]; + __shared__ unsigned int min_args[32]; + double cur_min = FLT_MAX; + unsigned int cur_arg = 0; + double val = 0; + + for (unsigned int i = threadIdx.x; i < height; i += 32) { + val = mat[blockIdx.x * height + i]; + + if (val < cur_min) { + cur_min = val; + cur_arg = i; + } + } + + min_vals[threadIdx.x] = cur_min; + min_args[threadIdx.x] = cur_arg; + + __syncthreads(); + + if (threadIdx.x == 0) { + cur_min = FLT_MAX; + cur_arg = 0; + + for (unsigned int i = 0; i < 32; i++) + if (min_vals[i] < cur_min) { + cur_min = min_vals[i]; + cur_arg = min_args[i]; + } + + target[blockIdx.x] = cur_arg; + } +} + +__global__ void kArgMinRowwise(double* mat, double* target, unsigned int width, unsigned int height) { + __shared__ double min_vals[32]; + __shared__ unsigned int min_args[32]; + double cur_min = FLT_MAX; + unsigned int cur_arg = 0; + double val = 0; + + for (unsigned int i = threadIdx.x; i < width; i += 32) { + val = mat[i * height + blockIdx.x]; + + if (val < cur_min) { + cur_min = val; + cur_arg = i; + } + } + + min_vals[threadIdx.x] = cur_min; + min_args[threadIdx.x] = cur_arg; + + __syncthreads(); + + if (threadIdx.x == 0) { + cur_min = FLT_MAX; + cur_arg = 0; + + for (unsigned int i = 0; i < 32; i++) + if (min_vals[i] < cur_min) { + cur_min = min_vals[i]; + cur_arg = min_args[i]; + } + + target[blockIdx.x] = cur_arg; + } +} + +__global__ void kArgMaxColumnwise(double* mat, double* target, unsigned int width, unsigned int height) { + __shared__ double max_vals[32]; + __shared__ unsigned int max_args[32]; + double cur_max = -FLT_MAX; + unsigned int cur_arg = 0; + double val = 0; + + for (unsigned int i = threadIdx.x; i < height; i += 32) { + val = mat[blockIdx.x * height + i]; + + if (val > cur_max) { + cur_max = val; + cur_arg = i; + } + } + + max_vals[threadIdx.x] = cur_max; + max_args[threadIdx.x] = cur_arg; + + __syncthreads(); + + if (threadIdx.x == 0) { + cur_max = -FLT_MAX; + cur_arg = 0; + + for (unsigned int i = 0; i < 32; i++) + if (max_vals[i] > cur_max) { + cur_max = max_vals[i]; + cur_arg = max_args[i]; + } + + target[blockIdx.x] = cur_arg; + } +} + +__global__ void kArgMaxRowwise(double* mat, double* target, unsigned int width, unsigned int height) { + __shared__ double max_vals[32]; + __shared__ unsigned int max_args[32]; + double cur_max = -FLT_MAX; + unsigned int cur_arg = 0; + double val = 0; + + for (unsigned int i = threadIdx.x; i < width; i += 32) { + val = mat[i * height + blockIdx.x]; + + if (val > cur_max) { + cur_max = val; + cur_arg = i; + } + } + + max_vals[threadIdx.x] = cur_max; + max_args[threadIdx.x] = cur_arg; + + __syncthreads(); + + if (threadIdx.x == 0) { + cur_max = -FLT_MAX; + cur_arg = 0; + + for (unsigned int i = 0; i < 32; i++) + if (max_vals[i] > cur_max) { + cur_max = max_vals[i]; + cur_arg = max_args[i]; + } + + target[blockIdx.x] = cur_arg; + } +} + +__global__ void kSign(double* mat, double* target, unsigned int len) { + const unsigned int idx = blockIdx.x * blockDim.x + threadIdx.x; + const unsigned int numThreads = blockDim.x * gridDim.x; + + for (unsigned int i = idx; i < len; i += numThreads) { + target[i] = mat[i] ? copysignf(1., mat[i]) : 0.; + } +} + +__global__ void kApplySigmoid(double* mat, double* target, unsigned int len) { + const unsigned int idx = blockIdx.x * blockDim.x + threadIdx.x; + const unsigned int numThreads = blockDim.x * gridDim.x; + + for (unsigned int i = idx; i < len; i += numThreads) { + target[i] = 1 / (1 + __expf(-mat[i])); + } +} + + +__global__ void kApplyTanh(double* mat, double* target, unsigned int len) { + const unsigned int idx = blockIdx.x * blockDim.x + threadIdx.x; + const unsigned int numThreads = blockDim.x * gridDim.x; + double mat_i, exp2x; + + for (unsigned int i = idx; i < len; i += numThreads) { + mat_i = mat[i]; + exp2x = __expf(2 * mat_i); + target[i] = 1 - 2 / (exp2x + 1); + } +} + +__global__ void kApplySoftThreshold(double* mat, double alpha, double* target, unsigned int len) { + const unsigned int idx = blockIdx.x * blockDim.x + threadIdx.x; + const unsigned int numThreads = blockDim.x * gridDim.x; + + for (unsigned int i = idx; i < len; i += numThreads) { + double f = mat[i]; + target[i] = f > 0 ? max(0., f - alpha) : min(0., f + alpha); + } +} + +__global__ void kApplyAbs(double* mat, double* target, unsigned int len) { + const unsigned int idx = blockIdx.x * blockDim.x + threadIdx.x; + const unsigned int numThreads = blockDim.x * gridDim.x; + + for (unsigned int i = idx; i < len; i += numThreads) { + target[i] = mat[i] * ((mat[i] > 0) - (mat[i] < 0)); + } +} + +__global__ void kApplyLog1PlusExp(double* mat, double* target, unsigned int len) { + const unsigned int idx = blockIdx.x * blockDim.x + threadIdx.x; + const unsigned int numThreads = blockDim.x * gridDim.x; + double mat_i; + + for (unsigned int i = idx; i < len; i += numThreads) { + mat_i = mat[i]; + if (mat_i > 0) + target[i] = (__logf(1 + __expf(-mat_i)) + mat_i); + else + target[i] = __logf(1 + __expf(mat_i)); + } +} + +__global__ void kLog(double* mat, double* target, unsigned int len) { + const unsigned int idx = blockIdx.x * blockDim.x + threadIdx.x; + const unsigned int numThreads = blockDim.x * gridDim.x; + + for (unsigned int i = idx; i < len; i += numThreads) { + target[i] = __logf(mat[i]); + } +} + +__global__ void kExp(double* mat, double* target, unsigned int len) { + const unsigned int idx = blockIdx.x * blockDim.x + threadIdx.x; + const unsigned int numThreads = blockDim.x * gridDim.x; + + for (unsigned int i = idx; i < len; i += numThreads) { + target[i] = __expf(mat[i]); + } +} + +__global__ void kGamma(double* mat, double* target, unsigned int len) { + const unsigned int idx = blockIdx.x * blockDim.x + threadIdx.x; + const unsigned int numThreads = blockDim.x * gridDim.x; + + for (unsigned int i = idx; i < len; i += numThreads) { + target[i] = tgammaf(mat[i]); + } +} + +__global__ void kLogGamma(double* mat, double* target, unsigned int len) { + const unsigned int idx = blockIdx.x * blockDim.x + threadIdx.x; + const unsigned int numThreads = blockDim.x * gridDim.x; + + for (unsigned int i = idx; i < len; i += numThreads) { + target[i] = lgammaf(mat[i]); + } +} + +__global__ void kSqrt(double* mat, double* target, unsigned int len) { + const unsigned int idx = blockIdx.x * blockDim.x + threadIdx.x; + const unsigned int numThreads = blockDim.x * gridDim.x; + + for (unsigned int i = idx; i < len; i += numThreads) { + target[i] = sqrt(mat[i]); + } +} + +__global__ void kPow(double* mat, double pow, double* target, unsigned int len) { + const unsigned int idx = blockIdx.x * blockDim.x + threadIdx.x; + const unsigned int numThreads = blockDim.x * gridDim.x; + + for (unsigned int i = idx; i < len; i += numThreads) { + target[i] = powf(mat[i], pow); + } +} + +__global__ void kPowMatrix(double* mat, double* pow, double* target, unsigned int len) { + const unsigned int idx = blockIdx.x * blockDim.x + threadIdx.x; + const unsigned int numThreads = blockDim.x * gridDim.x; + + for (unsigned int i = idx; i < len; i += numThreads) { + target[i] = powf(mat[i], pow[i]); + } +} + +__global__ void kReciprocal(double* mat, double* target, unsigned int len) { + const unsigned int idx = blockIdx.x * blockDim.x + threadIdx.x; + const unsigned int numThreads = blockDim.x * gridDim.x; + + for (unsigned int i = idx; i < len; i += numThreads) + target[i] = 1.f / mat[i]; +} + +__global__ void kAddColVector(double* mat, double* vec, double* tgtMat, unsigned int width, + unsigned int height) { + const unsigned int idx = blockIdx.x * blockDim.x + threadIdx.x; + const unsigned int numThreads = blockDim.x * gridDim.x; + + for (unsigned int i = idx; i < width * height; i += numThreads) { + tgtMat[i] = mat[i] + vec[i % height]; + } +} + +__global__ void kAddRowVector(double* mat, double* vec, double* tgtMat, unsigned int width, unsigned int height) { + const unsigned int idx = blockIdx.x * blockDim.x + threadIdx.x; + const unsigned int numThreads = blockDim.x * gridDim.x; + + for (unsigned int i = idx; i < width * height; i += numThreads) { + tgtMat[i] = mat[i] + vec[i / height]; + } +} + +__global__ void kAddColMult(double* mat, double* vec, double* tgtMat, double mult, + unsigned int width, unsigned int height) { + const unsigned int idx = blockIdx.x * blockDim.x + threadIdx.x; + const unsigned int numThreads = blockDim.x * gridDim.x; + + for (unsigned int i = idx; i < width * height; i += numThreads) { + tgtMat[i] = mat[i] + mult * vec[i % height]; + } +} + +__global__ void kMultByColVector(double* mat, double* vec, double* tgtMat, unsigned int width, unsigned int height) { + const unsigned int idx = blockIdx.x * blockDim.x + threadIdx.x; + const unsigned int numThreads = blockDim.x * gridDim.x; + + for (unsigned int i = idx; i < width * height; i += numThreads) { + tgtMat[i] = mat[i] * vec[i % height]; + } +} + +__global__ void kMultByRowVector(double* mat, double* vec, double* tgtMat, unsigned int width, unsigned int height) { + const unsigned int idx = blockIdx.x * blockDim.x + threadIdx.x; + const unsigned int numThreads = blockDim.x * gridDim.x; + + for (unsigned int i = idx; i < width * height; i += numThreads) { + tgtMat[i] = mat[i] * vec[i / height]; + } +} + +__global__ void kDivByColVector(double* mat, double* vec, double* tgtMat, unsigned int width, unsigned int height) { + const unsigned int idx = blockIdx.x * blockDim.x + threadIdx.x; + const unsigned int numThreads = blockDim.x * gridDim.x; + + for (unsigned int i = idx; i < width * height; i += numThreads) { + tgtMat[i] = mat[i] / vec[i % height]; + } +} + +__global__ void kDivByRowVector(double* mat, double* vec, double* tgtMat, unsigned int width, unsigned int height) { + const unsigned int idx = blockIdx.x * blockDim.x + threadIdx.x; + const unsigned int numThreads = blockDim.x * gridDim.x; + + for (unsigned int i = idx; i < width * height; i += numThreads) { + tgtMat[i] = mat[i] / vec[i / height]; + } +} + +__global__ void kAdd(double* a, double* b, double* dest, unsigned int numEls) { + const unsigned int idx = blockIdx.x * blockDim.x + threadIdx.x; + const unsigned int numThreads = blockDim.x * gridDim.x; + + for (unsigned int i = idx; i < numEls; i += numThreads) { + dest[i] = a[i] + b[i]; + } +} + +__global__ void kSubtract(double* a, double* b, double* dest, unsigned int numEls) { + const unsigned int idx = blockIdx.x * blockDim.x + threadIdx.x; + const unsigned int numThreads = blockDim.x * gridDim.x; + + for (unsigned int i = idx; i < numEls; i += numThreads) { + dest[i] = a[i] - b[i]; + } +} + +__global__ void kDivide(double* a, double* b, double* dest, unsigned int numEls) { + const unsigned int idx = blockIdx.x * blockDim.x + threadIdx.x; + const unsigned int numThreads = blockDim.x * gridDim.x; + + for (unsigned int i = idx; i < numEls; i += numThreads) { + dest[i] = a[i] / b[i]; + } +} + +__global__ void kMult(double* a, double* b, double* dest, unsigned int numEls) { + const unsigned int idx = blockIdx.x * blockDim.x + threadIdx.x; + const unsigned int numThreads = blockDim.x * gridDim.x; + + for (unsigned int i = idx; i < numEls; i += numThreads) { + dest[i] = a[i] * b[i]; + } +} + +__global__ void kMultScalar(double* mat, double alpha, double* dest, unsigned int len) { + const unsigned int idx = blockIdx.x * blockDim.x + threadIdx.x; + const unsigned int numThreads = blockDim.x * gridDim.x; + + for (unsigned int i = idx; i < len; i += numThreads) { + dest[i] = alpha * mat[i]; + } +} + +__global__ void kAssignScalar(double* dest, double alpha, unsigned int len) { + const unsigned int idx = blockIdx.x * blockDim.x + threadIdx.x; + const unsigned int numThreads = blockDim.x * gridDim.x; + + for (unsigned int i = idx; i < len; i += numThreads) { + dest[i] = alpha; + } +} + +__global__ void kDivideScalar(double* mat, double alpha, double* dest, unsigned int len) { + const unsigned int idx = blockIdx.x * blockDim.x + threadIdx.x; + const unsigned int numThreads = blockDim.x * gridDim.x; + + for (unsigned int i = idx; i < len; i += numThreads) { + dest[i] = mat[i] / alpha; + } +} + +__global__ void kAddScalar(double* a, double alpha, double* dest, unsigned int numEls) { + const unsigned int idx = blockIdx.x * blockDim.x + threadIdx.x; + const unsigned int numThreads = blockDim.x * gridDim.x; + + for (unsigned int i = idx; i < numEls; i += numThreads) { + dest[i] = a[i] + alpha; + } +} + +__global__ void kSelectRows(double* source, double* target, double* indices, int nRowIs, int nCols, int nSourceRows){ + __shared__ int sourceRowIndices[32]; + const int startTargetRowI = blockIdx.x * 32; + const int tid = threadIdx.x; + const int localNRowIs = min(32, nRowIs-startTargetRowI); + + // cooperatively load 32 row indices + if (tid < localNRowIs){ + sourceRowIndices[tid] = int(indices[startTargetRowI + tid]); + if (sourceRowIndices[tid]<0) + sourceRowIndices[tid] += nSourceRows; + if (sourceRowIndices[tid]<0 || sourceRowIndices[tid]>=nSourceRows) + sourceRowIndices[tid] = -1; + } + __syncthreads(); + + // copy 32 rows + for (int i=0; i=nTargetRows) + targetRowIndices[tid] = -1; + } + __syncthreads(); + + // copy 32 rows + for (int i=0; i= 0 && x < width && y >= 0 && y < height) + sum += source[j] * kernel[(kwidth * kheight / 2) + w * kheight + h]; + } + } + dest[i] = sum; + } +} diff --git a/ot/gpu/cudamat/cudamat/cudamat_kernels.cuh b/ot/gpu/cudamat/cudamat/cudamat_kernels.cuh new file mode 100644 index 0000000..25e2858 --- /dev/null +++ b/ot/gpu/cudamat/cudamat/cudamat_kernels.cuh @@ -0,0 +1,92 @@ +#ifndef NVMATRIX_KERNEL_H_ +#define NVMATRIX_KERNEL_H_ + +#define NUM_RND_BLOCKS 96 +#define NUM_RND_THREADS_PER_BLOCK 128 +#define NUM_RND_STREAMS (NUM_RND_BLOCKS * NUM_RND_THREADS_PER_BLOCK) + +/* + * Defines for getting the values at the lower and upper 32 bits + * of a 64-bit number. + */ +#define LOW_BITS(x) ((x) & 0xffffffff) +#define HIGH_BITS(x) ((x) >> 32) + +/* + * Number of iterations to run random number generator upon initialization. + */ +#define NUM_RND_BURNIN 100 + +/* + * CUDA grid dimensions for different types of kernels + */ +#define COPY_BLOCK_SIZE 16 +# +// element-wise kernels use min(ceil(N / 512), 4096) blocks of 512 threads +#define MAX_VECTOR_OP_BLOCKS 4096 +#define MAX_VECTOR_OP_THREADS_PER_BLOCK 512 +#define NUM_VECTOR_OP_BLOCKS(N) (min(((N) + MAX_VECTOR_OP_THREADS_PER_BLOCK - 1)/MAX_VECTOR_OP_THREADS_PER_BLOCK, MAX_VECTOR_OP_BLOCKS)) +#define NUM_VECTOR_OP_THREADS_PER_BLOCK(N) (min((N), MAX_VECTOR_OP_THREADS_PER_BLOCK)) + +#define PI 3.1415926535897932f + +__global__ void kSeedRandom(unsigned int* randMults, unsigned long long* randWords, unsigned int seed); +__global__ void kRandomUniform(unsigned int* randMults, unsigned long long* randWords, double* gData, unsigned int numElements); +__global__ void kRandomGaussian(unsigned int* rndMults, unsigned long long* rndWords, double* gData, unsigned int numElements); + +__global__ void kGetRowSlice(double* source, double* target, int start, int end, int width, int height); +__global__ void kTranspose(double *odata, double *idata, int width, int height); +__global__ void kSetRowSlice(double* source, double* target, int start, int end, int width, int height); + +__global__ void kLessThan(double* mat1, double* mat2, double* target, unsigned int len); +__global__ void kLessThanScalar(double* mat, double val, double* target, unsigned int len); +__global__ void kGreaterThan(double* mat1, double* mat2, double* target, unsigned int len); +__global__ void kGreaterThanScalar(double* mat, double val, double* target, unsigned int len); +__global__ void kEquals(double* mat1, double* mat2, double* target, unsigned int len); +__global__ void kEqualsScalar(double* mat, double val, double* target, unsigned int len); +__global__ void kMinimum(double* mat1, double* mat2, double* target, unsigned int len); +__global__ void kMinimumScalar(double* mat, double val, double* target, unsigned int len); +__global__ void kMaximum(double* mat1, double* mat2, double* target, unsigned int len); +__global__ void kMaximumScalar(double* mat, double val, double* target, unsigned int len); +__global__ void kMinColumnwise(double* mat, double* target, unsigned int width, unsigned int height); +__global__ void kMinRowwise(double* mat, double* target, unsigned int width, unsigned int height); +__global__ void kMaxColumnwise(double* mat, double* target, unsigned int width, unsigned int height); +__global__ void kMaxRowwise(double* mat, double* target, unsigned int width, unsigned int height); +__global__ void kArgMinColumnwise(double* mat, double* target, unsigned int width, unsigned int height); +__global__ void kArgMinRowwise(double* mat, double* target, unsigned int width, unsigned int height); +__global__ void kArgMaxColumnwise(double* mat, double* target, unsigned int width, unsigned int height); +__global__ void kArgMaxRowwise(double* mat, double* target, unsigned int width, unsigned int height); +__global__ void kSign(double* mat, double* target, unsigned int len); +__global__ void kApplySigmoid(double* mat, double* target, unsigned int len); +__global__ void kApplyTanh(double* mat, double* target, unsigned int len); +__global__ void kApplySoftThreshold(double* mat, double alpha, double* target, unsigned int len); +__global__ void kApplyAbs(double* mat, double* target, unsigned int len); +__global__ void kApplyLog1PlusExp(double* mat, double* target, unsigned int len); +__global__ void kLog(double* mat, double* target, unsigned int len); +__global__ void kExp(double* mat, double* target, unsigned int len); +__global__ void kGamma(double* mat, double* target, unsigned int len); +__global__ void kLogGamma(double* mat, double* target, unsigned int len); +__global__ void kSqrt(double* mat, double* target, unsigned int len); +__global__ void kPow(double* mat, double pow, double* target, unsigned int len); +__global__ void kPowMatrix(double* mat, double* pow, double* target, unsigned int len); +__global__ void kReciprocal(double* mat, double* target, unsigned int len); +__global__ void kAddColVector(double* mat, double* vec, double* tgtMat, unsigned int width, unsigned int height); +__global__ void kAddRowVector(double* mat, double* vec, double* tgtMat, unsigned int width, unsigned int height); +__global__ void kAddColMult(double* mat, double* vec, double* tgtMat, double mult, unsigned int width, unsigned int height); +__global__ void kMultByColVector(double* mat, double* vec, double* tgtMat, unsigned int width, unsigned int height); +__global__ void kMultByRowVector(double* mat, double* vec, double* tgtMat, unsigned int width, unsigned int height); +__global__ void kDivByColVector(double* mat, double* vec, double* tgtMat, unsigned int width, unsigned int height); +__global__ void kDivByRowVector(double* mat, double* vec, double* tgtMat, unsigned int width, unsigned int height); +__global__ void kAdd(double* a, double* b, double* dest, unsigned int numEls); +__global__ void kSubtract(double* a, double* b, double* dest, unsigned int numEls); +__global__ void kMult(double* a, double* b, double* dest, unsigned int numEls); +__global__ void kDivide(double* a, double* b, double* dest, unsigned int numEls); +__global__ void kMultScalar(double* mat, double alpha, double* dest, unsigned int len); +__global__ void kAssignScalar(double* dest, double alpha, unsigned int len); +__global__ void kDivideScalar(double* mat, double alpha, double* dest, unsigned int len); +__global__ void kAddScalar(double* a, double alpha, double* dest, unsigned int numEls); +__global__ void kSelectRows(double* source, double* target, double* indices, int nRowIs, int nCols, int nSourceRows); +__global__ void kSetSelectedRows(double* target, double* source, double* indices, int nRowIs, int nCols, int nTargetRows); +__global__ void kWhere(double* condition_mat, double* if_mat, double* else_mat, double* target, unsigned int len); +__global__ void kCorrelate(double* source, double* kernel, double* dest, int width, int height, int fwidth, int fheight); +#endif diff --git a/ot/gpu/cudamat/cudamat/learn.cu b/ot/gpu/cudamat/cudamat/learn.cu new file mode 100644 index 0000000..3d9260c --- /dev/null +++ b/ot/gpu/cudamat/cudamat/learn.cu @@ -0,0 +1,34 @@ +#include +#include +#include +#include "learn_kernels.cuh" +#include "cudamat.cuh" + +extern "C" { + +inline bool checkCUDAError() { + cudaError_t err = cudaGetLastError(); + + if (cudaSuccess != err) + printf("%s\n", cudaGetErrorString( err)); + return cudaSuccess != err; +} + +EXPORT int mult_by_sigmoid_deriv(cudamat* target, cudamat* acts) { + int len = acts->size[0]*acts->size[1]; + + if (acts->is_trans != target->is_trans) + return ERROR_TRANSPOSED; + + if (acts->size[0] != target->size[0] || acts->size[1] != target->size[1]) + return ERROR_INCOMPATIBLE_DIMENSIONS; + + kMultiplyBySigmoidGrad<<>>(acts->data_device, target->data_device, len); + + if (checkCUDAError()) + return CUDA_ERROR; + + return 0; +} + +} diff --git a/ot/gpu/cudamat/cudamat/learn.py b/ot/gpu/cudamat/cudamat/learn.py new file mode 100644 index 0000000..741ca13 --- /dev/null +++ b/ot/gpu/cudamat/cudamat/learn.py @@ -0,0 +1,21 @@ +import os + +import ctypes as ct +import numpy as np + +from cudamat import load_library, generate_exception + +_cudalearn = load_library('libcudalearn') + +_cudalearn.mult_by_sigmoid_deriv.restype = ct.c_int + +def mult_by_sigmoid_deriv(target, acts): + """ + target = target * acts * (1 - acts) + + Useful for doing backprop in neural networks with logistic units. + """ + + err_code = _cudalearn.mult_by_sigmoid_deriv(target.p_mat, acts.p_mat) + if err_code: + raise generate_exception(err_code) diff --git a/ot/gpu/cudamat/cudamat/learn_kernels.cu b/ot/gpu/cudamat/cudamat/learn_kernels.cu new file mode 100644 index 0000000..8e897ba --- /dev/null +++ b/ot/gpu/cudamat/cudamat/learn_kernels.cu @@ -0,0 +1,10 @@ +#include "learn_kernels.cuh" + +__global__ void kMultiplyBySigmoidGrad(double* act, double* target, const unsigned int len) { + const unsigned int idx = blockIdx.x * blockDim.x + threadIdx.x; + const unsigned int numThreads = blockDim.x * gridDim.x; + + for(unsigned int i = idx; i < len; i+= numThreads) { + target[i] = target[i] * act[i] * (1.0f - act[i]); + } +} diff --git a/ot/gpu/cudamat/cudamat/learn_kernels.cuh b/ot/gpu/cudamat/cudamat/learn_kernels.cuh new file mode 100644 index 0000000..26c8abd --- /dev/null +++ b/ot/gpu/cudamat/cudamat/learn_kernels.cuh @@ -0,0 +1,12 @@ +#ifndef EBM_KERNELS_H_ +#define EBM_KERNELS_H_ + +#define NUM_VECTOR_OP_BLOCKS 4096 +#define NUM_VECTOR_OP_THREADS_PER_BLOCK 512 + +#define NUM_SPARSE_GRAD_BLOCKS 4096 +#define NUM_SPARSE_GRAD_THREADS_PER_BLOCK 512 + +__global__ void kMultiplyBySigmoidGrad(double* act, double* target, const unsigned int len); + +#endif diff --git a/ot/gpu/cudamat/cudamat/rnd_multipliers_32bit.txt b/ot/gpu/cudamat/cudamat/rnd_multipliers_32bit.txt new file mode 100644 index 0000000..098d31d --- /dev/null +++ b/ot/gpu/cudamat/cudamat/rnd_multipliers_32bit.txt @@ -0,0 +1,16028 @@ +4294967118 +4294966893 +4294966830 +4294966284 +4294966164 +4294965708 +4294965675 +4294964880 +4294964568 +4294963860 +4294963023 +4294962654 +4294962540 +4294962330 +4294962063 +4294959894 +4294959600 +4294959030 +4294958094 +4294957665 +4294956564 +4294956270 +4294955895 +4294954305 +4294953888 +4294952739 +4294951410 +4294951185 +4294951143 +4294951008 +4294950714 +4294950423 +4294948140 +4294948110 +4294946055 +4294945578 +4294945488 +4294942995 +4294942893 +4294942830 +4294942734 +4294941468 +4294941180 +4294939374 +4294937085 +4294936635 +4294936434 +4294936383 +4294935210 +4294934769 +4294934685 +4294934370 +4294934250 +4294934073 +4294933095 +4294931880 +4294929423 +4294928088 +4294927374 +4294927215 +4294926153 +4294925778 +4294924500 +4294924359 +4294923360 +4294922925 +4294922058 +4294921653 +4294921119 +4294920288 +4294920084 +4294917345 +4294917189 +4294917144 +4294916664 +4294915479 +4294915275 +4294913553 +4294913154 +4294912938 +4294912005 +4294911834 +4294911699 +4294909554 +4294905279 +4294904778 +4294904745 +4294904640 +4294904313 +4294904085 +4294904043 +4294902279 +4294901070 +4294900389 +4294899474 +4294899090 +4294897890 +4294896159 +4294896045 +4294895538 +4294894824 +4294894635 +4294894260 +4294893354 +4294893174 +4294892304 +4294891674 +4294890594 +4294890414 +4294889508 +4294888800 +4294888515 +4294888389 +4294888080 +4294886025 +4294884075 +4294883355 +4294883064 +4294881060 +4294879554 +4294879065 +4294878213 +4294877943 +4294876794 +4294876773 +4294875189 +4294875180 +4294874934 +4294874724 +4294874190 +4294873869 +4294873434 +4294873089 +4294872678 +4294872219 +4294869930 +4294869489 +4294869165 +4294868898 +4294867500 +4294866324 +4294865769 +4294864878 +4294864875 +4294861674 +4294861623 +4294860975 +4294859733 +4294859274 +4294859184 +4294859175 +4294857699 +4294857213 +4294856655 +4294854744 +4294853718 +4294853373 +4294851438 +4294850955 +4294849935 +4294847094 +4294846905 +4294843659 +4294843395 +4294843194 +4294842114 +4294841493 +4294840065 +4294837218 +4294837188 +4294836468 +4294833009 +4294832850 +4294831623 +4294831299 +4294830168 +4294829364 +4294829250 +4294828380 +4294828314 +4294826889 +4294826238 +4294824885 +4294823889 +4294823478 +4294822794 +4294822140 +4294821258 +4294820463 +4294820220 +4294819599 +4294818975 +4294818909 +4294816869 +4294814655 +4294814523 +4294814268 +4294814130 +4294813458 +4294810593 +4294810569 +4294810089 +4294808889 +4294808880 +4294808190 +4294807329 +4294807035 +4294806738 +4294806393 +4294805679 +4294805088 +4294803003 +4294802784 +4294802088 +4294802013 +4294801290 +4294799649 +4294799244 +4294799010 +4294798278 +4294798068 +4294797303 +4294796670 +4294796205 +4294796169 +4294793733 +4294793004 +4294792704 +4294792425 +4294791864 +4294791495 +4294791303 +4294791255 +4294791150 +4294790475 +4294789950 +4294788489 +4294786695 +4294786683 +4294786035 +4294785759 +4294785129 +4294783893 +4294783248 +4294782978 +4294782639 +4294780698 +4294780488 +4294780083 +4294779903 +4294779303 +4294778853 +4294778328 +4294778055 +4294777128 +4294776729 +4294775649 +4294775640 +4294774338 +4294773969 +4294773813 +4294773384 +4294772583 +4294772145 +4294770984 +4294769505 +4294769340 +4294769013 +4294768155 +4294767918 +4294767459 +4294766868 +4294765284 +4294765125 +4294764999 +4294763964 +4294762758 +4294761588 +4294759479 +4294759239 +4294757493 +4294757325 +4294756584 +4294756473 +4294754334 +4294754154 +4294753485 +4294752678 +4294751994 +4294751118 +4294749768 +4294749669 +4294747380 +4294747194 +4294746300 +4294746249 +4294746114 +4294745400 +4294745199 +4294744650 +4294743813 +4294742649 +4294741788 +4294741218 +4294740429 +4294739280 +4294739085 +4294738794 +4294738455 +4294737513 +4294736934 +4294735863 +4294735413 +4294732494 +4294731300 +4294730154 +4294730019 +4294729710 +4294728984 +4294728849 +4294728765 +4294727514 +4294726434 +4294726320 +4294725939 +4294725888 +4294725393 +4294725294 +4294723284 +4294723089 +4294720794 +4294719840 +4294717728 +4294717710 +4294717104 +4294716879 +4294716348 +4294716063 +4294715460 +4294714158 +4294712625 +4294712094 +4294711608 +4294710645 +4294709130 +4294707408 +4294705485 +4294705404 +4294705170 +4294704744 +4294704468 +4294702398 +4294702380 +4294701465 +4294701393 +4294700835 +4294700763 +4294700418 +4294699260 +4294697679 +4294697469 +4294697169 +4294695924 +4294693833 +4294692615 +4294691805 +4294691343 +4294691178 +4294690620 +4294690308 +4294688814 +4294688550 +4294688328 +4294688034 +4294685535 +4294684665 +4294683525 +4294682973 +4294682448 +4294681740 +4294681590 +4294681404 +4294680963 +4294680699 +4294679940 +4294679919 +4294679619 +4294678305 +4294678128 +4294675890 +4294673970 +4294673253 +4294672494 +4294670385 +4294668195 +4294667508 +4294667415 +4294666293 +4294665978 +4294665465 +4294665324 +4294664844 +4294664643 +4294663488 +4294663443 +4294662819 +4294662174 +4294661838 +4294657848 +4294657050 +4294656549 +4294656309 +4294655574 +4294655520 +4294654134 +4294653075 +4294652130 +4294651578 +4294648113 +4294647474 +4294646079 +4294645563 +4294645305 +4294645005 +4294643478 +4294643088 +4294641690 +4294637829 +4294636263 +4294635765 +4294634778 +4294634055 +4294633605 +4294633014 +4294632009 +4294630719 +4294630173 +4294628169 +4294626843 +4294626663 +4294626213 +4294625994 +4294625238 +4294623705 +4294623483 +4294622373 +4294621899 +4294620114 +4294619913 +4294619469 +4294619019 +4294618665 +4294617600 +4294617150 +4294614570 +4294614498 +4294613553 +4294613358 +4294611495 +4294610523 +4294609290 +4294608843 +4294608270 +4294608219 +4294607910 +4294606203 +4294605969 +4294605579 +4294605210 +4294605069 +4294603929 +4294603830 +4294603155 +4294602114 +4294599963 +4294598745 +4294598658 +4294598229 +4294597065 +4294593978 +4294593663 +4294592694 +4294592508 +4294592238 +4294591728 +4294591440 +4294589190 +4294587954 +4294587909 +4294587825 +4294587813 +4294587309 +4294586604 +4294585398 +4294585044 +4294584699 +4294584393 +4294584159 +4294583889 +4294582704 +4294582368 +4294582050 +4294581819 +4294581123 +4294580379 +4294580214 +4294577115 +4294576425 +4294575474 +4294572588 +4294571280 +4294570740 +4294570215 +4294569930 +4294569240 +4294568220 +4294568004 +4294567509 +4294566918 +4294566459 +4294566339 +4294565778 +4294565724 +4294564359 +4294564344 +4294563558 +4294560528 +4294559955 +4294559673 +4294559394 +4294558539 +4294558278 +4294557708 +4294556265 +4294556199 +4294556028 +4294554048 +4294552503 +4294551588 +4294550979 +4294549995 +4294548408 +4294548168 +4294546854 +4294546830 +4294546098 +4294546065 +4294544904 +4294544373 +4294543440 +4294543305 +4294543140 +4294542183 +4294541343 +4294540899 +4294540743 +4294540479 +4294539873 +4294539429 +4294538580 +4294536270 +4294535619 +4294534818 +4294534275 +4294533900 +4294533885 +4294532439 +4294531548 +4294531410 +4294530663 +4294530024 +4294529694 +4294528668 +4294528635 +4294528455 +4294525950 +4294525779 +4294525038 +4294524564 +4294523589 +4294521099 +4294520880 +4294520403 +4294520118 +4294518555 +4294517328 +4294517160 +4294517055 +4294516875 +4294515783 +4294515603 +4294515414 +4294513725 +4294513443 +4294513110 +4294512939 +4294511283 +4294510689 +4294510398 +4294509660 +4294509429 +4294508205 +4294507269 +4294506984 +4294505613 +4294505274 +4294503534 +4294503225 +4294502793 +4294502694 +4294501809 +4294501443 +4294501389 +4294499313 +4294499214 +4294498635 +4294497270 +4294496643 +4294496205 +4294496130 +4294495404 +4294495140 +4294493229 +4294492620 +4294492053 +4294492035 +4294490793 +4294490109 +4294489230 +4294488813 +4294488453 +4294488063 +4294485774 +4294485408 +4294484544 +4294483488 +4294482570 +4294480653 +4294480335 +4294479693 +4294477614 +4294476138 +4294475715 +4294475394 +4294475328 +4294474644 +4294473918 +4294473678 +4294473504 +4294471359 +4294470984 +4294470204 +4294469850 +4294468785 +4294468365 +4294466508 +4294466310 +4294465923 +4294465824 +4294463874 +4294463769 +4294463610 +4294463340 +4294463109 +4294462443 +4294462353 +4294461414 +4294461129 +4294459470 +4294458813 +4294458348 +4294458294 +4294458279 +4294457805 +4294456704 +4294455165 +4294454925 +4294454718 +4294454688 +4294453740 +4294453464 +4294453110 +4294452888 +4294452459 +4294452225 +4294452153 +4294452120 +4294451124 +4294451103 +4294450194 +4294449270 +4294448718 +4294448583 +4294446999 +4294446600 +4294445085 +4294444890 +4294444878 +4294444713 +4294444353 +4294443573 +4294443483 +4294443054 +4294441395 +4294440669 +4294440513 +4294439313 +4294438314 +4294438065 +4294437420 +4294436235 +4294435713 +4294435494 +4294434555 +4294433073 +4294432404 +4294432128 +4294431714 +4294431519 +4294429404 +4294428909 +4294428423 +4294428069 +4294427238 +4294425555 +4294423965 +4294423644 +4294422588 +4294422513 +4294422495 +4294422315 +4294422054 +4294420548 +4294418514 +4294415628 +4294414089 +4294413345 +4294412328 +4294412325 +4294411665 +4294410795 +4294410198 +4294409019 +4294407909 +4294406289 +4294406283 +4294405893 +4294404474 +4294404243 +4294403433 +4294403148 +4294402230 +4294402218 +4294401954 +4294401363 +4294401324 +4294400340 +4294400253 +4294400250 +4294398825 +4294398780 +4294397460 +4294397028 +4294396968 +4294395993 +4294395924 +4294395453 +4294394949 +4294393878 +4294393803 +4294393110 +4294393008 +4294392858 +4294391970 +4294391358 +4294390329 +4294389540 +4294387833 +4294387413 +4294386909 +4294386123 +4294385445 +4294385424 +4294385214 +4294384473 +4294380714 +4294380348 +4294380285 +4294378788 +4294378203 +4294377645 +4294377633 +4294377234 +4294377120 +4294376754 +4294376043 +4294374843 +4294373949 +4294373940 +4294372089 +4294372029 +4294371858 +4294370169 +4294369785 +4294369548 +4294369275 +4294367649 +4294366305 +4294366140 +4294365795 +4294364565 +4294364469 +4294363533 +4294363500 +4294363044 +4294362285 +4294362093 +4294361205 +4294361148 +4294359804 +4294359753 +4294359609 +4294357245 +4294355169 +4294355064 +4294354758 +4294354365 +4294353780 +4294351374 +4294350669 +4294350045 +4294350039 +4294349178 +4294348065 +4294346190 +4294345650 +4294344435 +4294343775 +4294343469 +4294343049 +4294342899 +4294342239 +4294341744 +4294340844 +4294339329 +4294337103 +4294336395 +4294335813 +4294334343 +4294333848 +4294333338 +4294332654 +4294330815 +4294330314 +4294330284 +4294330143 +4294329480 +4294328919 +4294327224 +4294326060 +4294325400 +4294325238 +4294324140 +4294323879 +4294323843 +4294323720 +4294323045 +4294323015 +4294320525 +4294320444 +4294318920 +4294318404 +4294318260 +4294318110 +4294317609 +4294316880 +4294316124 +4294315059 +4294313625 +4294313574 +4294313259 +4294311705 +4294311645 +4294310580 +4294310424 +4294310028 +4294307580 +4294306203 +4294303740 +4294303308 +4294303020 +4294302900 +4294302585 +4294302318 +4294301844 +4294301340 +4294298205 +4294297209 +4294294749 +4294294548 +4294294509 +4294291905 +4294291770 +4294291284 +4294289373 +4294288230 +4294288218 +4294287849 +4294287648 +4294287303 +4294286694 +4294285854 +4294285824 +4294285689 +4294282263 +4294280388 +4294280250 +4294279683 +4294278639 +4294278414 +4294278210 +4294277193 +4294276650 +4294276584 +4294274808 +4294274589 +4294274349 +4294273950 +4294273494 +4294272465 +4294271379 +4294270134 +4294270125 +4294269714 +4294268625 +4294267923 +4294267788 +4294266348 +4294266105 +4294265853 +4294264719 +4294264338 +4294263468 +4294263294 +4294263198 +4294262808 +4294262454 +4294262400 +4294262304 +4294261020 +4294258605 +4294257933 +4294256715 +4294255143 +4294254420 +4294254039 +4294253724 +4294251333 +4294251048 +4294250259 +4294249935 +4294249569 +4294249428 +4294249308 +4294248399 +4294247265 +4294245168 +4294244655 +4294243098 +4294240215 +4294240035 +4294237773 +4294237713 +4294237020 +4294233054 +4294231473 +4294230498 +4294230270 +4294229568 +4294229493 +4294229070 +4294228743 +4294228440 +4294228134 +4294228125 +4294227969 +4294226970 +4294225113 +4294222428 +4294221228 +4294220199 +4294218159 +4294217520 +4294216659 +4294216230 +4294215120 +4294213299 +4294211313 +4294210584 +4294210500 +4294209135 +4294208520 +4294207608 +4294205184 +4294205019 +4294204683 +4294204188 +4294204113 +4294201884 +4294200168 +4294200063 +4294199379 +4294198968 +4294198875 +4294197990 +4294197753 +4294196565 +4294196199 +4294196178 +4294195515 +4294194639 +4294194588 +4294193904 +4294193574 +4294193454 +4294193274 +4294192908 +4294192248 +4294191873 +4294190715 +4294190538 +4294190523 +4294190220 +4294188045 +4294186989 +4294186608 +4294186200 +4294185903 +4294184049 +4294183794 +4294183764 +4294182345 +4294179960 +4294179615 +4294178175 +4294177908 +4294177383 +4294177320 +4294176873 +4294176810 +4294175154 +4294174884 +4294174479 +4294174038 +4294173225 +4294172370 +4294172049 +4294170984 +4294170630 +4294170420 +4294169865 +4294169514 +4294169505 +4294168509 +4294168455 +4294166730 +4294165779 +4294165533 +4294164915 +4294163670 +4294163445 +4294163313 +4294162503 +4294161924 +4294160988 +4294160568 +4294159764 +4294159605 +4294158819 +4294157358 +4294157304 +4294156698 +4294155939 +4294155354 +4294153374 +4294152579 +4294152540 +4294152498 +4294151805 +4294151658 +4294151520 +4294151190 +4294151148 +4294151100 +4294150980 +4294150224 +4294149978 +4294149894 +4294149639 +4294148880 +4294148769 +4294147143 +4294146309 +4294145799 +4294144890 +4294144449 +4294143594 +4294143489 +4294143414 +4294142175 +4294142124 +4294141515 +4294141479 +4294140969 +4294138443 +4294138080 +4294137783 +4294137393 +4294137228 +4294136670 +4294136484 +4294136295 +4294135440 +4294135404 +4294134285 +4294134129 +4294134054 +4294133943 +4294132809 +4294132773 +4294132578 +4294132554 +4294132473 +4294132080 +4294131513 +4294131444 +4294131378 +4294128558 +4294127703 +4294127409 +4294127244 +4294126968 +4294126353 +4294125330 +4294124943 +4294124703 +4294124205 +4294124103 +4294123599 +4294123020 +4294120503 +4294119549 +4294119123 +4294118685 +4294118394 +4294117290 +4294117128 +4294116405 +4294116354 +4294116330 +4294115979 +4294114863 +4294114098 +4294113939 +4294113699 +4294111794 +4294110240 +4294109169 +4294108710 +4294105578 +4294104225 +4294102389 +4294102029 +4294101033 +4294101000 +4294100985 +4294100028 +4294099284 +4294098270 +4294097709 +4294097178 +4294095429 +4294095093 +4294094073 +4294093065 +4294092915 +4294091658 +4294089108 +4294087539 +4294086468 +4294086168 +4294085664 +4294085229 +4294084968 +4294084443 +4294084425 +4294083039 +4294081140 +4294076499 +4294074603 +4294072398 +4294070103 +4294067799 +4294065114 +4294063035 +4294060149 +4294060134 +4294059153 +4294059033 +4294058100 +4294057194 +4294054905 +4294054800 +4294053678 +4294053210 +4294053168 +4294050033 +4294048218 +4294047558 +4294046739 +4294046700 +4294045719 +4294045560 +4294045185 +4294044900 +4294044555 +4294043358 +4294042695 +4294042515 +4294041864 +4294038090 +4294037898 +4294037760 +4294036575 +4294036080 +4294035399 +4294033803 +4294032249 +4294031595 +4294030125 +4294029360 +4294028838 +4294028730 +4294027779 +4294027578 +4294027239 +4294026945 +4294026843 +4294026318 +4294025793 +4294025403 +4294024395 +4294024245 +4294023648 +4294023375 +4294021248 +4294021080 +4294020300 +4294019430 +4294018599 +4294017414 +4294017069 +4294016580 +4294016550 +4294015305 +4294013289 +4294011963 +4294011645 +4294009599 +4294009299 +4294008495 +4294008144 +4294007829 +4294007583 +4294007439 +4294005630 +4294003710 +4294003689 +4294003554 +4294003539 +4294002894 +4294002168 +4294001298 +4294000800 +4294000698 +4294000674 +4293999324 +4293999060 +4293999030 +4293998115 +4293997305 +4293997023 +4293996909 +4293996714 +4293996519 +4293995013 +4293994878 +4293993969 +4293993429 +4293991899 +4293991065 +4293990393 +4293990189 +4293990048 +4293989733 +4293988503 +4293986823 +4293985890 +4293984213 +4293983079 +4293981879 +4293979593 +4293979434 +4293978804 +4293978345 +4293978165 +4293978069 +4293978048 +4293977940 +4293977883 +4293976938 +4293976725 +4293976314 +4293975633 +4293973488 +4293971928 +4293971463 +4293969225 +4293968010 +4293967128 +4293966645 +4293962454 +4293962394 +4293962193 +4293961449 +4293960723 +4293960660 +4293960405 +4293958530 +4293957783 +4293957699 +4293956994 +4293955989 +4293954885 +4293954783 +4293954633 +4293953613 +4293953499 +4293953154 +4293952785 +4293951858 +4293949803 +4293947850 +4293947754 +4293947610 +4293947145 +4293947124 +4293946698 +4293946383 +4293946350 +4293945663 +4293944829 +4293944535 +4293944094 +4293942000 +4293940815 +4293939525 +4293938778 +4293938664 +4293936513 +4293935934 +4293935313 +4293934575 +4293932853 +4293931995 +4293928038 +4293925965 +4293923850 +4293922380 +4293922164 +4293922008 +4293921969 +4293921408 +4293921285 +4293920379 +4293920133 +4293919125 +4293918768 +4293915165 +4293913884 +4293913020 +4293912999 +4293912903 +4293912588 +4293912150 +4293911490 +4293911373 +4293911280 +4293910608 +4293909390 +4293908613 +4293908535 +4293907518 +4293907113 +4293907098 +4293905673 +4293904578 +4293904494 +4293904278 +4293903480 +4293903054 +4293902628 +4293902310 +4293901863 +4293900534 +4293898635 +4293898605 +4293898125 +4293897114 +4293897063 +4293895584 +4293895500 +4293894549 +4293893619 +4293893514 +4293893229 +4293892239 +4293892113 +4293890979 +4293890928 +4293889488 +4293889410 +4293889194 +4293888879 +4293888870 +4293888075 +4293887979 +4293886014 +4293885354 +4293885273 +4293883824 +4293883800 +4293883473 +4293882039 +4293881280 +4293880755 +4293879810 +4293879183 +4293879168 +4293878784 +4293877533 +4293875019 +4293874914 +4293874779 +4293874494 +4293874308 +4293874284 +4293873195 +4293873165 +4293871515 +4293871008 +4293870195 +4293869604 +4293868683 +4293868353 +4293867270 +4293867114 +4293866409 +4293866328 +4293865983 +4293864648 +4293862308 +4293861738 +4293861603 +4293861573 +4293861543 +4293861150 +4293860700 +4293860634 +4293860088 +4293859923 +4293859878 +4293859380 +4293858954 +4293858570 +4293858555 +4293858519 +4293858450 +4293858348 +4293858015 +4293856995 +4293856314 +4293855840 +4293855180 +4293854643 +4293854388 +4293853254 +4293852645 +4293852549 +4293852333 +4293850623 +4293849999 +4293849939 +4293849678 +4293848853 +4293848403 +4293848049 +4293847620 +4293845799 +4293845664 +4293844590 +4293844098 +4293843585 +4293843564 +4293842640 +4293842190 +4293841905 +4293840813 +4293838473 +4293838053 +4293837864 +4293837429 +4293837348 +4293836970 +4293835584 +4293835143 +4293834579 +4293834228 +4293833145 +4293831228 +4293829308 +4293828444 +4293826368 +4293826209 +4293825885 +4293825114 +4293825054 +4293825030 +4293824193 +4293823950 +4293823920 +4293823068 +4293822039 +4293821988 +4293821754 +4293820410 +4293819543 +4293819255 +4293818094 +4293817584 +4293817515 +4293817389 +4293816009 +4293815874 +4293813600 +4293813039 +4293811368 +4293810060 +4293808263 +4293807525 +4293807129 +4293806670 +4293806139 +4293805773 +4293804504 +4293803823 +4293802110 +4293801888 +4293800754 +4293800733 +4293800238 +4293800175 +4293798969 +4293796398 +4293796209 +4293795618 +4293795513 +4293795255 +4293795000 +4293791055 +4293790290 +4293789345 +4293789123 +4293788103 +4293787239 +4293786378 +4293786288 +4293784923 +4293784155 +4293783180 +4293782874 +4293782373 +4293782253 +4293779910 +4293778650 +4293777405 +4293775743 +4293774750 +4293774633 +4293774528 +4293774054 +4293774024 +4293772713 +4293772680 +4293772215 +4293771438 +4293770628 +4293768774 +4293768708 +4293767694 +4293767625 +4293766785 +4293766539 +4293765558 +4293764628 +4293764145 +4293763494 +4293762903 +4293761064 +4293758598 +4293758304 +4293758265 +4293757278 +4293756339 +4293755949 +4293755088 +4293755070 +4293753924 +4293752445 +4293750084 +4293749613 +4293748908 +4293747720 +4293746568 +4293744465 +4293743718 +4293741453 +4293741438 +4293739803 +4293739413 +4293739008 +4293738678 +4293736545 +4293736215 +4293735948 +4293735549 +4293734340 +4293733830 +4293733734 +4293733053 +4293732843 +4293732444 +4293732159 +4293731730 +4293731235 +4293729879 +4293729825 +4293728613 +4293728193 +4293728190 +4293728139 +4293727833 +4293727788 +4293727530 +4293725814 +4293725220 +4293725028 +4293724434 +4293724203 +4293723105 +4293722055 +4293722019 +4293721584 +4293720399 +4293720150 +4293718488 +4293717069 +4293716139 +4293715083 +4293714423 +4293714264 +4293713829 +4293713184 +4293712599 +4293711453 +4293710478 +4293710433 +4293707748 +4293706833 +4293706494 +4293705108 +4293704598 +4293703308 +4293701049 +4293698328 +4293697395 +4293697218 +4293694314 +4293693369 +4293692763 +4293691008 +4293690933 +4293690828 +4293689823 +4293689259 +4293688134 +4293687870 +4293686124 +4293685593 +4293685488 +4293684564 +4293683619 +4293683043 +4293681495 +4293680973 +4293680565 +4293679755 +4293679689 +4293676890 +4293676854 +4293674925 +4293674805 +4293673788 +4293673779 +4293673440 +4293672123 +4293671865 +4293670284 +4293667455 +4293667278 +4293666840 +4293665979 +4293664695 +4293662439 +4293662400 +4293662238 +4293661770 +4293660789 +4293659985 +4293659595 +4293659280 +4293659154 +4293655530 +4293655059 +4293653934 +4293652944 +4293651855 +4293651165 +4293650619 +4293650259 +4293649689 +4293649053 +4293647883 +4293646185 +4293645825 +4293644505 +4293643788 +4293643068 +4293643053 +4293642960 +4293642849 +4293642618 +4293642318 +4293642144 +4293640350 +4293639594 +4293639300 +4293639255 +4293638523 +4293638145 +4293637089 +4293636765 +4293636348 +4293636150 +4293634638 +4293634539 +4293633729 +4293632994 +4293631740 +4293631068 +4293630969 +4293630129 +4293629415 +4293626448 +4293626289 +4293625839 +4293625575 +4293623349 +4293621879 +4293621804 +4293621279 +4293621048 +4293620208 +4293620010 +4293619494 +4293618339 +4293618084 +4293616890 +4293615963 +4293615159 +4293613878 +4293613248 +4293612429 +4293612288 +4293611868 +4293611703 +4293611433 +4293610695 +4293610455 +4293609834 +4293609039 +4293608154 +4293607395 +4293606945 +4293606423 +4293605925 +4293605604 +4293604770 +4293604350 +4293603399 +4293602895 +4293602829 +4293602769 +4293602598 +4293602424 +4293602064 +4293599904 +4293599658 +4293599124 +4293598938 +4293598890 +4293598329 +4293598245 +4293598179 +4293598149 +4293596658 +4293596289 +4293595404 +4293595038 +4293594378 +4293593745 +4293593064 +4293592740 +4293591930 +4293590685 +4293588519 +4293587709 +4293585399 +4293584970 +4293584469 +4293583923 +4293583458 +4293583455 +4293583413 +4293582954 +4293582555 +4293582243 +4293580893 +4293580014 +4293579939 +4293579459 +4293578388 +4293577974 +4293574020 +4293573660 +4293573315 +4293572904 +4293571488 +4293571059 +4293569733 +4293569493 +4293569265 +4293568695 +4293567144 +4293566508 +4293566235 +4293566229 +4293565704 +4293564855 +4293564639 +4293564243 +4293563799 +4293562623 +4293562350 +4293560535 +4293560388 +4293557379 +4293555798 +4293555678 +4293554769 +4293554760 +4293554280 +4293553830 +4293552315 +4293551745 +4293550920 +4293550779 +4293550743 +4293550434 +4293549018 +4293548973 +4293548904 +4293547653 +4293547149 +4293545049 +4293544704 +4293542433 +4293542394 +4293542175 +4293542088 +4293541614 +4293541560 +4293538605 +4293537633 +4293537555 +4293537090 +4293535905 +4293535350 +4293534918 +4293534840 +4293534579 +4293533733 +4293532668 +4293532605 +4293532299 +4293530769 +4293529185 +4293529134 +4293528963 +4293528744 +4293528120 +4293526644 +4293524775 +4293524460 +4293524178 +4293524154 +4293523869 +4293523473 +4293522684 +4293522570 +4293522345 +4293521643 +4293520554 +4293520134 +4293520005 +4293518088 +4293517824 +4293517749 +4293516888 +4293516795 +4293516234 +4293515964 +4293514575 +4293513453 +4293513420 +4293513288 +4293512475 +4293511983 +4293511278 +4293511125 +4293510030 +4293509970 +4293508869 +4293508800 +4293508224 +4293507714 +4293506595 +4293506568 +4293506169 +4293505545 +4293504993 +4293503634 +4293503628 +4293503073 +4293501390 +4293500820 +4293497538 +4293497229 +4293495078 +4293495060 +4293494799 +4293493905 +4293493200 +4293492645 +4293491415 +4293489738 +4293489039 +4293488853 +4293486498 +4293486303 +4293485658 +4293483360 +4293482688 +4293482319 +4293480498 +4293480159 +4293479190 +4293479058 +4293478644 +4293478509 +4293477699 +4293477159 +4293477123 +4293475905 +4293475719 +4293475044 +4293474414 +4293473589 +4293473535 +4293473493 +4293473448 +4293473040 +4293470895 +4293470439 +4293467703 +4293466905 +4293465525 +4293464133 +4293463875 +4293463704 +4293463524 +4293463368 +4293462900 +4293461304 +4293460743 +4293460275 +4293458688 +4293458658 +4293458625 +4293458025 +4293457923 +4293457530 +4293457500 +4293457188 +4293456663 +4293456243 +4293456135 +4293456090 +4293456075 +4293456054 +4293454299 +4293453954 +4293453699 +4293451599 +4293450423 +4293449844 +4293449103 +4293449055 +4293447660 +4293446454 +4293446445 +4293444924 +4293444783 +4293444258 +4293443874 +4293443700 +4293442908 +4293442098 +4293440970 +4293440670 +4293440283 +4293439254 +4293439050 +4293438885 +4293436518 +4293435720 +4293435180 +4293434925 +4293434523 +4293434325 +4293432849 +4293432288 +4293432099 +4293431934 +4293430965 +4293428943 +4293428469 +4293427608 +4293425613 +4293425223 +4293424434 +4293423669 +4293422190 +4293422094 +4293421308 +4293420300 +4293420153 +4293418779 +4293418569 +4293418515 +4293418428 +4293418419 +4293418335 +4293417555 +4293416550 +4293416259 +4293415929 +4293415425 +4293415395 +4293415020 +4293413409 +4293412515 +4293410898 +4293410463 +4293410379 +4293408735 +4293407028 +4293406488 +4293406089 +4293405840 +4293404808 +4293403899 +4293403509 +4293403155 +4293402984 +4293402564 +4293401433 +4293401328 +4293400809 +4293400284 +4293400098 +4293399549 +4293397458 +4293397233 +4293397044 +4293396813 +4293394470 +4293392550 +4293392328 +4293391473 +4293390405 +4293388239 +4293388224 +4293387804 +4293387150 +4293387015 +4293386214 +4293385878 +4293384870 +4293384213 +4293383844 +4293382995 +4293382989 +4293382089 +4293381564 +4293381099 +4293380763 +4293379179 +4293379095 +4293377703 +4293377634 +4293377088 +4293376503 +4293376440 +4293375633 +4293375558 +4293374463 +4293374394 +4293371568 +4293371505 +4293371004 +4293370830 +4293370509 +4293369465 +4293369375 +4293368715 +4293367140 +4293367014 +4293366888 +4293366099 +4293365718 +4293365058 +4293364734 +4293364725 +4293364680 +4293363780 +4293363378 +4293361929 +4293360708 +4293360585 +4293358938 +4293358755 +4293357729 +4293355983 +4293355125 +4293354294 +4293354264 +4293354075 +4293353175 +4293352413 +4293351678 +4293349980 +4293349620 +4293348330 +4293348045 +4293347964 +4293347730 +4293346884 +4293346608 +4293345525 +4293345255 +4293342219 +4293341538 +4293340953 +4293339450 +4293339363 +4293339084 +4293338574 +4293337284 +4293336474 +4293336225 +4293335805 +4293335343 +4293333339 +4293332919 +4293332700 +4293331944 +4293331395 +4293330699 +4293330459 +4293329928 +4293329808 +4293329529 +4293329223 +4293329118 +4293328194 +4293327813 +4293327750 +4293327438 +4293326283 +4293325074 +4293324993 +4293324708 +4293324273 +4293322458 +4293321213 +4293320748 +4293319533 +4293318588 +4293318270 +4293317904 +4293315834 +4293315264 +4293314934 +4293314019 +4293313440 +4293313344 +4293313050 +4293312525 +4293310890 +4293310524 +4293310455 +4293309258 +4293308820 +4293308634 +4293308358 +4293307455 +4293307443 +4293306633 +4293306324 +4293306120 +4293305850 +4293305694 +4293305589 +4293303924 +4293303363 +4293302649 +4293301335 +4293300999 +4293299883 +4293299268 +4293298953 +4293298170 +4293298044 +4293297714 +4293297618 +4293297408 +4293297174 +4293296304 +4293296238 +4293293955 +4293293775 +4293293763 +4293291939 +4293291870 +4293290505 +4293289794 +4293289440 +4293289035 +4293287724 +4293287319 +4293287025 +4293286878 +4293285774 +4293283449 +4293282993 +4293282000 +4293280383 +4293280089 +4293279168 +4293278004 +4293277539 +4293277065 +4293276945 +4293276519 +4293275598 +4293274788 +4293272640 +4293272565 +4293272229 +4293269904 +4293269340 +4293269130 +4293268533 +4293268134 +4293267285 +4293265458 +4293265404 +4293264753 +4293263304 +4293262839 +4293262134 +4293261819 +4293261003 +4293259638 +4293258213 +4293257703 +4293257295 +4293256029 +4293255930 +4293254880 +4293254610 +4293253875 +4293253320 +4293252804 +4293252435 +4293250779 +4293246789 +4293246300 +4293244989 +4293244389 +4293244014 +4293243240 +4293242940 +4293242064 +4293241770 +4293240840 +4293240105 +4293239208 +4293238935 +4293237408 +4293237390 +4293236370 +4293235509 +4293235500 +4293231978 +4293231435 +4293231144 +4293230238 +4293230199 +4293230028 +4293229158 +4293226563 +4293226353 +4293226134 +4293225474 +4293225279 +4293224460 +4293222900 +4293222864 +4293222204 +4293222174 +4293221688 +4293221304 +4293219720 +4293218655 +4293217683 +4293216669 +4293216189 +4293215214 +4293210855 +4293210768 +4293209964 +4293209448 +4293209403 +4293208344 +4293208095 +4293208023 +4293207684 +4293207255 +4293206049 +4293205665 +4293205068 +4293204378 +4293204348 +4293203829 +4293203475 +4293202938 +4293202494 +4293200199 +4293199629 +4293199218 +4293197895 +4293197793 +4293195738 +4293195630 +4293195399 +4293194895 +4293193929 +4293193719 +4293191670 +4293191538 +4293191475 +4293191385 +4293185949 +4293185403 +4293185244 +4293183915 +4293182799 +4293182115 +4293181815 +4293181593 +4293181488 +4293181068 +4293179043 +4293178683 +4293177195 +4293176559 +4293175914 +4293175488 +4293174648 +4293174165 +4293173799 +4293173739 +4293173325 +4293173025 +4293171828 +4293171690 +4293171120 +4293170418 +4293168225 +4293166170 +4293165843 +4293165798 +4293165105 +4293164823 +4293163983 +4293162378 +4293161484 +4293161214 +4293160929 +4293159135 +4293158919 +4293157428 +4293156990 +4293156825 +4293156633 +4293156465 +4293156330 +4293155754 +4293155088 +4293154923 +4293154545 +4293153555 +4293152934 +4293151980 +4293149829 +4293148950 +4293147738 +4293147603 +4293147270 +4293146415 +4293146319 +4293145179 +4293144549 +4293143853 +4293143820 +4293143499 +4293142479 +4293142404 +4293142293 +4293138288 +4293137703 +4293137220 +4293135909 +4293135843 +4293135420 +4293133599 +4293130938 +4293130035 +4293129729 +4293129270 +4293128034 +4293127929 +4293127368 +4293124095 +4293123075 +4293122358 +4293121698 +4293121425 +4293119523 +4293118689 +4293118038 +4293117930 +4293117864 +4293117855 +4293115884 +4293114729 +4293114225 +4293111879 +4293111663 +4293111360 +4293110934 +4293110580 +4293109929 +4293109584 +4293108183 +4293107229 +4293106014 +4293104970 +4293104724 +4293104103 +4293103848 +4293102915 +4293102645 +4293101349 +4293101163 +4293099369 +4293099273 +4293098298 +4293097830 +4293097530 +4293096969 +4293096783 +4293096648 +4293096444 +4293095829 +4293094938 +4293094914 +4293094518 +4293092853 +4293092454 +4293092289 +4293092088 +4293091860 +4293090858 +4293089604 +4293089580 +4293088449 +4293088233 +4293086979 +4293086913 +4293086175 +4293085854 +4293085824 +4293084993 +4293084624 +4293082650 +4293081684 +4293081369 +4293080583 +4293079593 +4293078054 +4293077778 +4293077229 +4293077178 +4293075993 +4293075813 +4293075528 +4293075510 +4293074589 +4293073113 +4293072819 +4293071550 +4293071238 +4293071079 +4293071040 +4293070608 +4293069753 +4293068793 +4293068514 +4293065649 +4293065544 +4293065304 +4293065160 +4293065034 +4293064905 +4293064884 +4293064698 +4293063549 +4293063480 +4293061965 +4293061605 +4293060663 +4293060495 +4293060234 +4293059694 +4293059394 +4293059340 +4293059175 +4293058005 +4293057348 +4293056844 +4293055788 +4293055665 +4293055005 +4293054774 +4293054540 +4293053754 +4293053730 +4293053400 +4293052758 +4293051969 +4293051738 +4293051024 +4293050370 +4293050319 +4293047715 +4293046785 +4293046590 +4293045720 +4293045585 +4293045564 +4293045099 +4293042690 +4293042615 +4293041934 +4293041103 +4293040338 +4293039519 +4293038634 +4293038193 +4293037608 +4293037353 +4293035988 +4293034590 +4293034563 +4293033549 +4293032535 +4293031839 +4293031755 +4293030840 +4293030720 +4293030678 +4293029793 +4293029244 +4293029088 +4293028794 +4293028440 +4293028164 +4293027438 +4293027000 +4293026694 +4293026184 +4293026160 +4293026058 +4293024645 +4293024429 +4293024345 +4293024015 +4293023220 +4293022719 +4293022614 +4293020850 +4293020334 +4293015264 +4293014529 +4293014514 +4293014484 +4293014073 +4293013710 +4293012900 +4293012528 +4293011985 +4293011100 +4293010083 +4293009024 +4293008949 +4293008439 +4293005703 +4293005493 +4293005193 +4293004428 +4293002448 +4293001803 +4293000318 +4293000129 +4292998230 +4292997189 +4292997159 +4292997093 +4292996988 +4292996823 +4292996133 +4292996085 +4292995560 +4292993820 +4292991765 +4292990439 +4292989545 +4292985990 +4292985639 +4292985255 +4292985108 +4292984730 +4292984340 +4292984115 +4292983875 +4292983560 +4292983275 +4292981199 +4292980749 +4292980719 +4292980350 +4292979615 +4292978985 +4292978838 +4292978175 +4292977743 +4292976609 +4292976204 +4292975790 +4292975553 +4292975295 +4292974479 +4292971779 +4292971749 +4292971515 +4292970174 +4292969880 +4292969799 +4292967999 +4292966730 +4292965893 +4292964648 +4292963424 +4292963094 +4292961645 +4292960154 +4292959350 +4292958969 +4292957718 +4292956494 +4292955960 +4292954499 +4292953608 +4292952720 +4292952564 +4292950539 +4292949660 +4292949165 +4292948994 +4292948583 +4292948103 +4292947575 +4292946519 +4292946168 +4292943780 +4292943630 +4292943003 +4292942403 +4292942370 +4292942160 +4292941905 +4292941614 +4292941290 +4292939139 +4292938914 +4292938860 +4292937144 +4292936100 +4292934918 +4292931483 +4292930643 +4292930400 +4292927448 +4292926035 +4292925873 +4292925573 +4292923860 +4292923215 +4292922324 +4292921520 +4292920383 +4292920329 +4292919960 +4292916789 +4292916684 +4292915733 +4292914389 +4292912823 +4292912730 +4292912274 +4292912049 +4292910345 +4292910159 +4292909934 +4292909475 +4292908803 +4292908380 +4292907150 +4292907123 +4292905440 +4292904750 +4292904285 +4292903859 +4292902134 +4292902005 +4292901210 +4292900568 +4292900328 +4292899323 +4292899035 +4292898945 +4292898615 +4292897388 +4292897379 +4292896998 +4292894439 +4292893983 +4292893029 +4292892744 +4292892645 +4292891319 +4292889858 +4292889774 +4292887863 +4292887593 +4292887248 +4292886060 +4292885940 +4292885208 +4292885133 +4292884050 +4292884044 +4292883489 +4292883138 +4292882559 +4292881614 +4292881608 +4292880849 +4292880528 +4292878929 +4292878533 +4292878485 +4292877858 +4292876808 +4292875644 +4292874960 +4292874330 +4292874048 +4292872869 +4292872449 +4292872074 +4292871198 +4292870895 +4292870505 +4292870043 +4292869575 +4292869278 +4292868405 +4292867580 +4292866173 +4292866140 +4292865900 +4292865885 +4292865114 +4292864955 +4292864565 +4292864325 +4292864169 +4292863839 +4292863044 +4292861679 +4292860683 +4292857929 +4292857134 +4292856504 +4292856114 +4292855343 +4292854863 +4292854719 +4292853663 +4292853528 +4292852373 +4292852220 +4292851848 +4292850894 +4292849244 +4292848320 +4292848239 +4292847525 +4292846568 +4292844753 +4292844720 +4292844678 +4292842683 +4292841159 +4292840760 +4292840343 +4292840289 +4292838525 +4292837193 +4292836995 +4292836575 +4292835645 +4292835114 +4292835015 +4292834790 +4292834745 +4292834628 +4292834388 +4292834334 +4292834148 +4292833488 +4292832435 +4292829945 +4292829273 +4292828553 +4292828160 +4292828025 +4292827980 +4292827140 +4292826273 +4292826228 +4292825769 +4292825064 +4292824590 +4292823804 +4292821128 +4292820699 +4292820393 +4292819994 +4292817693 +4292816913 +4292816529 +4292816319 +4292814405 +4292814330 +4292814063 +4292813028 +4292812908 +4292812083 +4292811690 +4292809479 +4292809239 +4292808918 +4292806434 +4292806230 +4292804055 +4292803800 +4292802909 +4292801799 +4292799705 +4292799120 +4292797818 +4292796633 +4292796054 +4292795079 +4292795043 +4292794488 +4292792748 +4292792700 +4292792580 +4292791254 +4292791200 +4292791125 +4292789844 +4292788359 +4292788188 +4292786778 +4292785518 +4292785455 +4292784144 +4292783580 +4292783529 +4292783508 +4292783178 +4292782413 +4292782329 +4292782119 +4292780730 +4292780304 +4292780208 +4292780115 +4292779863 +4292779539 +4292779524 +4292779350 +4292778759 +4292778663 +4292776668 +4292775114 +4292775000 +4292772690 +4292770065 +4292769453 +4292768985 +4292767605 +4292767575 +4292766603 +4292765190 +4292765160 +4292764764 +4292761623 +4292760639 +4292758965 +4292758890 +4292757729 +4292756343 +4292755689 +4292755293 +4292755134 +4292753223 +4292752395 +4292751288 +4292750874 +4292750070 +4292749725 +4292748549 +4292748495 +4292747340 +4292747100 +4292746095 +4292745960 +4292745318 +4292744994 +4292744730 +4292744655 +4292744025 +4292743974 +4292743728 +4292743113 +4292740968 +4292740683 +4292740038 +4292739543 +4292738799 +4292738208 +4292737734 +4292737545 +4292737293 +4292736024 +4292735793 +4292735235 +4292734164 +4292733933 +4292733915 +4292732940 +4292732388 +4292732193 +4292732130 +4292730714 +4292730615 +4292730429 +4292730090 +4292729913 +4292729523 +4292728779 +4292728530 +4292725083 +4292722215 +4292721795 +4292721768 +4292720949 +4292720379 +4292718375 +4292718318 +4292717724 +4292717583 +4292717400 +4292716455 +4292713680 +4292712423 +4292711214 +4292710149 +4292710110 +4292708883 +4292708763 +4292708139 +4292707908 +4292707494 +4292706264 +4292705613 +4292705373 +4292703129 +4292703108 +4292703045 +4292701854 +4292700609 +4292699958 +4292699949 +4292699460 +4292699298 +4292699148 +4292698203 +4292697744 +4292697384 +4292696838 +4292695770 +4292694999 +4292693838 +4292692920 +4292692725 +4292692419 +4292691348 +4292691129 +4292690778 +4292689905 +4292689848 +4292689593 +4292687913 +4292687685 +4292686650 +4292685138 +4292684640 +4292684514 +4292681925 +4292681328 +4292681028 +4292681004 +4292680698 +4292680479 +4292680374 +4292678754 +4292678439 +4292678355 +4292678064 +4292677653 +4292677053 +4292676774 +4292676708 +4292673948 +4292672538 +4292671734 +4292670915 +4292670870 +4292670798 +4292670588 +4292670504 +4292670144 +4292669265 +4292669064 +4292668764 +4292668338 +4292667744 +4292667273 +4292666955 +4292666379 +4292665725 +4292664738 +4292664363 +4292664003 +4292663025 +4292662704 +4292661633 +4292660403 +4292659545 +4292659323 +4292659194 +4292657178 +4292656578 +4292654913 +4292654520 +4292654154 +4292653959 +4292653650 +4292652285 +4292650440 +4292650425 +4292650083 +4292649600 +4292648214 +4292648079 +4292647338 +4292645133 +4292644308 +4292644224 +4292643915 +4292643519 +4292642823 +4292639694 +4292638869 +4292638845 +4292638680 +4292638293 +4292638038 +4292637975 +4292637408 +4292636220 +4292634750 +4292634204 +4292633565 +4292633280 +4292630880 +4292630874 +4292629914 +4292629488 +4292627088 +4292626350 +4292623740 +4292623179 +4292620314 +4292619570 +4292619525 +4292617584 +4292617245 +4292616579 +4292616324 +4292615859 +4292614980 +4292614620 +4292614383 +4292614053 +4292612988 +4292611533 +4292610150 +4292609559 +4292609208 +4292608143 +4292608008 +4292607558 +4292606928 +4292606199 +4292606058 +4292605668 +4292604609 +4292604555 +4292604063 +4292603853 +4292603388 +4292602653 +4292602560 +4292602158 +4292602095 +4292599575 +4292599359 +4292599269 +4292599263 +4292597868 +4292597814 +4292596965 +4292596185 +4292595450 +4292594460 +4292594190 +4292593668 +4292592909 +4292592900 +4292592543 +4292592195 +4292591184 +4292591115 +4292590875 +4292587530 +4292586240 +4292585310 +4292584878 +4292584443 +4292583615 +4292582799 +4292582493 +4292579394 +4292579094 +4292578935 +4292578734 +4292577393 +4292575908 +4292575899 +4292573859 +4292572788 +4292571864 +4292570880 +4292570154 +4292568378 +4292567709 +4292567613 +4292567595 +4292566563 +4292565963 +4292565828 +4292564745 +4292564445 +4292563425 +4292562450 +4292562333 +4292562315 +4292561733 +4292561724 +4292560923 +4292560833 +4292559999 +4292559540 +4292559039 +4292558508 +4292557500 +4292557398 +4292556540 +4292554638 +4292552535 +4292551254 +4292550783 +4292550405 +4292550225 +4292550093 +4292546790 +4292546109 +4292545404 +4292544564 +4292544375 +4292544063 +4292543823 +4292543205 +4292542794 +4292542479 +4292542155 +4292541525 +4292540850 +4292540370 +4292539989 +4292538648 +4292537058 +4292536974 +4292536344 +4292535918 +4292535810 +4292535213 +4292535114 +4292534088 +4292533470 +4292532495 +4292532243 +4292531718 +4292531643 +4292531274 +4292529804 +4292529678 +4292529414 +4292529048 +4292528745 +4292528604 +4292528505 +4292527758 +4292525865 +4292524599 +4292523600 +4292523429 +4292523399 +4292522010 +4292521329 +4292521323 +4292521185 +4292519295 +4292518824 +4292517654 +4292517645 +4292516310 +4292516259 +4292515548 +4292513454 +4292513034 +4292512908 +4292512743 +4292512440 +4292511804 +4292511345 +4292510025 +4292509269 +4292509035 +4292507523 +4292506629 +4292505120 +4292504775 +4292504460 +4292504433 +4292503404 +4292503314 +4292503263 +4292503230 +4292502669 +4292502150 +4292502135 +4292500758 +4292499210 +4292498973 +4292498358 +4292497698 +4292496894 +4292496663 +4292496564 +4292495739 +4292492853 +4292491845 +4292489670 +4292489004 +4292488149 +4292488014 +4292487069 +4292487039 +4292485443 +4292484999 +4292483730 +4292483538 +4292483373 +4292483145 +4292481168 +4292481033 +4292480904 +4292480268 +4292479710 +4292479089 +4292478168 +4292477769 +4292477724 +4292475948 +4292475183 +4292474703 +4292474334 +4292474004 +4292473794 +4292471613 +4292470308 +4292469489 +4292469450 +4292468598 +4292468064 +4292467368 +4292467128 +4292466858 +4292466645 +4292465589 +4292464209 +4292461908 +4292461005 +4292459715 +4292459088 +4292458614 +4292457753 +4292457489 +4292457333 +4292456139 +4292455809 +4292455188 +4292453385 +4292453334 +4292452965 +4292452155 +4292452020 +4292451414 +4292450508 +4292449929 +4292447970 +4292447670 +4292447265 +4292446998 +4292446908 +4292445510 +4292444499 +4292444268 +4292444043 +4292443725 +4292443338 +4292443050 +4292442873 +4292442360 +4292442090 +4292442084 +4292441943 +4292441634 +4292441448 +4292441439 +4292441373 +4292441340 +4292440599 +4292439789 +4292436819 +4292436588 +4292436510 +4292436399 +4292436300 +4292435409 +4292435148 +4292434833 +4292433663 +4292433144 +4292432445 +4292431485 +4292431095 +4292429718 +4292429484 +4292429319 +4292428665 +4292427228 +4292425500 +4292424720 +4292424585 +4292423883 +4292422323 +4292420364 +4292419458 +4292419230 +4292416644 +4292414544 +4292413818 +4292413485 +4292412705 +4292411553 +4292410563 +4292410368 +4292409759 +4292408778 +4292407479 +4292407218 +4292406285 +4292406105 +4292405928 +4292405850 +4292404788 +4292404458 +4292403513 +4292403435 +4292402313 +4292401104 +4292400033 +4292398983 +4292396655 +4292394909 +4292394639 +4292394168 +4292391750 +4292391669 +4292389449 +4292389395 +4292389254 +4292388219 +4292386215 +4292385333 +4292384745 +4292383590 +4292383119 +4292380749 +4292380515 +4292380314 +4292379399 +4292379264 +4292377239 +4292376954 +4292376654 +4292376345 +4292376318 +4292375295 +4292374878 +4292374569 +4292373678 +4292373648 +4292371578 +4292371494 +4292371263 +4292371110 +4292370498 +4292369079 +4292368335 +4292367084 +4292366154 +4292365929 +4292364894 +4292364795 +4292364774 +4292364480 +4292363955 +4292363685 +4292363379 +4292362788 +4292362338 +4292361174 +4292360898 +4292360565 +4292359788 +4292358825 +4292357730 +4292356290 +4292354493 +4292354220 +4292353743 +4292352744 +4292352195 +4292351640 +4292351505 +4292350149 +4292349720 +4292349204 +4292347815 +4292347710 +4292347554 +4292347050 +4292346939 +4292345715 +4292344938 +4292343999 +4292343510 +4292342355 +4292342349 +4292341989 +4292341053 +4292340885 +4292340018 +4292339268 +4292339064 +4292338863 +4292338533 +4292338239 +4292338098 +4292336289 +4292334924 +4292334345 +4292333109 +4292332458 +4292332260 +4292332140 +4292331513 +4292329569 +4292328258 +4292327985 +4292326758 +4292326128 +4292324685 +4292324079 +4292322789 +4292321958 +4292321538 +4292320899 +4292320539 +4292319093 +4292318835 +4292317944 +4292317209 +4292316780 +4292316318 +4292313723 +4292313660 +4292312688 +4292312178 +4292312163 +4292311203 +4292310063 +4292309745 +4292308968 +4292308800 +4292308698 +4292308209 +4292306469 +4292306103 +4292305104 +4292305050 +4292304504 +4292303433 +4292302755 +4292302188 +4292301585 +4292301429 +4292300775 +4292300628 +4292298564 +4292298339 +4292297799 +4292295369 +4292295045 +4292294928 +4292293974 +4292293914 +4292293410 +4292292354 +4292292303 +4292292243 +4292291709 +4292291430 +4292291205 +4292290575 +4292290329 +4292289240 +4292288829 +4292288460 +4292287884 +4292287425 +4292287023 +4292285970 +4292285499 +4292284863 +4292283894 +4292283588 +4292282709 +4292281614 +4292281404 +4292281299 +4292280510 +4292279649 +4292276565 +4292276268 +4292275293 +4292275239 +4292274789 +4292273478 +4292273328 +4292273070 +4292271135 +4292269989 +4292268804 +4292268729 +4292268405 +4292266989 +4292266713 +4292265234 +4292265075 +4292265033 +4292264088 +4292263428 +4292263335 +4292263239 +4292262525 +4292261718 +4292260929 +4292260398 +4292260218 +4292259378 +4292259114 +4292257740 +4292257080 +4292256300 +4292255238 +4292255229 +4292254704 +4292253000 +4292251440 +4292249925 +4292249298 +4292243073 +4292240184 +4292239575 +4292239563 +4292239104 +4292237598 +4292237544 +4292236410 +4292235930 +4292235150 +4292234604 +4292233569 +4292232768 +4292231823 +4292231184 +4292230200 +4292228058 +4292227890 +4292227518 +4292227014 +4292226735 +4292226504 +4292224035 +4292223780 +4292222700 +4292221809 +4292220528 +4292219475 +4292219100 +4292218083 +4292217534 +4292216460 +4292215443 +4292214093 +4292212410 +4292210490 +4292210469 +4292209053 +4292209050 +4292208189 +4292206110 +4292205483 +4292205450 +4292201028 +4292200995 +4292200725 +4292200590 +4292199735 +4292199495 +4292198970 +4292198679 +4292198343 +4292197404 +4292196810 +4292196699 +4292191644 +4292191590 +4292190894 +4292190735 +4292190495 +4292190120 +4292189763 +4292189280 +4292185278 +4292184963 +4292183259 +4292183178 +4292182728 +4292182503 +4292182338 +4292182110 +4292181243 +4292180598 +4292180094 +4292179179 +4292178675 +4292178339 +4292177349 +4292176938 +4292176524 +4292175483 +4292174190 +4292174160 +4292173833 +4292173623 +4292173290 +4292173275 +4292172468 +4292172279 +4292171709 +4292170989 +4292170965 +4292170683 +4292170224 +4292169663 +4292168415 +4292164170 +4292163624 +4292163183 +4292163018 +4292162148 +4292160543 +4292159298 +4292159268 +4292158095 +4292157618 +4292157273 +4292156904 +4292156658 +4292156574 +4292156049 +4292155878 +4292155014 +4292153904 +4292152089 +4292150799 +4292150574 +4292150460 +4292150358 +4292149695 +4292149485 +4292149344 +4292148804 +4292148189 +4292147919 +4292147838 +4292147550 +4292146893 +4292146584 +4292144883 +4292144424 +4292144163 +4292144160 +4292144085 +4292143245 +4292143044 +4292142114 +4292142009 +4292139750 +4292138940 +4292138289 +4292138214 +4292136744 +4292135244 +4292133534 +4292131488 +4292130918 +4292130270 +4292129733 +4292128758 +4292128575 +4292128293 +4292127279 +4292126145 +4292124738 +4292124474 +4292124144 +4292124054 +4292123394 +4292123073 +4292122164 +4292121555 +4292119275 +4292118984 +4292118948 +4292118663 +4292118549 +4292117304 +4292116683 +4292115708 +4292115705 +4292114958 +4292114673 +4292114484 +4292113875 +4292113629 +4292113593 +4292113539 +4292112093 +4292112075 +4292111829 +4292111403 +4292111325 +4292109939 +4292108313 +4292106255 +4292103534 +4292102424 +4292100714 +4292100105 +4292099820 +4292099670 +4292097159 +4292096598 +4292096478 +4292095545 +4292093778 +4292093709 +4292093349 +4292093190 +4292092839 +4292090253 +4292089788 +4292088918 +4292088549 +4292088120 +4292087859 +4292087544 +4292087178 +4292084175 +4292081895 +4292081403 +4292080320 +4292079909 +4292079840 +4292079324 +4292077854 +4292076438 +4292076024 +4292075520 +4292074125 +4292074029 +4292072940 +4292070915 +4292070795 +4292070333 +4292070210 +4292069454 +4292069208 +4292067804 +4292067690 +4292066640 +4292066439 +4292066190 +4292064048 +4292063739 +4292063553 +4292063448 +4292062500 +4292061618 +4292060463 +4292060169 +4292059968 +4292059308 +4292057544 +4292057484 +4292056635 +4292056263 +4292056020 +4292055414 +4292054373 +4292052213 +4292051343 +4292050698 +4292050680 +4292050344 +4292050059 +4292048388 +4292046225 +4292046009 +4292045565 +4292044833 +4292044104 +4292043855 +4292043834 +4292041785 +4292041488 +4292040180 +4292038113 +4292037315 +4292037078 +4292036355 +4292035539 +4292035095 +4292034174 +4292033880 +4292033625 +4292033394 +4292032248 +4292031990 +4292031093 +4292031003 +4292030883 +4292030868 +4292030583 +4292029623 +4292029224 +4292028798 +4292028114 +4292027568 +4292027124 +4292025993 +4292025963 +4292025930 +4292025918 +4292024745 +4292023410 +4292022789 +4292022603 +4292022453 +4292021544 +4292020305 +4292018244 +4292016939 +4292015208 +4292014959 +4292014170 +4292013384 +4292013213 +4292011113 +4292010738 +4292010213 +4292009643 +4292009214 +4292008245 +4292007795 +4292006349 +4292006265 +4292005563 +4292005143 +4292005014 +4292000880 +4292000193 +4291999785 +4291999599 +4291998693 +4291998390 +4291997088 +4291995819 +4291995783 +4291995468 +4291995279 +4291995063 +4291994565 +4291994448 +4291993905 +4291993314 +4291993278 +4291991595 +4291991100 +4291990815 +4291990008 +4291989765 +4291987203 +4291986954 +4291986153 +4291985850 +4291985703 +4291985673 +4291985670 +4291985235 +4291984959 +4291984773 +4291984770 +4291984644 +4291984605 +4291983648 +4291983018 +4291982955 +4291982694 +4291981134 +4291978440 +4291977375 +4291976949 +4291976430 +4291975404 +4291975359 +4291973745 +4291973673 +4291973259 +4291972299 +4291971969 +4291967625 +4291966629 +4291966395 +4291965600 +4291964073 +4291964013 +4291963893 +4291963779 +4291962009 +4291961283 +4291960044 +4291959699 +4291958970 +4291958103 +4291957263 +4291956063 +4291955904 +4291955709 +4291954500 +4291954155 +4291952664 +4291951959 +4291950795 +4291950558 +4291949385 +4291949025 +4291948533 +4291946868 +4291946484 +4291946205 +4291945953 +4291945029 +4291944753 +4291944090 +4291943463 +4291943193 +4291942704 +4291942689 +4291942140 +4291941183 +4291941060 +4291940694 +4291939659 +4291938840 +4291937805 +4291937340 +4291937208 +4291936620 +4291936563 +4291935945 +4291935705 +4291935333 +4291934610 +4291934379 +4291933848 +4291931685 +4291931469 +4291931124 +4291930560 +4291929459 +4291927848 +4291927704 +4291927494 +4291927128 +4291924815 +4291924638 +4291923198 +4291922115 +4291921998 +4291921383 +4291921269 +4291921110 +4291919670 +4291918809 +4291918440 +4291918170 +4291917180 +4291916898 +4291916760 +4291916688 +4291916574 +4291916004 +4291915230 +4291912905 +4291912899 +4291912824 +4291912674 +4291912593 +4291912203 +4291912050 +4291911984 +4291911174 +4291910610 +4291909818 +4291906659 +4291906044 +4291905540 +4291903920 +4291902993 +4291902615 +4291901640 +4291901460 +4291901424 +4291900635 +4291900170 +4291899900 +4291899660 +4291899144 +4291898805 +4291897914 +4291896969 +4291894629 +4291894290 +4291893894 +4291892709 +4291892340 +4291890249 +4291889574 +4291888404 +4291887813 +4291886793 +4291886655 +4291886325 +4291885515 +4291885158 +4291884168 +4291884084 +4291883958 +4291883673 +4291882308 +4291881489 +4291881438 +4291880790 +4291878930 +4291878699 +4291877829 +4291876860 +4291876818 +4291876545 +4291876470 +4291876365 +4291876113 +4291874313 +4291873584 +4291872969 +4291872870 +4291872639 +4291871088 +4291870089 +4291868880 +4291868589 +4291868343 +4291868208 +4291868190 +4291865550 +4291864974 +4291863915 +4291863768 +4291862949 +4291862235 +4291861089 +4291860525 +4291859460 +4291858968 +4291857669 +4291855680 +4291855575 +4291854825 +4291854810 +4291852683 +4291851285 +4291851243 +4291850340 +4291850199 +4291850178 +4291849185 +4291848735 +4291847853 +4291847649 +4291847505 +4291845585 +4291845573 +4291845138 +4291844964 +4291844100 +4291843860 +4291841658 +4291841343 +4291840854 +4291840458 +4291839810 +4291838319 +4291838109 +4291837518 +4291834104 +4291833498 +4291833204 +4291831335 +4291830789 +4291828590 +4291827984 +4291826238 +4291825734 +4291825620 +4291825023 +4291823325 +4291822368 +4291820940 +4291820004 +4291819488 +4291817409 +4291816674 +4291814604 +4291814553 +4291813533 +4291812384 +4291810593 +4291810335 +4291810194 +4291808223 +4291807113 +4291807038 +4291805574 +4291802184 +4291802178 +4291800903 +4291800768 +4291800159 +4291799730 +4291799223 +4291797915 +4291797453 +4291796880 +4291796613 +4291795920 +4291794879 +4291793943 +4291793724 +4291792965 +4291792050 +4291791369 +4291791294 +4291790985 +4291790634 +4291789305 +4291788168 +4291788060 +4291787604 +4291786995 +4291786575 +4291785030 +4291783548 +4291783464 +4291781550 +4291778823 +4291778778 +4291778529 +4291777539 +4291776504 +4291775313 +4291774674 +4291773825 +4291772808 +4291772154 +4291771515 +4291771368 +4291770693 +4291770399 +4291770075 +4291769208 +4291768125 +4291767864 +4291767654 +4291767534 +4291767348 +4291766730 +4291766058 +4291766013 +4291763619 +4291763370 +4291762968 +4291762293 +4291762230 +4291762035 +4291759779 +4291757214 +4291756689 +4291753980 +4291753800 +4291753110 +4291752984 +4291752660 +4291752345 +4291751880 +4291751733 +4291749888 +4291748859 +4291748385 +4291747473 +4291747143 +4291747095 +4291746819 +4291746684 +4291746030 +4291744644 +4291744338 +4291743414 +4291742544 +4291742028 +4291741620 +4291741599 +4291740435 +4291740123 +4291739844 +4291739739 +4291739460 +4291739373 +4291738533 +4291735980 +4291735944 +4291735530 +4291735500 +4291734405 +4291734363 +4291734159 +4291732938 +4291732599 +4291731639 +4291730850 +4291730148 +4291729980 +4291729788 +4291729623 +4291729053 +4291728588 +4291727868 +4291727595 +4291726395 +4291726293 +4291725810 +4291723785 +4291723485 +4291723155 +4291722513 +4291720818 +4291720350 +4291718913 +4291718175 +4291718088 +4291717674 +4291716654 +4291716483 +4291716108 +4291715274 +4291714899 +4291714869 +4291714665 +4291714239 +4291714119 +4291714110 +4291713483 +4291712634 +4291712235 +4291711434 +4291710993 +4291710009 +4291709988 +4291708935 +4291708569 +4291706910 +4291706835 +4291706613 +4291706460 +4291706439 +4291705365 +4291705323 +4291703745 +4291703580 +4291703043 +4291701858 +4291701708 +4291701438 +4291700589 +4291699944 +4291696080 +4291696050 +4291695309 +4291694298 +4291694130 +4291693554 +4291693089 +4291692975 +4291692714 +4291688925 +4291687839 +4291687680 +4291686810 +4291685649 +4291684995 +4291684794 +4291683774 +4291683690 +4291682598 +4291682493 +4291682430 +4291681974 +4291681905 +4291681014 +4291679349 +4291677948 +4291677810 +4291677579 +4291676823 +4291675473 +4291674693 +4291674534 +4291673910 +4291673079 +4291671600 +4291671534 +4291670703 +4291670379 +4291669725 +4291669599 +4291668795 +4291666449 +4291665594 +4291665588 +4291665204 +4291664238 +4291663740 +4291663725 +4291663713 +4291661118 +4291659753 +4291659585 +4291659084 +4291658820 +4291657875 +4291656435 +4291656285 +4291655313 +4291654134 +4291653729 +4291652154 +4291652070 +4291651713 +4291651533 +4291649829 +4291649460 +4291648194 +4291647804 +4291646229 +4291645809 +4291644603 +4291643874 +4291641963 +4291641345 +4291640325 +4291639185 +4291637883 +4291637775 +4291637754 +4291637748 +4291636818 +4291636020 +4291633608 +4291633554 +4291633368 +4291632990 +4291632558 +4291632555 +4291632108 +4291631085 +4291631070 +4291630803 +4291630059 +4291630035 +4291628925 +4291628040 +4291627938 +4291626198 +4291625568 +4291624734 +4291623288 +4291622340 +4291622193 +4291621344 +4291620528 +4291620249 +4291619394 +4291619139 +4291618968 +4291615194 +4291614708 +4291614558 +4291613289 +4291611969 +4291611813 +4291611285 +4291608735 +4291607754 +4291607253 +4291606098 +4291604703 +4291604484 +4291604043 +4291603578 +4291602309 +4291601454 +4291601403 +4291599360 +4291598004 +4291597869 +4291597554 +4291597275 +4291597113 +4291594935 +4291592598 +4291591179 +4291590915 +4291590369 +4291590135 +4291589868 +4291588644 +4291588545 +4291588143 +4291587180 +4291587144 +4291585464 +4291584393 +4291584048 +4291583238 +4291582335 +4291581285 +4291580994 +4291579278 +4291579260 +4291577199 +4291576470 +4291574790 +4291574325 +4291574253 +4291573020 +4291572768 +4291571679 +4291571145 +4291571049 +4291570389 +4291569924 +4291569645 +4291568970 +4291568919 +4291568769 +4291568580 +4291567938 +4291567680 +4291567509 +4291564908 +4291564473 +4291563405 +4291562763 +4291561359 +4291561140 +4291560453 +4291559310 +4291558344 +4291557798 +4291556724 +4291556634 +4291556265 +4291556205 +4291556199 +4291555959 +4291555434 +4291554813 +4291553133 +4291552980 +4291552263 +4291552140 +4291551474 +4291551363 +4291550088 +4291549638 +4291549575 +4291548870 +4291547685 +4291546593 +4291545753 +4291545678 +4291545258 +4291544454 +4291543098 +4291542975 +4291542384 +4291541619 +4291541310 +4291539339 +4291538649 +4291537743 +4291537389 +4291536885 +4291535379 +4291532148 +4291529793 +4291528779 +4291528653 +4291528053 +4291527288 +4291526613 +4291526589 +4291525668 +4291524930 +4291524195 +4291523340 +4291523274 +4291522959 +4291522770 +4291522674 +4291522479 +4291522044 +4291519314 +4291518834 +4291518348 +4291517994 +4291514955 +4291514253 +4291512714 +4291512564 +4291512015 +4291510068 +4291508808 +4291508025 +4291506978 +4291506960 +4291506864 +4291506744 +4291504785 +4291504188 +4291504008 +4291503324 +4291502319 +4291501683 +4291499835 +4291497435 +4291496328 +4291495503 +4291494270 +4291491645 +4291488918 +4291488810 +4291488459 +4291487439 +4291486638 +4291486440 +4291486233 +4291483794 +4291483629 +4291482954 +4291482930 +4291482018 +4291480674 +4291480068 +4291479828 +4291479303 +4291478859 +4291478430 +4291477695 +4291476603 +4291476594 +4291475835 +4291475628 +4291475625 +4291475583 +4291473810 +4291473198 +4291471695 +4291471218 +4291470765 +4291469418 +4291468809 +4291466595 +4291466373 +4291466058 +4291465680 +4291465065 +4291465023 +4291464918 +4291464693 +4291464408 +4291461384 +4291459389 +4291459128 +4291457754 +4291456680 +4291455189 +4291454343 +4291454265 +4291453929 +4291452969 +4291452684 +4291452609 +4291451589 +4291450773 +4291449543 +4291449240 +4291448889 +4291448595 +4291448055 +4291445544 +4291445394 +4291445214 +4291443099 +4291441629 +4291441125 +4291441035 +4291440690 +4291440138 +4291439874 +4291439733 +4291438665 +4291438149 +4291436553 +4291435335 +4291434993 +4291434708 +4291434243 +4291434045 +4291431663 +4291429593 +4291429065 +4291428099 +4291427838 +4291426830 +4291425048 +4291424823 +4291424268 +4291420770 +4291420398 +4291420335 +4291419945 +4291419900 +4291419825 +4291419510 +4291419390 +4291415688 +4291415565 +4291415523 +4291414734 +4291414623 +4291412874 +4291411113 +4291410858 +4291410264 +4291410219 +4291410144 +4291409823 +4291409394 +4291409220 +4291407909 +4291407414 +4291407114 +4291406295 +4291405893 +4291403949 +4291403718 +4291403550 +4291402284 +4291399719 +4291397988 +4291397715 +4291396569 +4291396254 +4291394469 +4291393314 +4291393119 +4291392525 +4291391343 +4291390815 +4291390284 +4291387833 +4291387065 +4291386879 +4291385778 +4291385763 +4291384839 +4291384548 +4291384404 +4291384083 +4291381263 +4291380864 +4291380378 +4291379805 +4291379190 +4291378923 +4291378764 +4291378005 +4291377504 +4291375005 +4291374960 +4291374030 +4291373724 +4291373148 +4291372554 +4291371495 +4291371444 +4291369923 +4291369680 +4291369584 +4291369323 +4291368900 +4291368063 +4291367664 +4291364745 +4291364463 +4291363293 +4291362384 +4291361955 +4291361280 +4291360920 +4291360743 +4291360653 +4291360548 +4291360158 +4291359084 +4291357995 +4291357818 +4291357650 +4291356198 +4291355754 +4291355244 +4291354518 +4291352820 +4291351959 +4291351263 +4291350969 +4291350603 +4291350024 +4291349355 +4291348458 +4291348398 +4291346349 +4291346193 +4291346100 +4291345968 +4291345953 +4291344693 +4291344543 +4291344300 +4291342158 +4291341648 +4291341204 +4291340964 +4291340604 +4291340115 +4291339755 +4291338828 +4291337730 +4291336974 +4291335429 +4291335300 +4291334865 +4291333788 +4291333149 +4291332855 +4291331904 +4291331460 +4291330014 +4291329678 +4291329285 +4291329189 +4291328394 +4291328013 +4291327935 +4291323690 +4291323648 +4291323525 +4291322640 +4291322619 +4291322355 +4291322073 +4291320789 +4291319934 +4291318590 +4291318218 +4291317513 +4291317078 +4291315299 +4291314408 +4291313745 +4291313178 +4291312674 +4291311975 +4291311825 +4291311249 +4291310814 +4291310049 +4291309728 +4291309098 +4291306164 +4291305234 +4291304895 +4291304808 +4291302783 +4291302105 +4291301103 +4291301055 +4291300704 +4291300359 +4291298658 +4291295238 +4291293264 +4291293063 +4291292358 +4291292109 +4291290408 +4291290354 +4291290099 +4291289253 +4291288608 +4291288170 +4291287759 +4291286979 +4291286793 +4291286130 +4291284984 +4291284558 +4291283463 +4291282554 +4291282140 +4291281279 +4291280640 +4291279170 +4291278435 +4291278333 +4291277670 +4291276389 +4291273215 +4291272000 +4291271433 +4291269954 +4291269639 +4291269519 +4291268940 +4291268499 +4291268484 +4291268145 +4291266903 +4291266225 +4291266189 +4291265814 +4291265550 +4291262988 +4291262328 +4291261665 +4291261170 +4291260060 +4291259145 +4291258014 +4291256994 +4291255605 +4291254474 +4291254150 +4291253820 +4291253235 +4291251804 +4291251618 +4291250115 +4291249425 +4291249068 +4291248843 +4291248204 +4291247919 +4291247763 +4291245573 +4291245264 +4291245069 +4291243998 +4291243965 +4291243155 +4291242054 +4291240569 +4291240560 +4291240260 +4291240068 +4291239870 +4291238883 +4291237959 +4291236855 +4291235844 +4291233945 +4291233738 +4291233003 +4291232268 +4291230933 +4291230558 +4291230123 +4291229958 +4291229943 +4291226988 +4291226733 +4291226409 +4291225470 +4291225284 +4291224573 +4291224129 +4291224045 +4291222584 +4291222515 +4291220604 +4291220499 +4291219164 +4291218105 +4291217400 +4291215444 +4291215180 +4291215150 +4291214679 +4291214460 +4291213788 +4291212939 +4291212288 +4291212210 +4291207884 +4291205358 +4291205310 +4291204113 +4291203519 +4291203180 +4291202124 +4291201923 +4291199664 +4291199493 +4291198743 +4291198488 +4291197879 +4291197249 +4291197084 +4291196040 +4291194678 +4291194048 +4291193478 +4291193175 +4291192290 +4291192074 +4291191990 +4291191978 +4291190664 +4291190658 +4291190490 +4291189899 +4291188210 +4291185438 +4291185303 +4291184040 +4291183524 +4291183410 +4291182930 +4291182453 +4291182255 +4291181910 +4291179549 +4291179429 +4291177134 +4291175880 +4291175013 +4291174383 +4291174119 +4291173330 +4291172994 +4291171650 +4291171428 +4291170138 +4291169784 +4291169058 +4291168299 +4291168020 +4291167795 +4291167123 +4291166253 +4291165515 +4291164384 +4291164333 +4291163898 +4291163565 +4291162809 +4291162095 +4291158588 +4291157829 +4291157340 +4291157109 +4291156215 +4291154964 +4291154658 +4291152438 +4291151784 +4291151535 +4291150953 +4291150314 +4291148490 +4291147575 +4291147560 +4291147164 +4291146609 +4291145559 +4291144299 +4291143615 +4291143510 +4291143144 +4291143048 +4291141170 +4291140150 +4291139370 +4291139238 +4291138563 +4291138215 +4291137915 +4291136253 +4291135815 +4291135623 +4291134849 +4291134498 +4291133799 +4291132614 +4291131558 +4291131333 +4291125714 +4291124970 +4291124799 +4291123668 +4291123329 +4291123248 +4291122948 +4291122939 +4291122375 +4291121694 +4291121484 +4291121040 +4291119645 +4291119243 +4291118490 +4291118349 +4291117440 +4291116285 +4291113825 +4291113795 +4291112778 +4291111299 +4291110348 +4291109808 +4291109658 +4291109649 +4291109238 +4291108965 +4291108395 +4291106814 +4291106235 +4291105554 +4291104138 +4291103085 +4291102443 +4291101573 +4291101555 +4291101030 +4291100880 +4291100565 +4291099914 +4291098489 +4291097958 +4291097574 +4291097130 +4291096983 +4291096095 +4291095963 +4291095915 +4291095774 +4291095150 +4291095018 +4291094808 +4291093698 +4291092879 +4291092090 +4291090044 +4291089519 +4291089153 +4291087953 +4291087428 +4291086645 +4291086558 +4291086063 +4291085445 +4291085163 +4291084839 +4291083984 +4291083720 +4291081509 +4291081035 +4291080234 +4291079490 +4291079325 +4291078779 +4291078620 +4291077393 +4291077294 +4291076310 +4291075923 +4291075629 +4291075104 +4291075089 +4291073178 +4291072980 +4291072950 +4291071783 +4291071225 +4291071069 +4291070385 +4291069824 +4291069644 +4291068894 +4291068435 +4291068360 +4291064568 +4291064049 +4291063785 +4291063503 +4291063425 +4291060749 +4291060473 +4291060443 +4291058238 +4291057860 +4291056258 +4291055730 +4291054725 +4291053324 +4291051413 +4291050069 +4291049979 +4291049904 +4291049019 +4291048839 +4291046745 +4291046214 +4291044765 +4291043520 +4291043403 +4291042383 +4291042335 +4291038855 +4291038333 +4291037703 +4291036179 +4291036134 +4291034055 +4291032234 +4291031004 +4291030644 +4291030533 +4291030380 +4291030089 +4291029828 +4291029345 +4291028610 +4291028388 +4291026474 +4291025409 +4291025025 +4291024374 +4291023423 +4291023408 +4291022058 +4291021875 +4291020819 +4291020669 +4291020054 +4291019988 +4291018578 +4291018173 +4291017480 +4291015623 +4291015473 +4291015419 +4291015275 +4291013439 +4291013364 +4291011963 +4291010130 +4291008225 +4291008018 +4291006728 +4291006425 +4291006200 +4291006083 +4291006014 +4291004964 +4291003689 +4291002273 +4291001709 +4291001634 +4291001139 +4291000983 +4291000713 +4290999198 +4290999093 +4290998433 +4290998379 +4290996780 +4290996423 +4290996015 +4290993915 +4290993888 +4290993378 +4290992193 +4290992160 +4290992079 +4290988800 +4290988203 +4290987855 +4290987609 +4290987039 +4290986493 +4290985125 +4290984543 +4290984453 +4290981933 +4290981189 +4290978744 +4290978378 +4290978249 +4290977748 +4290976998 +4290976578 +4290976338 +4290975540 +4290974505 +4290974385 +4290974040 +4290973644 +4290973575 +4290973533 +4290973149 +4290972750 +4290972519 +4290971784 +4290970878 +4290970620 +4290970320 +4290969690 +4290968970 +4290968739 +4290965040 +4290964845 +4290962355 +4290960588 +4290960348 +4290958875 +4290958728 +4290958098 +4290957870 +4290956958 +4290956883 +4290956694 +4290955530 +4290955248 +4290954870 +4290953028 +4290952158 +4290951678 +4290949923 +4290949869 +4290949368 +4290948273 +4290945438 +4290944259 +4290944064 +4290944028 +4290943824 +4290943653 +4290942708 +4290941910 +4290941844 +4290941490 +4290939369 +4290938310 +4290938244 +4290937224 +4290936849 +4290936663 +4290935439 +4290935208 +4290934815 +4290933963 +4290933450 +4290932460 +4290930609 +4290928500 +4290927030 +4290926790 +4290926400 +4290926130 +4290926088 +4290925215 +4290923463 +4290922464 +4290922410 +4290922128 +4290921915 +4290921558 +4290921150 +4290920700 +4290919578 +4290918975 +4290918240 +4290917940 +4290917895 +4290916980 +4290916764 +4290915579 +4290914934 +4290914745 +4290914355 +4290913794 +4290913665 +4290913473 +4290912648 +4290912318 +4290912234 +4290911388 +4290911295 +4290910494 +4290909693 +4290909195 +4290909078 +4290908964 +4290908604 +4290906420 +4290906390 +4290906108 +4290905454 +4290903258 +4290903105 +4290903000 +4290902250 +4290901938 +4290901809 +4290901119 +4290899409 +4290899250 +4290898689 +4290898179 +4290897033 +4290896973 +4290896529 +4290895539 +4290895104 +4290893523 +4290891720 +4290891639 +4290891279 +4290890484 +4290890388 +4290890010 +4290888783 +4290888273 +4290886935 +4290885684 +4290885663 +4290885450 +4290885399 +4290884493 +4290884388 +4290881745 +4290878934 +4290878019 +4290877554 +4290877500 +4290876273 +4290875373 +4290873888 +4290872775 +4290870930 +4290868935 +4290868890 +4290866670 +4290865914 +4290864105 +4290863874 +4290863673 +4290863130 +4290862740 +4290862653 +4290860715 +4290859539 +4290859443 +4290858573 +4290858438 +4290857319 +4290857175 +4290855444 +4290852420 +4290852315 +4290852138 +4290850569 +4290849603 +4290849249 +4290849219 +4290847830 +4290846645 +4290845475 +4290844713 +4290844635 +4290843933 +4290843864 +4290843579 +4290842193 +4290841929 +4290841218 +4290840333 +4290840039 +4290839580 +4290839418 +4290837474 +4290836880 +4290836208 +4290836100 +4290835839 +4290835323 +4290834978 +4290834744 +4290834009 +4290833790 +4290832623 +4290831420 +4290831324 +4290831090 +4290831084 +4290829434 +4290827730 +4290826830 +4290824943 +4290824868 +4290824763 +4290824259 +4290822774 +4290822213 +4290821919 +4290820569 +4290819249 +4290816465 +4290815553 +4290815460 +4290815343 +4290815175 +4290814899 +4290812550 +4290812289 +4290812118 +4290811689 +4290811134 +4290810639 +4290808989 +4290807003 +4290806808 +4290803604 +4290803583 +4290802944 +4290800655 +4290800295 +4290798783 +4290796875 +4290795675 +4290794730 +4290794193 +4290793368 +4290791748 +4290790908 +4290789930 +4290789555 +4290789180 +4290789084 +4290788949 +4290788619 +4290788445 +4290787824 +4290787620 +4290787233 +4290786918 +4290786465 +4290784653 +4290784308 +4290783999 +4290783960 +4290782859 +4290782454 +4290781860 +4290781500 +4290780030 +4290779205 +4290778749 +4290778554 +4290778260 +4290777498 +4290776538 +4290776160 +4290775629 +4290775035 +4290775005 +4290771219 +4290770349 +4290769353 +4290768954 +4290768480 +4290768315 +4290766350 +4290766320 +4290766038 +4290765333 +4290763620 +4290762015 +4290761550 +4290760845 +4290759885 +4290759705 +4290758928 +4290758835 +4290756765 +4290756543 +4290756324 +4290755490 +4290753039 +4290752280 +4290752049 +4290751815 +4290750369 +4290748320 +4290747525 +4290747378 +4290745305 +4290745233 +4290744753 +4290743835 +4290742788 +4290742230 +4290741423 +4290741390 +4290739614 +4290739440 +4290739029 +4290736569 +4290734778 +4290733119 +4290732990 +4290731130 +4290731073 +4290730779 +4290729939 +4290726618 +4290726588 +4290725265 +4290722733 +4290722229 +4290721728 +4290720210 +4290718683 +4290718005 +4290717768 +4290717198 +4290716130 +4290715470 +4290715329 +4290713220 +4290711375 +4290711033 +4290709914 +4290708408 +4290708345 +4290706194 +4290706029 +4290705993 +4290705705 +4290705558 +4290703989 +4290701940 +4290701820 +4290699180 +4290698859 +4290698559 +4290698280 +4290697449 +4290696915 +4290696564 +4290696129 +4290696060 +4290694845 +4290694740 +4290694644 +4290694395 +4290694275 +4290694014 +4290693249 +4290690978 +4290688065 +4290687579 +4290686790 +4290686244 +4290685869 +4290684900 +4290683298 +4290681270 +4290681228 +4290680388 +4290679713 +4290679665 +4290679599 +4290676053 +4290675198 +4290674895 +4290674580 +4290674178 +4290673194 +4290672669 +4290672063 +4290672039 +4290670908 +4290670389 +4290669699 +4290669369 +4290668928 +4290668535 +4290666660 +4290666180 +4290665289 +4290665235 +4290664980 +4290664713 +4290664533 +4290664404 +4290664209 +4290661269 +4290660759 +4290660309 +4290659880 +4290659763 +4290658344 +4290658314 +4290655860 +4290652989 +4290652155 +4290651669 +4290651204 +4290649950 +4290649299 +4290648045 +4290647745 +4290645975 +4290645903 +4290644118 +4290643995 +4290642114 +4290641433 +4290641403 +4290641313 +4290638334 +4290637758 +4290637050 +4290635109 +4290634509 +4290633765 +4290633060 +4290632904 +4290632604 +4290631869 +4290631470 +4290631014 +4290630489 +4290629979 +4290629724 +4290629433 +4290628935 +4290628440 +4290628329 +4290627948 +4290626913 +4290626868 +4290624105 +4290623454 +4290623034 +4290621963 +4290621378 +4290618879 +4290616785 +4290616764 +4290615108 +4290615045 +4290614868 +4290614823 +4290614775 +4290614418 +4290613353 +4290613218 +4290612768 +4290612093 +4290611919 +4290611535 +4290610908 +4290610365 +4290607095 +4290606330 +4290605415 +4290605040 +4290602448 +4290598635 +4290598455 +4290598038 +4290597258 +4290596880 +4290596229 +4290595875 +4290594495 +4290594243 +4290594213 +4290593814 +4290593595 +4290592995 +4290592440 +4290590709 +4290589035 +4290588189 +4290585738 +4290584553 +4290582954 +4290581655 +4290581520 +4290580965 +4290576363 +4290576324 +4290575715 +4290575430 +4290575169 +4290573519 +4290573405 +4290572385 +4290572238 +4290571908 +4290571839 +4290571173 +4290570990 +4290570189 +4290568773 +4290568764 +4290568383 +4290567270 +4290566034 +4290564729 +4290563928 +4290563238 +4290562629 +4290562338 +4290562263 +4290561159 +4290561024 +4290559560 +4290559440 +4290559395 +4290558063 +4290557694 +4290556200 +4290555045 +4290554850 +4290554415 +4290554364 +4290553134 +4290552759 +4290551958 +4290550944 +4290550638 +4290550500 +4290550239 +4290549894 +4290548409 +4290546909 +4290546564 +4290546555 +4290546015 +4290545598 +4290545370 +4290544494 +4290543960 +4290543753 +4290543300 +4290542733 +4290541029 +4290540828 +4290540525 +4290539370 +4290539079 +4290538260 +4290537858 +4290537558 +4290535443 +4290535248 +4290534690 +4290533388 +4290532770 +4290531969 +4290531033 +4290529638 +4290529173 +4290529050 +4290526575 +4290524733 +4290524559 +4290523068 +4290521178 +4290520200 +4290518454 +4290517680 +4290516774 +4290515844 +4290515823 +4290515358 +4290514749 +4290513834 +4290512835 +4290512043 +4290509805 +4290509139 +4290506814 +4290503670 +4290503229 +4290500823 +4290500109 +4290497988 +4290497835 +4290497769 +4290497718 +4290497223 +4290496098 +4290496020 +4290495060 +4290494925 +4290494598 +4290493770 +4290491784 +4290491733 +4290491700 +4290490440 +4290490323 +4290489483 +4290489210 +4290489204 +4290487890 +4290487683 +4290487299 +4290486693 +4290486549 +4290485148 +4290484353 +4290483084 +4290482973 +4290482250 +4290481548 +4290481338 +4290480390 +4290480363 +4290479730 +4290479445 +4290479424 +4290478170 +4290477474 +4290475329 +4290473748 +4290473010 +4290472914 +4290472770 +4290472749 +4290472458 +4290472158 +4290471780 +4290470904 +4290468378 +4290466005 +4290465555 +4290465525 +4290464715 +4290464274 +4290463875 +4290463254 +4290462720 +4290461118 +4290460380 +4290460329 +4290460173 +4290459459 +4290458595 +4290457578 +4290456663 +4290454995 +4290452850 +4290452670 +4290452025 +4290450834 +4290450468 +4290450393 +4290447300 +4290446649 +4290446340 +4290446103 +4290445479 +4290444534 +4290443730 +4290443703 +4290442989 +4290442980 +4290442248 +4290442113 +4290441963 +4290441960 +4290439515 +4290438300 +4290437985 +4290437073 +4290436599 +4290434163 +4290433299 +4290432714 +4290432294 +4290432210 +4290432153 +4290431820 +4290431418 +4290430833 +4290428955 +4290428400 +4290428283 +4290428250 +4290428154 +4290427623 +4290427179 +4290427149 +4290426735 +4290426684 +4290425910 +4290425853 +4290425364 +4290425055 +4290423450 +4290422349 +4290422310 +4290421665 +4290420975 +4290420330 +4290420279 +4290420144 +4290419640 +4290419493 +4290419379 +4290418755 +4290418299 +4290416745 +4290416493 +4290414624 +4290412944 +4290411324 +4290409728 +4290407955 +4290405405 +4290404424 +4290404013 +4290403998 +4290403839 +4290403683 +4290403638 +4290402843 +4290402828 +4290402324 +4290401769 +4290401718 +4290400803 +4290400695 +4290399048 +4290398868 +4290398355 +4290398253 +4290398010 +4290397980 +4290397410 +4290397110 +4290395259 +4290395040 +4290394989 +4290394758 +4290394659 +4290393570 +4290393288 +4290392238 +4290392100 +4290391455 +4290390945 +4290390534 +4290390174 +4290388488 +4290388359 +4290388263 +4290386604 +4290386394 +4290386313 +4290385968 +4290385569 +4290385419 +4290385359 +4290384714 +4290383208 +4290382845 +4290382488 +4290382314 +4290382083 +4290380715 +4290380358 +4290380094 +4290379884 +4290378090 +4290377964 +4290377883 +4290377763 +4290377220 +4290376494 +4290376284 +4290375354 +4290374868 +4290374385 +4290372849 +4290372654 +4290370995 +4290370749 +4290369615 +4290369600 +4290369048 +4290368958 +4290368460 +4290366675 +4290365268 +4290364350 +4290361773 +4290361293 +4290361020 +4290359850 +4290359580 +4290358110 +4290357819 +4290357585 +4290356850 +4290356055 +4290355989 +4290355818 +4290354804 +4290354519 +4290354039 +4290353934 +4290353868 +4290353679 +4290353583 +4290352734 +4290351204 +4290350919 +4290350700 +4290350508 +4290349629 +4290349458 +4290349428 +4290349089 +4290348930 +4290347514 +4290347319 +4290346878 +4290346359 +4290345963 +4290345720 +4290345564 +4290344829 +4290344553 +4290342693 +4290341628 +4290341583 +4290340923 +4290340914 +4290340719 +4290339648 +4290338829 +4290337539 +4290337338 +4290337113 +4290336555 +4290334404 +4290334245 +4290333939 +4290333873 +4290333750 +4290332649 +4290330129 +4290329625 +4290328920 +4290328068 +4290327534 +4290327315 +4290327219 +4290326715 +4290326544 +4290326493 +4290324813 +4290324540 +4290323058 +4290322560 +4290322293 +4290322125 +4290322044 +4290321699 +4290321663 +4290320088 +4290320034 +4290319053 +4290318669 +4290318660 +4290318315 +4290314895 +4290314475 +4290314439 +4290313599 +4290312519 +4290312090 +4290311700 +4290309933 +4290309894 +4290309495 +4290308139 +4290307413 +4290306258 +4290305244 +4290304599 +4290304074 +4290303480 +4290303165 +4290303099 +4290302850 +4290300759 +4290300750 +4290300549 +4290300414 +4290299628 +4290299319 +4290299145 +4290298713 +4290297915 +4290297903 +4290296919 +4290296895 +4290296760 +4290296703 +4290296160 +4290295005 +4290294879 +4290294624 +4290293628 +4290293568 +4290293460 +4290292644 +4290291780 +4290291579 +4290290448 +4290289653 +4290289443 +4290288243 +4290288003 +4290285924 +4290285534 +4290281373 +4290279864 +4290279225 +4290278574 +4290278019 +4290276423 +4290275730 +4290275514 +4290274980 +4290273228 +4290272334 +4290271980 +4290271158 +4290269430 +4290268008 +4290266595 +4290265575 +4290264945 +4290264903 +4290264585 +4290263934 +4290263493 +4290263460 +4290261864 +4290261753 +4290261639 +4290261318 +4290260985 +4290259359 +4290257454 +4290257334 +4290257280 +4290256293 +4290256113 +4290254178 +4290254055 +4290253449 +4290253173 +4290253119 +4290252573 +4290251685 +4290251040 +4290250350 +4290249843 +4290246378 +4290246045 +4290244638 +4290244350 +4290244194 +4290243825 +4290242493 +4290242244 +4290241275 +4290239985 +4290239208 +4290238323 +4290236709 +4290235578 +4290235365 +4290233373 +4290230778 +4290230463 +4290229335 +4290229275 +4290228453 +4290226590 +4290224613 +4290224595 +4290224154 +4290222714 +4290222408 +4290222210 +4290222054 +4290222009 +4290221418 +4290220719 +4290219975 +4290219669 +4290218808 +4290217980 +4290217695 +4290217659 +4290216903 +4290216819 +4290216264 +4290215769 +4290213618 +4290213459 +4290210555 +4290210270 +4290208083 +4290207918 +4290207270 +4290206808 +4290206604 +4290204000 +4290202320 +4290201333 +4290198324 +4290197388 +4290196968 +4290196890 +4290196548 +4290196380 +4290194844 +4290193110 +4290191940 +4290191655 +4290189204 +4290189165 +4290188910 +4290188505 +4290188010 +4290187998 +4290186888 +4290186039 +4290185985 +4290185115 +4290183798 +4290183324 +4290183114 +4290182769 +4290180075 +4290179670 +4290179568 +4290177948 +4290177204 +4290176904 +4290175770 +4290171348 +4290168708 +4290168354 +4290167685 +4290167535 +4290167328 +4290167208 +4290166503 +4290163563 +4290162840 +4290162504 +4290162483 +4290159405 +4290158790 +4290156933 +4290156699 +4290156660 +4290156375 +4290154959 +4290154554 +4290154203 +4290152940 +4290150243 +4290149484 +4290148764 +4290148233 +4290148134 +4290147795 +4290147000 +4290146574 +4290144690 +4290142878 +4290142284 +4290141393 +4290141324 +4290141054 +4290140994 +4290140799 +4290140610 +4290137799 +4290137040 +4290136794 +4290135855 +4290135048 +4290133200 +4290131880 +4290131634 +4290131580 +4290130788 +4290130725 +4290129339 +4290128928 +4290128925 +4290128499 +4290128358 +4290128304 +4290127275 +4290126618 +4290125178 +4290124158 +4290123270 +4290123138 +4290121338 +4290121044 +4290120975 +4290120855 +4290120735 +4290120660 +4290120114 +4290119490 +4290118890 +4290118260 +4290118005 +4290116865 +4290116799 +4290115203 +4290115065 +4290114738 +4290114144 +4290113778 +4290112809 +4290112695 +4290111783 +4290111423 +4290109890 +4290108483 +4290108438 +4290107493 +4290107265 +4290106014 +4290105123 +4290104988 +4290102018 +4290101934 +4290101649 +4290101409 +4290101394 +4290100335 +4290099855 +4290099519 +4290099450 +4290099444 +4290099213 +4290097215 +4290097140 +4290095139 +4290093234 +4290093150 +4290092694 +4290092253 +4290091650 +4290091440 +4290091398 +4290090273 +4290089970 +4290089754 +4290089454 +4290089445 +4290089229 +4290088323 +4290087978 +4290087924 +4290087534 +4290087348 +4290086718 +4290085449 +4290085350 +4290084765 +4290083118 +4290080700 +4290079479 +4290078870 +4290078390 +4290078303 +4290076803 +4290075483 +4290074109 +4290074073 +4290073845 +4290073749 +4290072444 +4290072345 +4290072228 +4290071673 +4290071238 +4290070893 +4290069960 +4290069729 +4290068418 +4290067530 +4290066669 +4290065748 +4290063588 +4290062979 +4290061989 +4290061698 +4290059298 +4290059250 +4290059094 +4290058473 +4290058323 +4290058128 +4290057045 +4290056109 +4290055899 +4290051183 +4290051060 +4290050850 +4290050175 +4290049890 +4290048633 +4290048033 +4290047628 +4290047223 +4290046983 +4290043554 +4290042900 +4290041778 +4290040383 +4290039195 +4290039054 +4290035583 +4290034803 +4290033615 +4290032628 +4290031035 +4290029808 +4290029148 +4290028995 +4290028809 +4290028365 +4290028170 +4290027894 +4290026823 +4290026649 +4290026268 +4290023739 +4290021270 +4290020175 +4290020148 +4290019860 +4290019215 +4290015495 +4290015015 +4290014730 +4290014634 +4290013488 +4290012165 +4290010308 +4290010248 +4290010128 +4290009669 +4290009648 +4290009240 +4290008694 +4290007848 +4290006624 +4290005454 +4290005004 +4290004710 +4290004314 +4290004143 +4290004005 +4290003579 +4290001395 +4290000639 +4290000483 +4290000363 +4289999895 +4289999328 +4289998965 +4289998704 +4289998359 +4289997123 +4289995230 +4289993625 +4289993244 +4289992734 +4289992374 +4289992209 +4289992143 +4289991594 +4289991300 +4289989530 +4289986758 +4289984985 +4289983944 +4289982873 +4289981685 +4289979273 +4289977923 +4289977740 +4289976825 +4289976468 +4289976414 +4289976024 +4289974503 +4289974338 +4289974083 +4289973924 +4289969799 +4289968908 +4289968863 +4289968635 +4289968044 +4289967150 +4289966793 +4289965953 +4289965209 +4289964183 +4289962698 +4289962404 +4289960904 +4289960895 +4289960484 +4289960193 +4289959770 +4289958798 +4289955873 +4289955690 +4289955648 +4289953779 +4289953500 +4289953428 +4289952513 +4289951970 +4289950815 +4289949993 +4289949504 +4289948748 +4289948208 +4289947290 +4289946528 +4289946489 +4289946324 +4289945808 +4289945478 +4289945469 +4289945409 +4289943834 +4289943489 +4289943234 +4289942670 +4289942343 +4289942118 +4289941104 +4289940654 +4289940483 +4289939535 +4289939214 +4289936829 +4289936034 +4289935014 +4289934120 +4289933259 +4289932368 +4289931855 +4289931555 +4289931504 +4289931120 +4289930793 +4289929524 +4289929248 +4289929110 +4289928585 +4289928273 +4289928060 +4289927598 +4289927589 +4289927283 +4289927220 +4289927169 +4289927025 +4289925990 +4289925549 +4289923554 +4289922759 +4289922183 +4289922153 +4289922033 +4289921259 +4289920929 +4289920734 +4289920530 +4289920449 +4289920383 +4289919735 +4289919030 +4289918820 +4289917995 +4289916924 +4289915829 +4289915745 +4289915670 +4289914803 +4289913318 +4289912514 +4289911713 +4289911158 +4289911143 +4289910924 +4289910435 +4289909688 +4289907678 +4289907453 +4289907270 +4289905905 +4289905644 +4289905599 +4289905569 +4289904774 +4289903289 +4289902749 +4289901945 +4289901813 +4289900559 +4289900454 +4289900403 +4289899803 +4289898609 +4289898150 +4289897610 +4289897205 +4289896479 +4289896008 +4289895630 +4289894358 +4289893863 +4289893389 +4289892339 +4289892120 +4289891553 +4289890053 +4289889408 +4289889315 +4289888868 +4289888238 +4289887398 +4289886975 +4289885883 +4289885178 +4289884614 +4289883528 +4289883375 +4289883333 +4289882253 +4289881659 +4289880483 +4289879973 +4289879595 +4289879475 +4289877369 +4289876433 +4289875878 +4289872935 +4289872173 +4289871915 +4289871663 +4289871453 +4289871279 +4289870880 +4289870868 +4289870334 +4289869893 +4289869533 +4289869143 +4289869053 +4289868744 +4289868243 +4289866995 +4289866710 +4289866614 +4289866458 +4289866239 +4289864484 +4289863959 +4289863644 +4289863173 +4289862678 +4289861994 +4289859573 +4289859420 +4289857269 +4289857038 +4289856900 +4289856810 +4289855535 +4289855340 +4289854404 +4289853993 +4289853555 +4289853534 +4289853303 +4289853270 +4289852748 +4289852553 +4289852475 +4289850405 +4289848695 +4289848233 +4289846763 +4289846739 +4289846235 +4289845425 +4289844888 +4289843430 +4289843199 +4289842305 +4289842245 +4289841123 +4289841018 +4289840919 +4289840325 +4289838225 +4289837814 +4289836545 +4289835243 +4289834433 +4289834094 +4289833938 +4289833728 +4289833635 +4289832288 +4289831598 +4289831049 +4289829150 +4289828310 +4289828229 +4289826288 +4289824455 +4289823774 +4289823768 +4289823000 +4289821338 +4289820765 +4289819499 +4289818860 +4289818494 +4289815983 +4289815620 +4289815164 +4289813064 +4289812479 +4289811975 +4289811819 +4289810745 +4289810514 +4289809578 +4289809443 +4289808948 +4289808888 +4289808498 +4289808414 +4289807409 +4289806125 +4289804403 +4289803710 +4289803644 +4289803605 +4289803593 +4289803335 +4289802648 +4289802219 +4289802189 +4289800740 +4289800140 +4289799939 +4289799240 +4289798994 +4289798898 +4289798469 +4289797959 +4289797725 +4289797305 +4289797224 +4289797158 +4289796708 +4289796654 +4289796315 +4289796045 +4289795193 +4289795154 +4289793828 +4289792814 +4289791518 +4289791500 +4289791218 +4289790624 +4289790483 +4289790273 +4289789883 +4289788734 +4289787129 +4289786679 +4289786028 +4289785500 +4289785095 +4289784225 +4289783760 +4289782563 +4289781900 +4289781324 +4289779833 +4289776440 +4289776158 +4289775120 +4289774634 +4289774433 +4289774175 +4289772609 +4289771403 +4289771304 +4289770014 +4289767545 +4289767305 +4289767233 +4289765910 +4289764695 +4289764569 +4289764500 +4289764230 +4289764083 +4289763435 +4289763213 +4289762529 +4289760783 +4289760735 +4289760483 +4289760285 +4289755995 +4289754885 +4289754663 +4289754609 +4289753424 +4289753373 +4289753334 +4289753208 +4289752320 +4289751633 +4289751474 +4289750430 +4289750139 +4289749875 +4289749290 +4289748123 +4289747628 +4289745633 +4289745603 +4289745054 +4289743998 +4289743038 +4289742180 +4289741949 +4289740974 +4289740209 +4289739555 +4289739198 +4289738265 +4289737539 +4289736783 +4289735745 +4289735730 +4289734113 +4289733918 +4289732643 +4289732355 +4289732139 +4289731974 +4289731260 +4289731113 +4289730879 +4289727960 +4289726358 +4289726013 +4289725893 +4289725068 +4289725044 +4289724543 +4289723124 +4289722995 +4289722725 +4289722494 +4289722260 +4289721570 +4289721030 +4289719980 +4289718600 +4289718174 +4289717694 +4289717013 +4289716689 +4289716410 +4289714805 +4289712519 +4289711760 +4289710803 +4289710773 +4289709630 +4289709579 +4289708970 +4289706870 +4289706744 +4289705238 +4289705160 +4289705040 +4289704923 +4289702949 +4289702700 +4289702010 +4289701218 +4289700588 +4289699265 +4289699130 +4289697513 +4289696460 +4289696295 +4289696085 +4289695005 +4289694759 +4289694249 +4289692590 +4289692125 +4289691558 +4289691540 +4289690238 +4289688858 +4289686359 +4289685099 +4289684673 +4289684178 +4289684055 +4289683959 +4289683809 +4289683695 +4289682729 +4289681460 +4289680965 +4289680578 +4289679819 +4289677974 +4289677938 +4289677635 +4289677143 +4289674479 +4289673519 +4289672304 +4289672175 +4289670594 +4289669448 +4289669154 +4289668869 +4289668629 +4289668215 +4289668008 +4289667390 +4289665920 +4289665590 +4289665380 +4289665164 +4289665059 +4289664114 +4289663658 +4289663388 +4289662830 +4289662170 +4289662035 +4289660514 +4289660364 +4289659605 +4289655933 +4289655795 +4289655180 +4289655144 +4289654358 +4289653035 +4289652639 +4289651100 +4289651085 +4289650545 +4289650473 +4289649633 +4289649234 +4289648334 +4289648100 +4289647689 +4289646810 +4289645673 +4289645664 +4289644680 +4289643954 +4289643690 +4289643648 +4289643303 +4289642010 +4289641998 +4289641104 +4289638893 +4289637495 +4289636595 +4289636529 +4289636154 +4289635335 +4289633304 +4289633085 +4289632854 +4289631849 +4289630313 +4289630229 +4289630199 +4289629953 +4289629293 +4289628993 +4289628750 +4289627229 +4289626869 +4289625045 +4289625003 +4289624424 +4289624205 +4289622498 +4289620944 +4289620560 +4289620443 +4289620350 +4289619474 +4289618565 +4289617755 +4289615013 +4289613930 +4289613720 +4289613363 +4289612409 +4289612379 +4289612103 +4289611704 +4289611443 +4289609820 +4289609490 +4289607885 +4289607813 +4289607084 +4289606595 +4289605824 +4289603580 +4289602110 +4289601438 +4289601408 +4289600460 +4289600325 +4289600250 +4289599425 +4289598159 +4289596794 +4289596764 +4289596428 +4289596293 +4289595549 +4289595309 +4289595288 +4289591418 +4289589345 +4289589324 +4289589165 +4289588934 +4289588379 +4289587254 +4289587029 +4289586819 +4289583534 +4289582778 +4289582265 +4289582109 +4289579073 +4289578740 +4289578734 +4289578623 +4289577069 +4289575290 +4289575248 +4289574519 +4289574345 +4289574309 +4289574150 +4289574003 +4289573064 +4289572428 +4289571405 +4289571060 +4289571033 +4289570988 +4289570724 +4289569509 +4289569275 +4289568633 +4289567295 +4289566809 +4289563803 +4289563533 +4289563395 +4289562975 +4289561715 +4289561694 +4289561223 +4289561220 +4289560689 +4289560254 +4289559309 +4289558283 +4289556990 +4289556855 +4289555985 +4289555865 +4289554815 +4289554083 +4289553735 +4289553360 +4289552484 +4289552340 +4289552079 +4289551728 +4289550504 +4289546313 +4289545704 +4289545608 +4289545218 +4289544729 +4289544090 +4289543739 +4289542314 +4289540094 +4289538405 +4289538153 +4289537328 +4289536734 +4289536674 +4289535183 +4289534520 +4289534004 +4289533809 +4289533143 +4289532609 +4289532453 +4289532015 +4289531610 +4289531553 +4289529825 +4289529615 +4289528949 +4289528514 +4289527515 +4289526414 +4289525910 +4289523663 +4289522649 +4289521650 +4289520753 +4289518845 +4289518623 +4289518530 +4289516505 +4289515980 +4289515308 +4289514465 +4289513103 +4289511915 +4289509353 +4289508825 +4289508690 +4289508588 +4289508480 +4289507493 +4289507298 +4289507025 +4289506800 +4289506320 +4289505648 +4289504133 +4289503368 +4289503239 +4289501754 +4289501103 +4289500314 +4289500239 +4289499810 +4289498988 +4289498928 +4289497170 +4289495958 +4289495640 +4289495595 +4289495364 +4289495055 +4289494629 +4289494110 +4289493645 +4289492724 +4289492544 +4289492094 +4289491899 +4289491815 +4289487984 +4289487294 +4289486115 +4289484900 +4289484828 +4289482998 +4289482653 +4289481909 +4289480775 +4289480373 +4289478765 +4289477739 +4289477493 +4289476254 +4289474133 +4289473959 +4289473944 +4289471730 +4289470974 +4289470905 +4289469519 +4289468049 +4289467665 +4289467455 +4289467428 +4289464128 +4289463408 +4289461419 +4289459604 +4289457999 +4289456598 +4289456289 +4289453748 +4289453673 +4289453250 +4289453055 +4289451669 +4289451054 +4289450343 +4289450310 +4289450088 +4289449914 +4289449239 +4289448954 +4289448453 +4289448435 +4289448420 +4289447973 +4289447778 +4289447505 +4289445090 +4289444769 +4289444304 +4289443584 +4289442219 +4289441439 +4289440680 +4289440398 +4289440350 +4289439840 +4289438100 +4289437083 +4289436480 +4289435799 +4289434338 +4289432880 +4289432694 +4289431410 +4289430045 +4289429919 +4289429913 +4289429133 +4289428779 +4289428638 +4289428023 +4289427534 +4289426643 +4289426445 +4289424270 +4289424258 +4289423670 +4289422980 +4289422893 +4289421585 +4289420964 +4289420628 +4289419650 +4289419575 +4289419560 +4289419035 +4289418528 +4289417433 +4289416638 +4289416509 +4289415030 +4289413623 +4289413218 +4289412543 +4289411559 +4289409099 +4289408730 +4289407254 +4289405415 +4289405025 +4289403600 +4289403474 +4289403258 +4289401269 +4289401245 +4289400975 +4289399973 +4289399259 +4289397540 +4289396979 +4289395599 +4289395473 +4289395374 +4289394978 +4289394828 +4289393079 +4289392749 +4289392614 +4289392434 +4289389548 +4289388435 +4289388225 +4289386755 +4289386713 +4289386620 +4289384604 +4289383938 +4289383083 +4289382699 +4289382630 +4289382240 +4289381745 +4289380854 +4289379564 +4289378544 +4289377818 +4289376063 +4289372619 +4289372295 +4289371164 +4289370783 +4289370009 +4289369988 +4289369544 +4289368929 +4289367498 +4289366973 +4289365893 +4289365884 +4289365368 +4289364495 +4289363130 +4289360274 +4289360124 +4289359689 +4289359683 +4289359500 +4289357913 +4289356860 +4289356683 +4289356578 +4289356104 +4289355993 +4289354349 +4289353638 +4289352789 +4289352420 +4289352399 +4289351403 +4289350308 +4289349879 +4289348349 +4289348259 +4289343954 +4289343840 +4289343159 +4289342835 +4289341890 +4289341245 +4289340210 +4289339913 +4289339823 +4289339628 +4289338863 +4289338788 +4289337954 +4289336469 +4289336229 +4289335029 +4289333889 +4289333625 +4289333163 +4289332563 +4289332269 +4289331744 +4289331630 +4289330604 +4289328765 +4289327868 +4289327010 +4289326809 +4289326383 +4289326098 +4289325873 +4289325675 +4289325663 +4289325600 +4289323848 +4289323284 +4289323023 +4289322765 +4289322063 +4289322003 +4289321994 +4289320869 +4289320413 +4289319984 +4289319723 +4289319459 +4289319399 +4289316300 +4289315808 +4289314980 +4289314800 +4289313183 +4289313033 +4289311845 +4289311545 +4289309253 +4289309190 +4289309088 +4289308815 +4289306679 +4289306589 +4289305245 +4289305233 +4289305200 +4289305068 +4289304549 +4289304024 +4289303928 +4289303478 +4289303448 +4289303430 +4289302629 +4289302620 +4289301234 +4289300583 +4289299629 +4289299509 +4289299155 +4289298243 +4289297229 +4289296653 +4289296593 +4289295069 +4289294388 +4289293008 +4289292960 +4289292693 +4289292069 +4289290755 +4289289948 +4289289570 +4289289198 +4289289168 +4289287779 +4289287299 +4289286660 +4289286330 +4289285703 +4289285559 +4289285304 +4289285145 +4289284968 +4289284803 +4289284620 +4289282703 +4289280390 +4289279958 +4289279475 +4289279034 +4289278530 +4289277915 +4289277828 +4289277564 +4289275803 +4289274690 +4289273829 +4289272974 +4289272260 +4289269794 +4289269125 +4289268810 +4289268213 +4289266275 +4289265015 +4289264910 +4289264694 +4289264094 +4289263893 +4289263665 +4289263299 +4289263053 +4289262510 +4289259873 +4289259498 +4289259369 +4289258520 +4289257764 +4289256474 +4289255529 +4289255073 +4289254758 +4289254458 +4289253918 +4289253423 +4289252604 +4289252460 +4289252019 +4289250798 +4289250759 +4289250489 +4289250420 +4289249733 +4289249634 +4289249625 +4289249328 +4289249070 +4289248965 +4289248158 +4289246109 +4289245635 +4289245479 +4289244903 +4289244513 +4289243694 +4289242035 +4289241099 +4289239830 +4289239398 +4289237394 +4289237193 +4289236455 +4289236164 +4289235819 +4289235705 +4289234925 +4289233395 +4289232648 +4289232204 +4289231163 +4289231085 +4289230860 +4289230854 +4289229855 +4289228853 +4289228643 +4289228559 +4289227899 +4289227818 +4289227395 +4289226198 +4289226060 +4289225304 +4289224779 +4289224038 +4289223939 +4289223723 +4289222568 +4289221803 +4289221284 +4289219760 +4289219298 +4289218380 +4289217204 +4289216853 +4289216169 +4289215815 +4289214879 +4289213508 +4289213073 +4289212383 +4289210700 +4289210310 +4289209968 +4289209545 +4289208738 +4289208714 +4289207109 +4289203983 +4289203968 +4289202618 +4289202585 +4289201904 +4289201553 +4289200704 +4289200224 +4289198094 +4289197965 +4289197254 +4289196444 +4289196198 +4289196063 +4289194689 +4289193258 +4289191959 +4289189868 +4289189673 +4289189595 +4289189199 +4289189073 +4289188440 +4289188134 +4289187615 +4289187489 +4289187423 +4289186760 +4289186244 +4289185593 +4289184663 +4289182899 +4289181729 +4289181204 +4289181000 +4289179434 +4289178384 +4289178144 +4289178054 +4289177814 +4289174085 +4289173299 +4289172480 +4289172273 +4289171613 +4289170929 +4289170770 +4289170698 +4289170035 +4289167554 +4289167464 +4289165643 +4289165109 +4289164683 +4289164053 +4289162223 +4289161119 +4289160510 +4289160204 +4289159805 +4289159409 +4289157990 +4289156385 +4289155095 +4289154780 +4289154108 +4289153580 +4289153445 +4289152743 +4289152164 +4289152155 +4289151984 +4289151684 +4289150733 +4289149899 +4289149194 +4289148279 +4289147994 +4289147553 +4289146305 +4289146134 +4289145819 +4289145675 +4289143263 +4289142849 +4289142249 +4289142063 +4289142033 +4289141619 +4289141454 +4289141025 +4289140773 +4289139723 +4289138139 +4289137968 +4289136873 +4289134023 +4289133960 +4289133243 +4289133210 +4289132679 +4289131923 +4289129655 +4289129319 +4289127345 +4289127165 +4289126973 +4289126673 +4289126544 +4289126253 +4289125728 +4289125644 +4289125410 +4289124630 +4289124330 +4289123898 +4289123004 +4289122215 +4289122125 +4289122053 +4289121984 +4289120613 +4289119173 +4289118924 +4289118459 +4289118030 +4289117829 +4289117610 +4289117118 +4289116590 +4289116140 +4289115888 +4289115279 +4289113830 +4289112540 +4289112294 +4289110215 +4289108274 +4289108208 +4289107794 +4289107539 +4289106540 +4289106375 +4289106228 +4289105763 +4289104653 +4289104539 +4289103930 +4289102970 +4289102625 +4289101593 +4289100894 +4289100810 +4289100303 +4289099640 +4289099190 +4289098308 +4289098134 +4289096580 +4289096253 +4289095965 +4289095338 +4289094810 +4289093835 +4289091198 +4289090844 +4289087748 +4289087739 +4289087508 +4289086788 +4289085198 +4289083125 +4289080635 +4289079114 +4289078958 +4289078505 +4289078235 +4289077638 +4289077404 +4289077044 +4289076309 +4289075400 +4289072793 +4289072163 +4289071380 +4289071344 +4289070465 +4289069790 +4289069730 +4289069679 +4289069529 +4289069253 +4289068290 +4289067168 +4289066430 +4289066145 +4289066010 +4289062518 +4289060775 +4289060634 +4289059164 +4289057205 +4289056098 +4289055900 +4289052900 +4289051970 +4289049765 +4289049150 +4289048769 +4289047398 +4289046693 +4289045724 +4289044728 +4289044668 +4289043639 +4289042334 +4289041800 +4289040615 +4289040558 +4289040318 +4289039133 +4289039103 +4289038839 +4289038689 +4289038215 +4289036193 +4289035794 +4289035329 +4289035110 +4289034498 +4289034099 +4289033013 +4289032515 +4289031564 +4289029419 +4289029380 +4289029305 +4289029083 +4289027844 +4289027160 +4289026080 +4289026050 +4289025924 +4289025894 +4289025888 +4289024643 +4289024598 +4289023653 +4289022423 +4289022375 +4289018508 +4289018040 +4289016333 +4289016318 +4289015820 +4289015640 +4289014884 +4289014770 +4289013924 +4289012223 +4289010489 +4289010444 +4289010165 +4289008758 +4289008044 +4289007513 +4289007030 +4289006820 +4289006685 +4289006259 +4289006028 +4289004618 +4289003610 +4289001828 +4289000523 +4289000208 +4288999779 +4288999119 +4288997553 +4288996995 +4288995789 +4288995630 +4288995360 +4288995159 +4288994253 +4288994214 +4288993464 +4288992933 +4288992105 +4288991178 +4288990749 +4288990425 +4288989339 +4288989228 +4288988670 +4288987890 +4288987149 +4288986498 +4288986348 +4288984500 +4288984119 +4288980003 +4288979628 +4288979079 +4288978545 +4288977780 +4288976658 +4288975458 +4288974948 +4288974240 +4288974210 +4288973163 +4288972800 +4288972029 +4288971795 +4288971384 +4288969788 +4288969410 +4288969074 +4288967850 +4288967709 +4288967703 +4288967100 +4288967019 +4288965603 +4288965444 +4288965348 +4288965135 +4288964505 +4288963365 +4288963323 +4288963029 +4288962753 +4288959729 +4288959363 +4288959114 +4288958805 +4288958115 +4288958019 +4288956309 +4288954899 +4288953885 +4288953450 +4288952559 +4288950849 +4288949610 +4288949505 +4288949358 +4288947090 +4288946649 +4288945779 +4288944810 +4288943664 +4288941084 +4288940718 +4288940565 +4288934814 +4288933680 +4288933209 +4288932774 +4288932714 +4288932660 +4288932633 +4288932114 +4288930098 +4288930008 +4288929498 +4288928019 +4288927989 +4288927629 +4288926855 +4288926300 +4288925619 +4288925340 +4288923870 +4288923735 +4288922484 +4288921974 +4288921260 +4288921125 +4288920270 +4288920198 +4288919103 +4288917888 +4288915680 +4288915239 +4288913949 +4288912965 +4288912248 +4288911093 +4288909014 +4288907820 +4288906194 +4288905348 +4288904550 +4288903473 +4288902813 +4288902729 +4288901835 +4288901643 +4288901589 +4288900395 +4288899924 +4288897800 +4288897173 +4288896963 +4288896753 +4288893558 +4288893489 +4288891473 +4288890549 +4288889274 +4288886538 +4288886064 +4288885284 +4288883973 +4288883835 +4288883109 +4288882200 +4288881780 +4288880883 +4288880604 +4288880529 +4288880088 +4288879413 +4288878825 +4288875513 +4288875375 +4288874523 +4288871994 +4288871949 +4288870869 +4288870698 +4288870068 +4288868454 +4288868034 +4288866063 +4288865580 +4288865523 +4288865328 +4288864914 +4288864119 +4288863999 +4288862838 +4288862700 +4288862535 +4288862079 +4288861188 +4288860078 +4288859769 +4288859403 +4288859154 +4288859004 +4288858089 +4288857714 +4288857630 +4288856634 +4288856145 +4288856040 +4288855380 +4288854954 +4288854864 +4288854633 +4288853799 +4288853469 +4288852653 +4288851330 +4288851069 +4288851015 +4288850928 +4288849995 +4288849188 +4288849134 +4288847913 +4288847820 +4288846959 +4288846743 +4288846518 +4288845543 +4288844694 +4288843989 +4288843833 +4288843425 +4288843020 +4288842750 +4288842585 +4288841388 +4288841334 +4288839894 +4288839570 +4288839474 +4288839303 +4288838805 +4288838265 +4288837950 +4288837584 +4288837005 +4288836354 +4288835724 +4288835445 +4288835088 +4288834305 +4288834035 +4288834005 +4288833114 +4288833078 +4288832208 +4288831263 +4288830675 +4288829970 +4288829709 +4288829313 +4288828974 +4288828278 +4288827654 +4288826568 +4288826400 +4288825878 +4288825770 +4288823370 +4288823364 +4288821879 +4288820703 +4288820235 +4288819935 +4288819023 +4288818528 +4288816650 +4288816479 +4288816290 +4288815879 +4288814103 +4288813368 +4288812045 +4288811565 +4288811028 +4288810284 +4288810260 +4288810149 +4288809510 +4288809393 +4288807095 +4288806114 +4288805790 +4288805118 +4288804980 +4288804665 +4288804533 +4288803924 +4288803900 +4288803774 +4288803090 +4288802988 +4288801914 +4288801893 +4288801560 +4288801044 +4288800573 +4288800498 +4288799448 +4288796118 +4288795164 +4288792473 +4288792065 +4288790898 +4288789449 +4288786764 +4288786515 +4288785354 +4288784340 +4288782330 +4288782279 +4288781463 +4288780734 +4288780308 +4288780305 +4288779363 +4288779360 +4288778199 +4288777905 +4288776570 +4288776180 +4288775463 +4288775163 +4288775034 +4288774050 +4288773543 +4288773138 +4288772199 +4288772178 +4288772124 +4288771473 +4288770759 +4288770009 +4288769838 +4288769490 +4288769178 +4288767348 +4288765260 +4288764690 +4288763739 +4288763058 +4288762383 +4288761303 +4288760775 +4288759938 +4288759629 +4288759614 +4288759548 +4288758978 +4288758138 +4288757889 +4288757649 +4288757208 +4288755849 +4288755534 +4288755108 +4288754295 +4288753323 +4288752495 +4288750893 +4288750830 +4288746120 +4288745859 +4288745730 +4288745595 +4288745133 +4288744755 +4288744563 +4288744185 +4288743480 +4288742550 +4288742433 +4288742319 +4288742193 +4288741605 +4288740135 +4288739973 +4288738209 +4288737444 +4288737384 +4288737318 +4288737033 +4288735338 +4288734903 +4288734744 +4288734303 +4288734294 +4288734168 +4288734135 +4288734114 +4288733553 +4288733385 +4288730970 +4288730505 +4288730334 +4288730304 +4288729749 +4288729518 +4288729005 +4288728339 +4288727994 +4288726455 +4288726260 +4288725813 +4288725750 +4288725225 +4288724544 +4288724310 +4288724019 +4288723983 +4288721919 +4288720938 +4288720668 +4288719849 +4288718460 +4288717929 +4288717083 +4288715289 +4288714944 +4288714923 +4288712169 +4288710615 +4288709283 +4288709169 +4288709160 +4288709139 +4288708740 +4288708110 +4288708074 +4288708035 +4288707018 +4288706973 +4288705365 +4288705335 +4288702584 +4288702404 +4288701798 +4288701339 +4288699779 +4288699569 +4288698594 +4288697880 +4288696293 +4288695033 +4288694934 +4288693773 +4288692924 +4288692210 +4288691238 +4288690650 +4288689768 +4288689480 +4288689324 +4288688409 +4288688325 +4288688139 +4288686963 +4288686675 +4288685994 +4288682964 +4288682238 +4288681974 +4288681839 +4288681278 +4288679820 +4288679664 +4288679289 +4288678434 +4288677744 +4288677678 +4288676049 +4288675593 +4288675005 +4288672884 +4288671528 +4288670949 +4288670280 +4288670013 +4288669458 +4288669293 +4288668243 +4288668174 +4288668105 +4288667460 +4288667193 +4288665990 +4288664358 +4288664040 +4288664028 +4288662855 +4288662075 +4288661634 +4288661205 +4288660374 +4288660230 +4288659864 +4288659639 +4288659054 +4288658499 +4288658034 +4288657875 +4288656825 +4288656384 +4288654944 +4288653825 +4288651059 +4288650378 +4288649055 +4288648944 +4288647909 +4288647879 +4288646523 +4288644645 +4288644534 +4288643784 +4288643268 +4288643199 +4288642368 +4288642329 +4288641939 +4288641369 +4288641075 +4288641003 +4288640415 +4288638453 +4288637718 +4288636668 +4288635378 +4288633530 +4288633383 +4288632999 +4288632225 +4288631880 +4288631208 +4288630848 +4288630419 +4288630314 +4288630083 +4288629675 +4288627974 +4288627584 +4288625664 +4288625598 +4288625400 +4288625169 +4288624833 +4288624548 +4288622379 +4288622229 +4288621884 +4288621245 +4288619940 +4288619763 +4288618995 +4288618683 +4288618650 +4288618305 +4288618200 +4288617675 +4288617345 +4288616799 +4288616073 +4288615509 +4288615500 +4288615239 +4288612665 +4288611888 +4288611045 +4288610859 +4288610808 +4288609038 +4288608609 +4288608450 +4288608420 +4288604199 +4288603098 +4288602804 +4288601709 +4288601343 +4288601274 +4288599885 +4288598508 +4288597320 +4288597248 +4288596504 +4288594194 +4288592430 +4288590843 +4288590003 +4288589955 +4288584984 +4288583340 +4288583334 +4288582779 +4288582500 +4288582443 +4288580898 +4288580028 +4288579398 +4288579134 +4288575354 +4288574949 +4288574268 +4288573140 +4288573128 +4288572975 +4288571838 +4288571619 +4288571265 +4288570293 +4288569888 +4288569573 +4288569333 +4288569183 +4288569144 +4288568889 +4288568724 +4288567713 +4288567428 +4288566360 +4288566078 +4288564194 +4288562943 +4288562484 +4288562439 +4288561914 +4288561908 +4288560624 +4288560603 +4288560534 +4288558848 +4288558734 +4288558389 +4288557033 +4288555635 +4288554990 +4288554534 +4288554453 +4288554345 +4288553835 +4288553328 +4288553193 +4288551828 +4288550829 +4288550070 +4288549119 +4288546524 +4288545948 +4288544778 +4288542585 +4288542315 +4288540200 +4288539900 +4288539858 +4288537263 +4288537254 +4288537020 +4288535739 +4288535733 +4288535340 +4288534299 +4288533609 +4288533129 +4288532883 +4288532718 +4288531974 +4288530114 +4288529475 +4288527540 +4288524975 +4288524885 +4288524294 +4288523568 +4288522344 +4288521795 +4288521579 +4288520904 +4288519530 +4288518690 +4288518333 +4288517883 +4288517268 +4288516650 +4288516608 +4288515258 +4288514700 +4288514529 +4288514364 +4288513734 +4288513260 +4288512963 +4288512084 +4288510863 +4288510758 +4288510395 +4288509603 +4288509480 +4288508649 +4288508493 +4288507329 +4288506963 +4288506699 +4288506099 +4288505124 +4288504878 +4288503348 +4288502373 +4288498773 +4288498128 +4288497954 +4288495449 +4288495095 +4288493565 +4288492944 +4288491789 +4288491159 +4288490709 +4288489584 +4288485393 +4288484994 +4288483815 +4288483533 +4288482855 +4288482525 +4288482519 +4288482075 +4288481193 +4288481088 +4288480869 +4288480245 +4288479720 +4288478718 +4288478169 +4288477740 +4288477620 +4288476933 +4288476648 +4288476384 +4288475484 +4288475169 +4288474464 +4288473519 +4288473510 +4288473444 +4288473210 +4288472823 +4288472613 +4288471404 +4288470843 +4288470648 +4288470243 +4288469358 +4288467294 +4288467114 +4288466400 +4288466160 +4288466040 +4288465419 +4288464234 +4288462650 +4288462053 +4288460928 +4288460748 +4288459605 +4288457304 +4288455960 +4288455219 +4288455093 +4288453200 +4288452825 +4288451889 +4288451868 +4288451475 +4288449609 +4288449189 +4288447278 +4288446345 +4288446309 +4288444593 +4288443153 +4288442553 +4288441503 +4288441398 +4288440663 +4288440429 +4288439310 +4288439274 +4288438665 +4288438308 +4288437558 +4288434654 +4288434303 +4288433850 +4288433505 +4288431600 +4288431594 +4288431405 +4288430574 +4288430469 +4288429920 +4288429893 +4288429683 +4288427508 +4288427403 +4288427109 +4288426425 +4288426023 +4288425585 +4288424535 +4288424034 +4288423965 +4288421238 +4288421145 +4288421025 +4288419000 +4288418604 +4288417710 +4288417098 +4288416744 +4288416459 +4288413948 +4288413309 +4288413225 +4288413108 +4288412418 +4288411704 +4288410468 +4288409328 +4288408113 +4288407903 +4288407444 +4288407255 +4288407213 +4288407198 +4288406415 +4288404099 +4288403955 +4288403790 +4288402725 +4288402335 +4288401759 +4288401753 +4288401528 +4288401444 +4288400994 +4288400934 +4288399923 +4288399863 +4288399029 +4288398093 +4288396299 +4288395903 +4288395264 +4288394244 +4288393845 +4288393473 +4288393443 +4288392780 +4288392129 +4288390020 +4288389783 +4288388763 +4288387710 +4288387335 +4288387020 +4288386543 +4288385688 +4288385409 +4288385223 +4288382964 +4288381833 +4288380963 +4288380720 +4288380708 +4288380075 +4288380039 +4288379700 +4288379313 +4288378899 +4288377015 +4288376214 +4288376133 +4288375518 +4288375059 +4288374138 +4288372863 +4288371498 +4288371165 +4288370985 +4288370805 +4288369389 +4288369263 +4288368468 +4288366185 +4288366068 +4288365420 +4288365390 +4288365339 +4288365270 +4288364298 +4288362699 +4288362030 +4288360413 +4288359405 +4288358334 +4288358040 +4288357704 +4288357590 +4288357503 +4288356945 +4288356723 +4288356549 +4288355559 +4288354350 +4288353459 +4288353030 +4288352853 +4288351245 +4288351119 +4288348758 +4288347063 +4288345725 +4288345698 +4288344639 +4288344600 +4288344138 +4288343850 +4288343820 +4288341519 +4288341000 +4288340784 +4288340199 +4288340175 +4288340019 +4288337259 +4288337154 +4288336899 +4288336095 +4288335474 +4288335045 +4288333785 +4288333509 +4288333500 +4288333413 +4288333155 +4288332279 +4288331928 +4288331463 +4288331025 +4288330443 +4288328484 +4288327749 +4288327653 +4288327314 +4288327239 +4288324608 +4288324440 +4288324050 +4288323693 +4288323129 +4288321488 +4288320594 +4288318740 +4288316595 +4288315659 +4288315449 +4288314864 +4288314474 +4288314093 +4288314063 +4288312329 +4288312113 +4288311804 +4288311060 +4288310529 +4288308003 +4288307313 +4288306944 +4288306224 +4288304685 +4288303719 +4288302465 +4288301979 +4288299078 +4288296879 +4288296654 +4288296468 +4288295733 +4288294668 +4288294515 +4288293630 +4288292610 +4288292244 +4288290300 +4288287330 +4288286100 +4288285638 +4288284594 +4288284195 +4288283595 +4288283520 +4288283208 +4288283193 +4288282899 +4288282668 +4288282038 +4288279623 +4288279098 +4288278804 +4288278735 +4288275510 +4288274814 +4288274520 +4288273878 +4288273305 +4288273095 +4288272570 +4288271154 +4288270983 +4288269258 +4288267845 +4288265694 +4288264344 +4288264149 +4288263504 +4288262109 +4288261944 +4288261410 +4288260453 +4288259955 +4288255020 +4288254804 +4288252809 +4288252695 +4288252239 +4288251444 +4288250295 +4288249833 +4288249824 +4288249344 +4288248684 +4288248138 +4288247055 +4288246983 +4288246905 +4288246464 +4288246269 +4288244019 +4288243140 +4288242210 +4288241109 +4288240818 +4288240104 +4288239795 +4288239549 +4288239483 +4288239450 +4288238979 +4288238904 +4288237929 +4288237824 +4288237530 +4288235664 +4288234845 +4288232700 +4288232298 +4288231605 +4288229700 +4288229163 +4288226874 +4288225938 +4288224555 +4288223700 +4288223523 +4288223043 +4288222365 +4288222125 +4288221519 +4288220424 +4288218273 +4288218084 +4288217790 +4288216080 +4288215258 +4288212435 +4288212390 +4288212288 +4288211754 +4288211355 +4288209408 +4288208340 +4288207785 +4288206264 +4288205778 +4288205523 +4288203918 +4288203435 +4288201923 +4288200960 +4288200693 +4288200654 +4288199700 +4288199550 +4288198824 +4288197798 +4288196829 +4288196055 +4288195353 +4288195338 +4288193778 +4288192770 +4288190985 +4288190868 +4288190760 +4288189575 +4288189410 +4288188894 +4288188630 +4288188618 +4288188423 +4288187028 +4288186638 +4288186533 +4288186365 +4288185420 +4288184955 +4288184820 +4288183935 +4288183833 +4288182690 +4288178013 +4288177914 +4288177593 +4288177524 +4288177500 +4288177179 +4288176165 +4288176135 +4288174365 +4288173219 +4288172679 +4288170228 +4288169214 +4288169208 +4288167924 +4288167759 +4288167495 +4288167213 +4288167105 +4288166214 +4288164495 +4288163484 +4288162164 +4288161549 +4288161090 +4288160808 +4288159704 +4288158888 +4288158663 +4288158378 +4288156713 +4288156698 +4288155093 +4288154769 +4288152585 +4288152414 +4288152195 +4288151118 +4288151064 +4288150305 +4288149813 +4288149474 +4288149459 +4288147359 +4288147263 +4288146444 +4288145988 +4288144458 +4288143315 +4288143279 +4288142610 +4288141683 +4288140900 +4288140723 +4288140099 +4288138785 +4288138560 +4288137594 +4288137570 +4288137498 +4288135743 +4288135629 +4288135290 +4288134999 +4288134924 +4288134525 +4288134159 +4288131765 +4288131624 +4288130274 +4288129518 +4288129074 +4288128645 +4288128369 +4288127745 +4288127625 +4288126590 +4288125723 +4288124520 +4288124478 +4288123815 +4288123398 +4288123203 +4288121859 +4288121424 +4288119279 +4288118154 +4288117275 +4288115775 +4288115658 +4288115319 +4288115100 +4288114923 +4288114893 +4288112583 +4288111854 +4288111560 +4288111404 +4288108923 +4288107183 +4288106724 +4288105965 +4288105395 +4288104735 +4288104609 +4288102089 +4288101195 +4288100445 +4288099758 +4288098720 +4288098240 +4288096410 +4288093680 +4288093248 +4288092759 +4288092693 +4288092234 +4288091940 +4288091808 +4288091790 +4288090584 +4288089774 +4288089219 +4288088754 +4288086363 +4288085553 +4288085439 +4288085124 +4288085058 +4288084764 +4288084143 +4288082469 +4288081740 +4288080765 +4288079778 +4288077870 +4288077354 +4288076274 +4288076064 +4288075764 +4288075725 +4288073790 +4288072689 +4288072650 +4288072020 +4288071798 +4288071558 +4288071480 +4288071429 +4288070808 +4288068228 +4288067238 +4288067169 +4288064913 +4288064808 +4288063674 +4288063293 +4288062549 +4288062393 +4288062129 +4288061448 +4288061364 +4288060815 +4288060479 +4288060113 +4288059954 +4288059063 +4288056843 +4288056258 +4288056228 +4288055598 +4288055493 +4288054968 +4288053714 +4288053675 +4288052943 +4288051908 +4288050840 +4288049685 +4288049445 +4288048905 +4288046958 +4288046559 +4288046463 +4288045839 +4288044339 +4288044204 +4288044174 +4288042920 +4288042185 +4288041498 +4288041078 +4288039188 +4288037970 +4288037589 +4288036485 +4288034463 +4288034439 +4288033953 +4288032663 +4288032615 +4288032120 +4288031154 +4288030449 +4288030278 +4288029588 +4288028013 +4288027584 +4288026030 +4288025274 +4288024674 +4288024125 +4288024074 +4288022805 +4288021584 +4288021239 +4288020840 +4288020609 +4288019088 +4288019004 +4288017873 +4288017735 +4288017513 +4288017504 +4288013730 +4288012200 +4288011348 +4288009998 +4288009824 +4288009725 +4288009158 +4288008690 +4288007604 +4288006350 +4288005588 +4288005375 +4288003779 +4288003170 +4288002915 +4288002144 +4288001913 +4288001145 +4288001103 +4288000248 +4287998595 +4287998154 +4287997923 +4287997719 +4287997608 +4287996348 +4287995829 +4287993195 +4287992658 +4287991014 +4287990783 +4287990588 +4287990279 +4287989805 +4287989424 +4287988635 +4287986853 +4287986613 +4287986253 +4287985239 +4287984378 +4287984273 +4287982425 +4287982368 +4287982005 +4287981474 +4287981414 +4287980448 +4287980385 +4287980355 +4287980283 +4287980199 +4287980163 +4287976110 +4287974715 +4287974613 +4287973068 +4287972954 +4287972129 +4287971985 +4287970149 +4287968079 +4287967935 +4287967659 +4287967278 +4287967230 +4287967194 +4287965199 +4287964065 +4287962439 +4287960159 +4287960048 +4287958653 +4287958170 +4287955560 +4287953880 +4287953745 +4287952704 +4287952479 +4287951279 +4287951015 +4287950913 +4287950754 +4287949674 +4287948978 +4287948204 +4287947223 +4287946509 +4287946395 +4287946374 +4287945354 +4287945108 +4287944454 +4287943920 +4287943635 +4287943110 +4287941799 +4287941250 +4287940950 +4287939885 +4287939864 +4287939405 +4287939159 +4287939138 +4287937893 +4287937659 +4287937533 +4287937233 +4287936408 +4287936390 +4287936294 +4287936225 +4287935868 +4287935370 +4287933795 +4287929694 +4287928653 +4287928608 +4287928470 +4287927489 +4287926889 +4287923769 +4287923358 +4287923310 +4287923235 +4287922329 +4287922014 +4287921975 +4287920580 +4287919944 +4287919734 +4287918375 +4287917955 +4287917595 +4287917514 +4287915369 +4287913869 +4287913764 +4287912714 +4287912564 +4287911544 +4287911145 +4287909834 +4287909795 +4287909150 +4287908640 +4287908583 +4287906609 +4287904230 +4287903684 +4287902529 +4287902520 +4287902370 +4287902139 +4287901359 +4287900603 +4287900240 +4287898734 +4287897624 +4287897528 +4287896595 +4287896478 +4287895710 +4287895575 +4287894729 +4287894054 +4287893634 +4287892839 +4287892089 +4287891903 +4287890913 +4287889893 +4287889269 +4287886560 +4287886545 +4287886035 +4287885273 +4287885159 +4287884889 +4287884499 +4287884235 +4287883734 +4287883575 +4287882669 +4287882285 +4287881664 +4287881064 +4287880833 +4287878514 +4287877524 +4287877359 +4287876969 +4287874650 +4287873393 +4287872970 +4287872898 +4287871545 +4287870033 +4287870009 +4287869334 +4287868530 +4287868098 +4287867129 +4287867078 +4287866658 +4287865680 +4287865284 +4287864855 +4287863475 +4287862698 +4287862458 +4287861843 +4287861345 +4287860883 +4287860709 +4287859743 +4287858345 +4287857925 +4287857238 +4287856824 +4287856734 +4287856473 +4287855495 +4287854985 +4287854595 +4287854154 +4287852540 +4287852345 +4287850860 +4287849909 +4287849174 +4287848895 +4287848619 +4287848355 +4287846975 +4287843183 +4287842634 +4287842583 +4287842313 +4287841164 +4287840543 +4287840060 +4287839865 +4287838983 +4287837303 +4287837069 +4287835923 +4287835674 +4287834864 +4287834129 +4287834078 +4287833934 +4287831399 +4287831273 +4287830853 +4287829125 +4287829083 +4287828888 +4287828459 +4287828354 +4287826410 +4287825720 +4287825645 +4287825543 +4287824604 +4287822678 +4287820575 +4287820458 +4287820248 +4287818349 +4287818283 +4287818118 +4287817050 +4287815490 +4287814299 +4287813729 +4287812289 +4287810969 +4287810828 +4287809508 +4287808488 +4287806739 +4287804543 +4287804198 +4287804144 +4287804003 +4287803880 +4287803373 +4287802095 +4287800868 +4287799983 +4287798894 +4287798585 +4287798285 +4287797604 +4287797508 +4287795708 +4287795480 +4287795135 +4287794544 +4287792780 +4287791703 +4287791388 +4287790533 +4287789789 +4287789735 +4287789474 +4287787395 +4287786699 +4287786240 +4287783288 +4287783255 +4287782754 +4287781869 +4287780099 +4287779154 +4287779058 +4287775434 +4287772530 +4287772413 +4287772398 +4287772299 +4287771108 +4287770829 +4287770343 +4287770208 +4287770124 +4287770073 +4287770004 +4287769779 +4287767625 +4287767118 +4287767034 +4287766518 +4287765648 +4287765558 +4287765165 +4287764925 +4287764544 +4287763674 +4287763578 +4287762909 +4287760884 +4287760455 +4287757713 +4287757005 +4287755823 +4287755073 +4287753834 +4287753009 +4287752994 +4287752910 +4287752454 +4287751083 +4287750204 +4287748410 +4287747729 +4287747255 +4287747120 +4287746868 +4287746328 +4287746289 +4287746214 +4287745239 +4287744105 +4287743865 +4287743820 +4287743754 +4287743460 +4287742560 +4287740838 +4287740700 +4287740589 +4287740253 +4287739419 +4287738105 +4287737355 +4287736113 +4287734940 +4287734904 +4287732588 +4287732519 +4287731565 +4287730800 +4287729018 +4287728868 +4287728163 +4287727794 +4287727734 +4287727674 +4287724794 +4287723063 +4287722505 +4287722190 +4287721830 +4287719808 +4287718728 +4287718398 +4287717915 +4287717084 +4287716058 +4287715389 +4287715260 +4287715209 +4287715200 +4287714894 +4287714273 +4287713478 +4287712290 +4287711804 +4287711504 +4287710568 +4287709989 +4287709950 +4287709083 +4287708018 +4287707448 +4287707385 +4287705723 +4287705624 +4287705213 +4287705063 +4287704805 +4287704058 +4287702753 +4287702033 +4287701529 +4287701145 +4287700773 +4287700569 +4287700149 +4287700044 +4287697503 +4287697410 +4287695310 +4287694563 +4287694185 +4287693903 +4287693774 +4287693135 +4287692268 +4287691533 +4287691170 +4287690924 +4287690693 +4287689859 +4287689439 +4287689208 +4287688623 +4287688263 +4287687789 +4287687549 +4287687153 +4287686709 +4287685410 +4287684243 +4287683685 +4287683550 +4287683310 +4287682260 +4287680580 +4287680358 +4287679143 +4287678969 +4287678300 +4287678198 +4287676959 +4287676809 +4287676095 +4287674340 +4287674205 +4287673740 +4287673290 +4287673200 +4287673134 +4287672738 +4287670968 +4287669429 +4287669120 +4287668838 +4287668763 +4287667485 +4287667419 +4287667320 +4287667215 +4287666900 +4287665208 +4287664869 +4287664158 +4287663414 +4287662700 +4287662445 +4287661413 +4287659859 +4287659853 +4287659190 +4287658908 +4287658830 +4287657954 +4287655959 +4287654258 +4287653469 +4287653418 +4287651744 +4287650580 +4287649773 +4287649110 +4287649080 +4287648669 +4287648480 +4287646884 +4287646419 +4287646368 +4287646173 +4287644805 +4287643668 +4287642603 +4287642390 +4287641700 +4287641505 +4287641484 +4287641148 +4287639369 +4287639105 +4287638979 +4287637194 +4287636300 +4287631125 +4287630525 +4287630255 +4287630180 +4287629478 +4287628035 +4287627693 +4287626784 +4287626649 +4287625089 +4287624495 +4287623829 +4287623430 +4287622965 +4287622440 +4287621654 +4287621645 +4287620784 +4287620319 +4287620133 +4287619143 +4287619083 +4287618573 +4287618453 +4287614523 +4287614409 +4287613965 +4287612708 +4287612600 +4287610650 +4287610293 +4287609525 +4287608598 +4287607479 +4287605784 +4287605574 +4287605085 +4287604944 +4287604254 +4287603795 +4287603603 +4287602838 +4287602073 +4287600318 +4287600315 +4287600009 +4287599658 +4287598455 +4287596925 +4287595278 +4287594585 +4287593814 +4287593793 +4287592350 +4287592008 +4287591423 +4287591189 +4287590604 +4287589953 +4287589425 +4287589038 +4287588669 +4287587160 +4287586938 +4287586533 +4287585345 +4287584874 +4287584490 +4287584388 +4287584199 +4287583923 +4287583035 +4287582795 +4287582729 +4287582663 +4287582150 +4287581823 +4287581100 +4287580494 +4287580359 +4287580254 +4287579105 +4287578190 +4287577098 +4287576963 +4287576480 +4287576093 +4287575820 +4287575325 +4287574038 +4287573030 +4287572220 +4287571890 +4287569145 +4287568185 +4287567549 +4287566799 +4287565413 +4287565113 +4287564984 +4287562800 +4287561759 +4287561750 +4287561444 +4287559398 +4287559254 +4287558075 +4287555579 +4287555468 +4287555105 +4287552885 +4287551724 +4287551673 +4287550098 +4287549978 +4287549768 +4287548940 +4287548553 +4287547845 +4287547500 +4287547134 +4287546690 +4287546213 +4287545769 +4287545055 +4287544983 +4287544479 +4287543405 +4287543333 +4287542253 +4287542154 +4287541863 +4287541455 +4287539859 +4287539508 +4287538575 +4287536874 +4287536799 +4287536043 +4287535770 +4287535680 +4287535173 +4287535140 +4287534450 +4287533193 +4287532614 +4287531324 +4287528645 +4287525849 +4287525009 +4287522618 +4287521799 +4287520608 +4287519729 +4287518250 +4287518079 +4287516810 +4287516780 +4287516630 +4287516105 +4287516075 +4287515580 +4287514560 +4287513534 +4287513348 +4287513114 +4287511833 +4287510660 +4287510540 +4287510408 +4287510030 +4287509910 +4287509808 +4287509640 +4287508854 +4287508695 +4287508560 +4287507939 +4287507090 +4287505659 +4287504963 +4287504618 +4287504060 +4287503835 +4287503559 +4287503214 +4287503085 +4287502458 +4287501813 +4287501420 +4287499938 +4287499779 +4287497508 +4287497469 +4287496410 +4287496053 +4287495693 +4287494475 +4287494424 +4287494265 +4287493575 +4287493269 +4287492903 +4287492018 +4287491859 +4287491559 +4287491463 +4287491283 +4287490035 +4287489060 +4287488883 +4287488688 +4287488358 +4287488268 +4287488190 +4287487149 +4287486705 +4287486420 +4287485994 +4287485859 +4287485814 +4287484524 +4287483873 +4287483045 +4287482949 +4287482703 +4287482328 +4287481758 +4287481254 +4287479325 +4287479313 +4287479145 +4287477690 +4287475914 +4287475155 +4287475149 +4287474723 +4287473925 +4287472419 +4287470403 +4287470148 +4287469935 +4287469824 +4287469779 +4287469704 +4287467859 +4287467598 +4287467283 +4287466359 +4287466260 +4287465789 +4287465360 +4287463809 +4287462954 +4287462693 +4287462060 +4287460800 +4287459198 +4287456138 +4287456078 +4287455964 +4287455793 +4287454743 +4287454065 +4287453915 +4287452739 +4287452709 +4287452079 +4287450000 +4287449685 +4287449619 +4287448803 +4287448164 +4287448005 +4287447549 +4287445785 +4287445008 +4287444210 +4287443853 +4287442764 +4287441495 +4287441198 +4287440829 +4287440409 +4287439785 +4287439590 +4287436413 +4287435993 +4287435810 +4287435069 +4287433995 +4287433755 +4287433554 +4287433239 +4287433074 +4287432630 +4287432525 +4287431595 +4287430734 +4287430710 +4287430299 +4287428043 +4287425988 +4287424608 +4287424305 +4287422784 +4287421773 +4287421734 +4287419598 +4287419394 +4287419184 +4287417300 +4287416385 +4287416259 +4287415485 +4287414633 +4287413280 +4287413154 +4287412914 +4287411819 +4287411009 +4287410673 +4287409689 +4287409014 +4287408735 +4287408720 +4287406950 +4287406518 +4287405855 +4287405750 +4287405435 +4287405228 +4287404463 +4287404415 +4287404253 +4287403923 +4287403218 +4287402783 +4287402498 +4287402390 +4287401103 +4287400050 +4287398685 +4287397938 +4287397518 +4287395628 +4287395454 +4287394944 +4287393258 +4287393060 +4287392490 +4287391989 +4287391713 +4287391269 +4287390114 +4287389958 +4287387993 +4287387810 +4287387600 +4287386619 +4287384930 +4287383955 +4287383940 +4287383880 +4287383439 +4287382800 +4287382770 +4287382680 +4287380865 +4287380238 +4287379899 +4287379725 +4287379488 +4287379263 +4287378558 +4287376098 +4287375639 +4287375429 +4287374880 +4287374703 +4287374430 +4287374364 +4287371079 +4287370554 +4287369855 +4287369633 +4287368895 +4287368469 +4287367353 +4287366093 +4287365109 +4287364683 +4287364044 +4287363198 +4287362145 +4287359910 +4287358314 +4287357900 +4287355695 +4287354903 +4287353019 +4287352704 +4287350145 +4287349653 +4287348840 +4287347865 +4287347799 +4287347238 +4287342570 +4287342339 +4287341904 +4287340188 +4287340035 +4287339990 +4287339000 +4287338790 +4287338133 +4287337803 +4287337488 +4287336063 +4287335205 +4287334833 +4287333270 +4287333018 +4287332304 +4287331014 +4287330990 +4287330813 +4287330033 +4287329028 +4287328374 +4287328305 +4287323985 +4287323325 +4287321828 +4287321009 +4287320709 +4287320520 +4287319665 +4287319434 +4287318738 +4287316125 +4287311535 +4287310944 +4287310860 +4287310779 +4287310419 +4287309993 +4287309633 +4287309048 +4287308703 +4287308298 +4287307923 +4287307788 +4287307773 +4287307374 +4287306504 +4287305373 +4287305343 +4287304545 +4287302655 +4287302580 +4287301308 +4287300513 +4287300234 +4287299835 +4287299355 +4287298935 +4287297549 +4287296520 +4287296088 +4287295449 +4287294315 +4287293724 +4287293388 +4287293193 +4287292449 +4287291468 +4287291459 +4287290103 +4287289908 +4287289770 +4287289215 +4287288783 +4287288579 +4287288318 +4287286584 +4287285969 +4287282660 +4287282609 +4287282363 +4287281808 +4287281349 +4287281283 +4287280833 +4287279303 +4287278670 +4287275445 +4287275370 +4287275040 +4287274680 +4287270438 +4287269325 +4287268998 +4287268839 +4287268338 +4287265059 +4287264255 +4287262773 +4287262239 +4287261789 +4287261054 +4287260829 +4287260208 +4287258855 +4287258690 +4287258219 +4287255720 +4287255153 +4287254814 +4287254280 +4287253995 +4287253848 +4287253368 +4287251979 +4287251820 +4287251055 +4287250404 +4287249039 +4287246909 +4287246705 +4287246249 +4287246078 +4287245700 +4287245214 +4287245028 +4287244830 +4287244155 +4287242760 +4287242364 +4287242319 +4287242244 +4287241743 +4287241098 +4287240900 +4287239979 +4287239475 +4287239073 +4287238119 +4287235749 +4287234600 +4287233760 +4287233688 +4287233208 +4287232635 +4287229959 +4287229845 +4287228735 +4287226965 +4287226803 +4287226464 +4287225459 +4287225318 +4287224628 +4287223839 +4287223689 +4287223029 +4287220140 +4287219270 +4287218319 +4287214560 +4287214188 +4287213993 +4287212973 +4287212280 +4287211764 +4287211593 +4287211290 +4287210954 +4287210504 +4287209949 +4287209730 +4287208584 +4287208260 +4287208065 +4287207435 +4287206598 +4287205560 +4287205473 +4287205074 +4287204225 +4287204204 +4287203763 +4287203343 +4287203049 +4287201939 +4287201915 +4287201744 +4287201579 +4287201534 +4287201234 +4287201063 +4287199239 +4287198618 +4287197178 +4287196773 +4287196473 +4287195720 +4287195255 +4287194994 +4287194730 +4287193998 +4287193305 +4287192714 +4287192558 +4287192339 +4287190818 +4287189753 +4287189684 +4287189138 +4287189093 +4287188118 +4287187005 +4287186630 +4287183480 +4287182310 +4287181170 +4287180834 +4287179673 +4287179490 +4287178923 +4287178293 +4287177375 +4287177345 +4287174780 +4287172704 +4287171798 +4287171195 +4287170874 +4287169059 +4287168159 +4287168030 +4287166980 +4287165828 +4287165123 +4287164889 +4287164265 +4287164223 +4287161928 +4287161613 +4287161199 +4287160134 +4287158898 +4287157575 +4287157113 +4287156705 +4287155565 +4287155298 +4287153948 +4287153729 +4287153633 +4287152523 +4287152250 +4287151968 +4287151350 +4287148995 +4287147864 +4287144783 +4287144519 +4287144063 +4287143484 +4287142728 +4287142455 +4287141900 +4287141300 +4287141120 +4287140763 +4287140469 +4287138255 +4287138135 +4287136845 +4287136635 +4287135849 +4287134640 +4287134460 +4287131574 +4287130644 +4287129129 +4287128100 +4287127704 +4287127395 +4287127113 +4287126900 +4287125958 +4287125220 +4287125115 +4287124164 +4287123879 +4287123219 +4287118785 +4287118494 +4287117780 +4287117603 +4287117213 +4287116634 +4287115908 +4287114735 +4287114609 +4287114594 +4287112245 +4287112203 +4287111990 +4287111828 +4287111690 +4287111123 +4287111048 +4287110430 +4287108495 +4287108099 +4287107145 +4287106998 +4287106755 +4287106458 +4287106263 +4287105684 +4287104835 +4287104193 +4287103170 +4287102279 +4287102165 +4287100608 +4287100485 +4287100245 +4287098955 +4287097914 +4287096975 +4287096618 +4287096558 +4287096429 +4287094944 +4287094689 +4287092958 +4287092418 +4287091725 +4287091140 +4287090939 +4287090849 +4287090423 +4287090048 +4287089883 +4287089838 +4287089724 +4287089145 +4287089103 +4287088359 +4287087924 +4287087750 +4287087435 +4287087399 +4287085473 +4287085389 +4287084858 +4287084234 +4287082890 +4287081078 +4287080913 +4287080754 +4287079509 +4287079368 +4287078528 +4287078270 +4287077859 +4287077604 +4287077520 +4287075915 +4287075765 +4287074295 +4287073605 +4287072459 +4287071829 +4287071643 +4287070863 +4287070800 +4287070569 +4287070380 +4287070059 +4287069954 +4287068574 +4287067233 +4287066414 +4287066390 +4287064020 +4287063954 +4287062745 +4287062235 +4287061338 +4287060249 +4287059853 +4287059418 +4287059280 +4287058125 +4287057873 +4287057810 +4287057654 +4287054933 +4287052398 +4287052224 +4287051069 +4287050580 +4287050544 +4287049299 +4287048930 +4287048909 +4287046158 +4287046083 +4287045783 +4287045708 +4287045108 +4287044364 +4287044280 +4287043710 +4287043704 +4287043314 +4287042879 +4287042819 +4287041550 +4287039429 +4287039384 +4287039063 +4287038133 +4287037023 +4287036948 +4287036579 +4287036234 +4287035364 +4287033675 +4287031749 +4287030900 +4287030054 +4287029118 +4287028893 +4287028185 +4287026808 +4287025545 +4287024453 +4287023049 +4287020844 +4287020064 +4287019968 +4287019749 +4287019653 +4287019308 +4287019005 +4287018699 +4287017904 +4287016665 +4287013650 +4287013368 +4287013335 +4287012999 +4287012693 +4287012570 +4287011298 +4287010374 +4287009399 +4287008715 +4287007509 +4287007278 +4287006255 +4287005673 +4287005070 +4287005064 +4287002889 +4287002328 +4287002175 +4287000135 +4286999568 +4286999013 +4286998404 +4286997294 +4286997063 +4286996325 +4286994648 +4286993949 +4286991420 +4286990613 +4286990193 +4286989794 +4286989785 +4286989425 +4286987388 +4286987118 +4286986968 +4286986770 +4286985009 +4286984568 +4286982633 +4286980533 +4286980218 +4286979759 +4286979669 +4286979324 +4286978745 +4286977848 +4286977344 +4286977068 +4286976405 +4286975874 +4286975724 +4286975343 +4286974488 +4286974350 +4286974173 +4286973669 +4286973120 +4286972640 +4286972415 +4286971674 +4286970528 +4286969529 +4286968929 +4286968638 +4286968038 +4286966418 +4286966295 +4286964939 +4286963859 +4286961684 +4286961624 +4286960520 +4286958984 +4286957760 +4286957583 +4286957463 +4286957163 +4286956983 +4286956554 +4286955924 +4286955135 +4286954559 +4286954190 +4286953308 +4286952054 +4286948784 +4286948055 +4286946699 +4286946048 +4286945733 +4286944284 +4286944110 +4286943969 +4286943918 +4286943804 +4286943618 +4286942934 +4286942649 +4286942313 +4286942145 +4286941074 +4286940894 +4286940360 +4286939229 +4286938599 +4286937828 +4286936043 +4286935995 +4286935185 +4286934870 +4286933253 +4286932005 +4286930583 +4286930109 +4286929668 +4286928708 +4286928219 +4286928108 +4286927715 +4286927685 +4286927424 +4286927238 +4286924745 +4286924484 +4286924400 +4286924130 +4286921649 +4286920470 +4286920239 +4286920044 +4286918604 +4286918019 +4286917845 +4286917458 +4286916423 +4286916399 +4286915919 +4286915820 +4286915703 +4286915580 +4286913825 +4286913099 +4286912535 +4286912379 +4286912244 +4286911959 +4286909934 +4286909049 +4286908989 +4286907915 +4286907414 +4286907135 +4286906403 +4286905734 +4286905269 +4286904909 +4286904609 +4286904453 +4286903655 +4286903565 +4286903400 +4286903124 +4286902833 +4286902143 +4286901279 +4286901249 +4286900043 +4286898015 +4286897160 +4286896263 +4286895618 +4286895129 +4286894325 +4286893374 +4286893278 +4286893134 +4286892090 +4286891910 +4286891544 +4286890590 +4286890089 +4286890014 +4286889810 +4286889528 +4286888520 +4286887734 +4286886609 +4286882664 +4286882055 +4286880633 +4286880228 +4286879943 +4286879859 +4286879544 +4286879238 +4286878575 +4286878179 +4286877804 +4286875593 +4286875389 +4286874339 +4286871750 +4286871333 +4286870229 +4286868489 +4286867199 +4286866989 +4286866725 +4286865780 +4286865723 +4286865015 +4286863614 +4286862588 +4286860908 +4286860590 +4286860410 +4286859753 +4286859318 +4286858583 +4286857884 +4286856420 +4286856318 +4286855805 +4286853459 +4286852883 +4286852445 +4286851743 +4286850555 +4286849598 +4286849343 +4286848068 +4286847795 +4286846520 +4286845629 +4286844834 +4286844273 +4286843664 +4286841933 +4286840589 +4286839023 +4286834754 +4286833638 +4286833335 +4286831193 +4286830194 +4286829765 +4286829723 +4286829510 +4286829105 +4286828388 +4286827929 +4286826900 +4286826843 +4286825658 +4286825025 +4286823213 +4286821293 +4286821005 +4286820558 +4286820348 +4286820303 +4286819580 +4286819505 +4286819040 +4286817048 +4286816070 +4286814885 +4286810328 +4286809128 +4286808954 +4286806965 +4286806275 +4286804898 +4286803734 +4286803629 +4286802738 +4286800710 +4286798748 +4286798460 +4286797359 +4286796954 +4286796549 +4286796183 +4286795625 +4286795358 +4286795283 +4286794998 +4286793885 +4286793174 +4286792775 +4286792475 +4286791818 +4286790975 +4286790549 +4286789904 +4286789493 +4286789034 +4286787639 +4286787414 +4286786610 +4286784195 +4286783940 +4286782758 +4286782314 +4286781444 +4286781189 +4286780118 +4286779974 +4286778459 +4286777130 +4286777013 +4286776173 +4286775258 +4286773818 +4286773188 +4286771760 +4286771565 +4286770878 +4286769738 +4286769459 +4286769345 +4286768640 +4286768004 +4286767845 +4286767518 +4286765490 +4286765484 +4286764629 +4286764275 +4286763108 +4286763024 +4286760423 +4286758848 +4286758815 +4286758230 +4286758140 +4286757903 +4286756088 +4286755914 +4286755530 +4286754828 +4286754219 +4286753343 +4286750709 +4286750520 +4286750280 +4286746014 +4286744574 +4286743320 +4286742864 +4286741208 +4286740965 +4286739183 +4286739168 +4286737713 +4286737218 +4286737059 +4286736228 +4286736195 +4286735928 +4286735790 +4286734968 +4286734908 +4286734803 +4286734203 +4286734149 +4286733660 +4286731410 +4286731203 +4286731008 +4286730978 +4286730855 +4286730843 +4286729919 +4286729004 +4286728974 +4286727378 +4286727189 +4286726895 +4286726793 +4286725689 +4286725278 +4286725023 +4286724729 +4286724114 +4286724018 +4286721333 +4286721099 +4286720658 +4286720514 +4286718843 +4286718423 +4286718294 +4286715030 +4286714475 +4286713803 +4286713584 +4286712825 +4286712438 +4286711379 +4286710674 +4286709894 +4286708805 +4286708175 +4286707995 +4286707458 +4286705508 +4286705193 +4286704455 +4286703543 +4286700804 +4286700075 +4286698725 +4286698134 +4286696778 +4286695983 +4286695953 +4286694999 +4286694828 +4286693655 +4286693250 +4286692980 +4286690379 +4286689590 +4286689233 +4286688693 +4286688078 +4286687340 +4286685969 +4286684685 +4286683464 +4286682345 +4286680905 +4286680824 +4286680620 +4286680083 +4286679948 +4286679930 +4286678310 +4286677923 +4286675895 +4286675655 +4286675268 +4286673753 +4286673390 +4286671389 +4286670114 +4286669799 +4286669595 +4286669373 +4286668368 +4286666718 +4286666658 +4286666319 +4286665293 +4286665053 +4286663160 +4286661834 +4286661519 +4286661123 +4286660379 +4286659530 +4286659425 +4286659089 +4286657418 +4286656800 +4286656590 +4286656023 +4286654910 +4286654715 +4286654709 +4286654634 +4286654223 +4286654190 +4286652270 +4286650035 +4286649933 +4286648553 +4286648238 +4286647848 +4286647455 +4286647284 +4286645790 +4286645583 +4286645214 +4286645205 +4286643423 +4286642445 +4286642118 +4286641644 +4286641434 +4286640720 +4286640189 +4286640093 +4286639934 +4286639859 +4286639673 +4286637249 +4286636424 +4286636268 +4286635305 +4286635023 +4286633895 +4286633730 +4286633538 +4286632218 +4286631384 +4286631180 +4286630895 +4286630880 +4286626245 +4286625948 +4286625405 +4286624880 +4286624718 +4286624229 +4286624100 +4286623173 +4286623008 +4286622459 +4286621619 +4286620560 +4286619909 +4286619693 +4286619540 +4286618805 +4286618670 +4286618538 +4286618280 +4286617749 +4286617200 +4286616729 +4286616375 +4286615763 +4286615070 +4286614095 +4286612988 +4286612925 +4286612679 +4286612244 +4286612184 +4286611824 +4286611164 +4286610354 +4286610069 +4286610030 +4286609895 +4286609559 +4286608308 +4286601723 +4286601210 +4286600505 +4286599113 +4286598279 +4286597190 +4286596098 +4286595474 +4286594280 +4286593188 +4286591508 +4286590680 +4286590539 +4286590059 +4286589069 +4286587794 +4286586963 +4286586345 +4286586198 +4286586003 +4286585535 +4286584518 +4286584428 +4286584380 +4286584095 +4286583735 +4286583279 +4286583204 +4286582970 +4286582538 +4286581845 +4286581473 +4286579874 +4286579115 +4286578890 +4286578449 +4286578353 +4286577294 +4286577123 +4286576049 +4286575929 +4286574330 +4286574255 +4286573889 +4286573865 +4286573724 +4286573133 +4286573028 +4286572848 +4286572008 +4286568438 +4286568165 +4286567559 +4286567130 +4286565444 +4286565405 +4286564793 +4286563923 +4286563905 +4286563725 +4286562885 +4286562873 +4286561628 +4286560599 +4286560458 +4286559864 +4286559534 +4286558379 +4286557470 +4286557323 +4286557065 +4286556183 +4286555028 +4286554503 +4286554383 +4286553330 +4286553228 +4286550798 +4286550294 +4286549415 +4286549070 +4286548908 +4286548719 +4286548218 +4286547858 +4286543910 +4286543838 +4286543793 +4286542533 +4286541633 +4286541435 +4286540643 +4286539713 +4286538465 +4286537979 +4286535735 +4286532060 +4286532039 +4286531595 +4286530920 +4286530050 +4286529768 +4286529504 +4286529204 +4286527608 +4286527110 +4286525778 +4286525124 +4286521098 +4286520204 +4286518545 +4286518419 +4286517954 +4286517795 +4286516814 +4286516709 +4286514408 +4286514399 +4286513574 +4286513535 +4286512650 +4286512443 +4286509800 +4286508945 +4286508855 +4286508030 +4286507883 +4286507769 +4286507490 +4286505888 +4286505663 +4286505285 +4286504673 +4286504109 +4286503899 +4286503704 +4286501565 +4286501325 +4286500578 +4286500368 +4286499243 +4286497599 +4286497194 +4286496483 +4286495850 +4286494869 +4286494470 +4286494269 +4286493849 +4286492043 +4286491770 +4286490573 +4286488965 +4286488809 +4286488449 +4286488065 +4286487558 +4286485749 +4286484030 +4286481324 +4286481060 +4286480694 +4286479953 +4286478528 +4286478390 +4286478153 +4286477334 +4286476335 +4286476113 +4286474865 +4286473488 +4286473380 +4286471925 +4286471658 +4286467809 +4286467779 +4286467044 +4286465595 +4286465064 +4286464953 +4286464398 +4286464260 +4286463615 +4286463480 +4286462379 +4286460849 +4286460510 +4286459925 +4286459043 +4286458659 +4286457939 +4286457330 +4286457294 +4286456865 +4286456478 +4286454933 +4286454504 +4286454063 +4286452689 +4286452005 +4286451594 +4286451063 +4286450034 +4286450010 +4286449125 +4286448603 +4286448540 +4286448474 +4286448060 +4286447430 +4286446725 +4286446689 +4286445120 +4286445108 +4286444379 +4286444358 +4286443848 +4286442483 +4286442420 +4286441538 +4286440530 +4286439993 +4286439660 +4286439594 +4286439354 +4286439114 +4286438373 +4286438274 +4286437245 +4286437179 +4286436939 +4286436915 +4286435664 +4286435655 +4286433714 +4286433315 +4286433249 +4286432808 +4286431215 +4286430753 +4286430630 +4286430354 +4286429910 +4286429484 +4286428965 +4286428140 +4286427678 +4286427384 +4286426424 +4286425998 +4286425650 +4286424870 +4286424495 +4286424198 +4286424183 +4286421819 +4286421390 +4286421180 +4286420613 +4286419905 +4286419518 +4286419224 +4286418858 +4286418618 +4286418069 +4286417784 +4286417169 +4286416905 +4286415813 +4286415024 +4286411970 +4286409918 +4286409840 +4286409450 +4286409294 +4286408295 +4286407203 +4286406360 +4286405955 +4286405385 +4286404650 +4286403990 +4286403294 +4286401173 +4286399454 +4286398833 +4286398065 +4286396850 +4286396385 +4286396163 +4286396028 +4286395788 +4286394600 +4286393814 +4286393265 +4286393028 +4286392200 +4286392095 +4286390688 +4286387880 +4286387673 +4286387544 +4286387304 +4286386350 +4286386179 +4286385864 +4286384553 +4286384289 +4286384178 +4286383449 +4286383149 +4286382339 +4286382303 +4286381034 +4286380950 +4286380323 +4286380125 +4286379690 +4286379459 +4286378928 +4286377353 +4286377158 +4286376564 +4286376060 +4286375793 +4286375493 +4286374008 +4286372664 +4286371614 +4286371374 +4286371083 +4286370723 +4286369718 +4286368500 +4286368233 +4286368215 +4286367723 +4286366913 +4286366700 +4286365140 +4286364720 +4286364585 +4286364564 +4286364468 +4286364234 +4286363550 +4286361609 +4286360130 +4286358945 +4286357319 +4286357313 +4286356878 +4286356719 +4286356380 +4286355774 +4286355303 +4286355150 +4286355129 +4286354418 +4286353554 +4286351979 +4286351109 +4286349795 +4286349693 +4286349570 +4286348844 +4286348064 +4286347113 +4286346468 +4286346450 +4286345778 +4286345535 +4286345325 +4286344254 +4286344188 +4286344089 +4286341653 +4286341200 +4286340723 +4286340684 +4286340645 +4286339553 +4286338824 +4286338005 +4286336760 +4286336370 +4286334753 +4286334720 +4286332938 +4286332179 +4286331843 +4286331180 +4286330544 +4286329848 +4286328639 +4286328243 +4286327874 +4286326308 +4286326155 +4286325804 +4286325690 +4286324775 +4286323683 +4286323530 +4286323095 +4286323065 +4286322570 +4286322039 +4286321949 +4286321880 +4286321208 +4286320659 +4286319990 +4286319939 +4286318778 +4286318118 +4286316009 +4286315304 +4286315103 +4286314689 +4286314443 +4286314119 +4286313900 +4286313660 +4286312148 +4286311755 +4286311653 +4286311485 +4286311158 +4286308404 +4286307984 +4286306988 +4286306793 +4286306739 +4286305959 +4286305758 +4286304924 +4286304258 +4286304153 +4286303238 +4286303220 +4286302674 +4286302599 +4286300910 +4286299323 +4286299143 +4286298918 +4286298324 +4286298285 +4286297028 +4286296434 +4286295768 +4286295144 +4286294958 +4286294778 +4286293824 +4286293548 +4286292480 +4286292465 +4286292345 +4286292303 +4286292135 +4286292015 +4286290533 +4286290104 +4286290035 +4286290008 +4286289339 +4286288853 +4286288244 +4286287299 +4286287104 +4286286669 +4286285934 +4286285289 +4286285043 +4286284653 +4286284290 +4286284215 +4286282964 +4286281284 +4286281194 +4286278143 +4286277939 +4286277429 +4286277324 +4286276298 +4286276259 +4286275665 +4286275374 +4286275089 +4286273160 +4286272725 +4286272710 +4286272338 +4286272254 +4286270814 +4286270679 +4286270535 +4286269860 +4286269503 +4286269209 +4286268255 +4286267910 +4286267763 +4286267154 +4286266803 +4286266668 +4286265504 +4286264874 +4286264688 +4286264190 +4286264100 +4286261910 +4286259543 +4286257248 +4286257029 +4286256399 +4286255400 +4286254203 +4286253825 +4286252670 +4286250024 +4286249814 +4286249793 +4286248560 +4286247498 +4286247363 +4286246544 +4286246403 +4286245743 +4286244465 +4286243379 +4286243199 +4286243193 +4286243079 +4286242884 +4286242530 +4286242290 +4286241840 +4286240700 +4286239299 +4286238090 +4286237733 +4286236890 +4286236755 +4286234778 +4286233764 +4286231145 +4286230689 +4286230473 +4286229744 +4286226969 +4286226693 +4286226369 +4286225679 +4286225025 +4286224419 +4286224269 +4286222865 +4286222508 +4286222025 +4286221773 +4286219784 +4286219085 +4286218770 +4286218713 +4286218014 +4286217960 +4286217300 +4286216658 +4286216178 +4286215704 +4286215293 +4286214504 +4286213568 +4286213499 +4286213310 +4286211720 +4286210934 +4286210760 +4286207583 +4286205759 +4286205063 +4286204004 +4286203680 +4286203194 +4286202159 +4286201730 +4286200569 +4286200404 +4286200068 +4286200059 +4286199828 +4286199333 +4286197968 +4286197398 +4286196363 +4286196309 +4286195814 +4286194428 +4286193705 +4286192205 +4286191815 +4286188080 +4286186964 +4286186385 +4286185974 +4286185758 +4286184684 +4286183370 +4286181453 +4286180808 +4286178678 +4286177829 +4286176284 +4286176119 +4286175309 +4286173263 +4286172969 +4286172549 +4286171889 +4286171865 +4286170794 +4286170644 +4286170335 +4286170080 +4286169369 +4286168823 +4286168424 +4286167194 +4286166885 +4286164599 +4286163945 +4286163633 +4286162625 +4286161995 +4286161029 +4286158299 +4286156535 +4286155359 +4286154045 +4286153598 +4286152200 +4286150874 +4286150088 +4286149605 +4286148849 +4286147640 +4286147610 +4286145975 +4286145939 +4286145813 +4286143503 +4286142315 +4286140908 +4286137239 +4286135433 +4286135094 +4286134938 +4286134923 +4286132028 +4286131233 +4286131170 +4286131158 +4286130219 +4286130183 +4286129790 +4286129610 +4286128833 +4286128029 +4286127000 +4286126355 +4286123919 +4286122785 +4286120940 +4286119515 +4286119020 +4286118669 +4286116605 +4286116455 +4286109744 +4286109033 +4286104458 +4286104359 +4286102523 +4286100738 +4286099334 +4286099160 +4286098935 +4286098698 +4286098554 +4286098263 +4286098104 +4286097579 +4286096328 +4286095878 +4286095623 +4286094498 +4286093640 +4286092578 +4286091414 +4286091363 +4286089185 +4286088789 +4286087508 +4286087373 +4286087043 +4286084790 +4286084073 +4286083569 +4286083338 +4286083098 +4286082750 +4286082003 +4286079519 +4286079378 +4286078760 +4286076453 +4286076129 +4286075883 +4286075424 +4286074449 +4286074233 +4286073804 +4286073120 +4286072103 +4286071359 +4286070900 +4286070810 +4286070318 +4286069355 +4286068734 +4286068233 +4286067999 +4286065854 +4286065560 +4286065260 +4286063589 +4286059155 +4286058999 +4286058300 +4286057550 +4286056584 +4286055378 +4286054913 +4286054709 +4286052573 +4286052324 +4286051874 +4286051400 +4286050308 +4286049495 +4286049018 +4286048574 +4286048403 +4286047689 +4286047170 +4286046609 +4286046549 +4286045994 +4286045490 +4286044140 +4286043354 +4286043165 +4286042304 +4286041518 +4286039715 +4286038413 +4286035335 +4286034480 +4286033724 +4286033013 +4286032983 +4286031693 +4286031309 +4286030715 +4286030358 +4286029968 +4286029830 +4286029203 +4286028018 +4286026899 +4286026200 +4286024835 +4286023689 +4286023245 +4286023050 +4286019969 +4286018913 +4286018544 +4286017644 +4286017230 +4286016315 +4286014290 +4286013264 +4286013000 +4286011365 +4286009619 +4286009460 +4286009190 +4286008839 +4286007789 +4286007579 +4286006823 +4286006724 +4286006625 +4286006418 +4286005788 +4286005710 +4286004918 +4286004204 +4286002794 +4286002563 +4286002560 +4286002068 +4286001993 +4286001669 +4286001483 +4286001090 +4285999653 +4285998834 +4285998339 +4285997754 +4285997625 +4285996599 +4285996239 +4285996200 +4285996119 +4285995168 +4285994478 +4285993890 +4285993503 +4285990479 +4285989723 +4285989504 +4285989405 +4285989039 +4285988850 +4285988370 +4285988103 +4285987980 +4285987494 +4285987068 +4285985994 +4285985508 +4285984584 +4285984143 +4285984113 +4285983525 +4285982835 +4285982478 +4285981479 +4285980849 +4285980168 +4285979955 +4285979859 +4285979130 +4285976499 +4285974675 +4285973439 +4285972374 +4285971324 +4285970988 +4285970925 +4285969833 +4285968798 +4285966803 +4285965750 +4285965510 +4285965243 +4285963209 +4285963143 +4285959828 +4285959300 +4285959084 +4285956849 +4285956360 +4285956255 +4285956069 +4285954974 +4285954704 +4285953075 +4285952595 +4285952439 +4285952340 +4285952130 +4285951230 +4285951083 +4285950744 +4285950588 +4285949598 +4285949499 +4285948845 +4285948773 +4285947954 +4285947093 +4285945749 +4285945104 +4285942440 +4285941834 +4285941240 +4285940895 +4285939743 +4285939623 +4285938879 +4285938405 +4285938378 +4285937664 +4285937544 +4285935909 +4285935120 +4285933560 +4285932798 +4285932168 +4285932075 +4285930173 +4285930020 +4285928118 +4285928109 +4285927395 +4285926489 +4285925553 +4285923900 +4285923468 +4285923309 +4285922928 +4285922913 +4285921629 +4285921458 +4285921389 +4285916250 +4285914954 +4285913799 +4285911078 +4285911069 +4285911063 +4285910958 +4285910265 +4285909473 +4285908273 +4285907088 +4285906350 +4285904628 +4285904544 +4285904328 +4285903824 +4285903104 +4285902225 +4285902060 +4285902018 +4285901490 +4285900554 +4285899834 +4285899015 +4285898193 +4285897665 +4285897533 +4285893939 +4285892304 +4285892043 +4285890684 +4285889643 +4285888050 +4285887228 +4285886505 +4285885014 +4285883628 +4285882485 +4285881480 +4285881390 +4285881024 +4285879830 +4285879680 +4285879569 +4285875144 +4285874565 +4285874358 +4285874334 +4285873779 +4285873539 +4285873434 +4285872414 +4285871724 +4285871688 +4285871628 +4285869948 +4285869873 +4285869600 +4285869483 +4285869414 +4285868544 +4285867929 +4285867764 +4285866438 +4285864959 +4285864089 +4285863708 +4285862748 +4285862643 +4285862124 +4285860900 +4285859499 +4285857603 +4285857420 +4285857000 +4285855740 +4285855545 +4285854495 +4285853433 +4285851918 +4285851699 +4285851423 +4285850625 +4285848783 +4285847499 +4285847145 +4285847124 +4285847079 +4285845990 +4285845549 +4285845084 +4285844160 +4285843569 +4285843338 +4285842315 +4285842288 +4285842000 +4285841895 +4285841328 +4285841040 +4285840665 +4285838598 +4285838553 +4285836570 +4285836540 +4285835433 +4285834434 +4285834293 +4285833768 +4285829679 +4285829604 +4285828713 +4285828380 +4285828053 +4285827444 +4285826670 +4285826385 +4285826088 +4285825740 +4285825629 +4285824915 +4285824618 +4285824375 +4285823055 +4285821459 +4285821375 +4285820598 +4285819683 +4285818558 +4285818360 +4285818330 +4285818225 +4285817508 +4285813530 +4285812975 +4285811265 +4285810515 +4285807263 +4285807179 +4285806318 +4285804635 +4285803819 +4285803570 +4285802058 +4285801563 +4285801488 +4285800630 +4285799610 +4285799280 +4285799253 +4285798599 +4285797243 +4285795863 +4285793844 +4285790760 +4285790244 +4285790004 +4285788960 +4285788870 +4285788765 +4285788573 +4285788039 +4285787583 +4285787373 +4285785843 +4285785825 +4285785084 +4285784928 +4285784433 +4285783554 +4285781739 +4285781625 +4285779624 +4285779480 +4285778493 +4285775415 +4285774800 +4285774758 +4285774008 +4285773840 +4285773750 +4285773744 +4285773708 +4285772280 +4285770663 +4285769823 +4285768983 +4285768803 +4285768179 +4285768023 +4285767834 +4285767738 +4285767675 +4285767540 +4285767198 +4285766793 +4285766313 +4285765200 +4285764990 +4285762818 +4285762155 +4285761060 +4285760904 +4285760193 +4285760088 +4285759155 +4285758825 +4285757763 +4285757670 +4285757334 +4285756440 +4285755759 +4285755459 +4285755435 +4285755369 +4285754628 +4285754574 +4285752810 +4285752504 +4285752495 +4285751133 +4285750740 +4285750353 +4285750278 +4285749564 +4285749105 +4285749033 +4285748640 +4285748628 +4285746813 +4285745604 +4285744959 +4285744254 +4285743843 +4285743363 +4285743120 +4285739979 +4285738464 +4285736754 +4285736544 +4285736379 +4285735998 +4285735968 +4285734759 +4285734705 +4285734150 +4285733514 +4285732725 +4285732104 +4285731054 +4285730919 +4285730454 +4285730349 +4285730343 +4285729965 +4285729833 +4285729665 +4285728744 +4285728429 +4285727838 +4285727769 +4285725039 +4285724994 +4285723650 +4285723524 +4285722924 +4285722864 +4285721649 +4285721544 +4285719564 +4285719369 +4285718589 +4285718268 +4285717899 +4285717044 +4285715724 +4285714980 +4285714539 +4285713978 +4285713453 +4285712130 +4285710348 +4285710264 +4285709319 +4285708764 +4285708479 +4285708428 +4285708269 +4285707768 +4285707018 +4285704750 +4285704069 +4285703730 +4285703679 +4285703190 +4285703145 +4285702095 +4285702065 +4285700754 +4285700679 +4285699089 +4285698558 +4285697919 +4285694313 +4285693608 +4285693095 +4285691928 +4285690725 +4285688298 +4285688268 +4285688148 +4285687770 +4285687239 +4285686483 +4285686264 +4285685283 +4285684914 +4285684740 +4285684665 +4285684368 +4285684290 +4285683918 +4285680609 +4285680510 +4285680150 +4285679268 +4285678743 +4285677318 +4285677159 +4285676883 +4285676589 +4285675773 +4285675464 +4285675005 +4285673649 +4285672854 +4285672740 +4285672695 +4285672245 +4285672194 +4285670415 +4285670025 +4285669905 +4285669839 +4285668945 +4285667874 +4285667079 +4285666143 +4285665480 +4285665459 +4285664283 +4285664190 +4285663950 +4285662864 +4285662828 +4285661205 +4285661010 +4285660329 +4285659369 +4285657374 +4285656555 +4285656423 +4285656063 +4285655283 +4285653228 +4285652778 +4285652175 +4285651845 +4285651284 +4285650348 +4285650120 +4285649490 +4285648983 +4285648809 +4285648674 +4285648569 +4285648443 +4285648038 +4285646658 +4285646268 +4285645899 +4285645875 +4285645740 +4285644609 +4285643463 +4285642098 +4285641615 +4285638840 +4285638015 +4285636668 +4285636110 +4285635018 +4285632798 +4285632615 +4285632009 +4285631160 +4285628700 +4285626723 +4285625475 +4285623273 +4285622010 +4285621404 +4285620729 +4285620669 +4285619700 +4285619658 +4285619205 +4285618188 +4285617423 +4285614723 +4285614348 +4285612329 +4285612068 +4285609608 +4285609560 +4285609200 +4285609044 +4285608930 +4285608459 +4285608309 +4285606374 +4285606335 +4285606065 +4285605378 +4285605348 +4285604763 +4285604760 +4285603035 +4285600935 +4285600383 +4285600305 +4285600275 +4285599864 +4285599858 +4285598610 +4285598079 +4285598040 +4285597533 +4285597455 +4285596600 +4285596390 +4285595985 +4285595565 +4285594818 +4285591113 +4285590069 +4285589073 +4285588044 +4285587999 +4285585968 +4285585545 +4285585185 +4285585113 +4285583499 +4285583379 +4285582743 +4285582188 +4285581390 +4285581054 +4285578528 +4285578408 +4285578375 +4285577730 +4285577604 +4285575654 +4285575288 +4285574319 +4285574289 +4285573344 +4285572819 +4285572795 +4285571034 +4285570554 +4285567863 +4285566894 +4285565109 +4285564620 +4285564599 +4285564395 +4285564353 +4285562718 +4285561938 +4285561890 +4285561395 +4285561113 +4285557720 +4285557519 +4285557330 +4285556604 +4285556409 +4285556205 +4285554105 +4285553838 +4285552998 +4285552239 +4285552089 +4285551459 +4285550388 +4285549764 +4285549548 +4285549023 +4285547964 +4285546014 +4285545825 +4285544913 +4285544250 +4285543764 +4285543725 +4285543104 +4285541700 +4285541400 +4285538925 +4285538778 +4285538199 +4285538034 +4285537404 +4285537305 +4285536969 +4285536438 +4285535943 +4285534884 +4285534638 +4285533624 +4285533234 +4285532583 +4285532250 +4285532130 +4285531533 +4285531194 +4285529058 +4285528953 +4285528560 +4285528254 +4285527729 +4285526505 +4285526208 +4285526049 +4285525383 +4285524594 +4285524039 +4285523889 +4285522968 +4285521819 +4285521219 +4285520574 +4285519929 +4285518783 +4285518420 +4285517955 +4285517544 +4285517160 +4285516185 +4285515573 +4285515564 +4285515465 +4285515288 +4285515150 +4285514724 +4285512573 +4285512255 +4285510074 +4285510020 +4285509948 +4285509879 +4285509684 +4285508694 +4285507668 +4285507653 +4285507569 +4285507509 +4285507233 +4285505049 +4285504998 +4285503699 +4285501689 +4285500663 +4285500549 +4285500204 +4285499088 +4285498650 +4285496454 +4285495515 +4285495170 +4285494870 +4285494660 +4285494543 +4285494468 +4285494300 +4285490604 +4285489725 +4285488660 +4285486155 +4285484853 +4285482993 +4285482150 +4285481418 +4285480428 +4285477374 +4285474824 +4285474710 +4285474380 +4285473870 +4285473480 +4285473270 +4285473150 +4285471848 +4285470735 +4285469979 +4285469853 +4285469679 +4285467285 +4285467030 +4285465875 +4285465290 +4285464009 +4285463589 +4285463169 +4285461024 +4285459968 +4285459344 +4285459053 +4285458705 +4285457109 +4285456404 +4285454064 +4285453143 +4285452093 +4285451388 +4285449510 +4285449084 +4285447893 +4285447605 +4285446789 +4285443855 +4285442820 +4285441203 +4285440384 +4285437033 +4285436328 +4285435164 +4285435059 +4285434279 +4285433514 +4285433460 +4285433403 +4285432800 +4285432494 +4285431954 +4285429443 +4285427154 +4285427055 +4285426803 +4285426410 +4285426248 +4285426158 +4285425984 +4285424829 +4285424748 +4285423770 +4285423659 +4285420473 +4285419498 +4285418163 +4285418103 +4285417860 +4285417008 +4285415925 +4285414335 +4285412913 +4285412799 +4285412655 +4285410975 +4285410438 +4285408158 +4285408113 +4285407780 +4285407468 +4285406679 +4285406334 +4285405410 +4285405368 +4285403514 +4285401030 +4285400469 +4285399503 +4285398594 +4285398264 +4285397985 +4285397508 +4285397058 +4285394583 +4285393425 +4285392768 +4285390689 +4285389828 +4285389594 +4285389024 +4285388364 +4285385769 +4285385259 +4285384578 +4285384500 +4285384008 +4285383984 +4285383018 +4285382928 +4285381644 +4285381299 +4285380669 +4285379964 +4285377753 +4285376250 +4285375653 +4285375158 +4285374675 +4285373790 +4285373673 +4285373445 +4285373034 +4285372164 +4285371114 +4285370613 +4285370148 +4285369395 +4285368963 +4285368828 +4285368534 +4285367418 +4285366233 +4285365075 +4285364784 +4285364538 +4285364313 +4285363713 +4285362990 +4285361673 +4285361178 +4285359828 +4285358994 +4285358799 +4285358475 +4285358274 +4285357770 +4285356864 +4285356528 +4285356405 +4285355430 +4285354074 +4285354038 +4285353843 +4285350489 +4285349820 +4285349298 +4285349055 +4285348635 +4285348245 +4285348038 +4285347123 +4285346964 +4285346169 +4285344915 +4285344798 +4285344048 +4285342059 +4285341780 +4285340814 +4285339800 +4285338510 +4285338048 +4285337988 +4285336398 +4285336299 +4285335204 +4285334964 +4285334424 +4285334079 +4285333815 +4285333254 +4285333095 +4285332315 +4285331949 +4285331259 +4285330698 +4285327878 +4285327383 +4285326825 +4285325175 +4285325043 +4285324044 +4285321608 +4285320954 +4285320843 +4285320213 +4285317993 +4285317648 +4285316790 +4285314480 +4285314330 +4285312935 +4285312755 +4285311609 +4285311435 +4285311339 +4285310235 +4285309689 +4285309230 +4285309029 +4285308963 +4285308588 +4285308144 +4285307169 +4285306653 +4285305123 +4285304373 +4285304199 +4285304178 +4285303464 +4285303338 +4285303308 +4285301739 +4285301613 +4285299564 +4285299348 +4285298739 +4285298268 +4285297533 +4285297380 +4285297155 +4285296849 +4285296204 +4285295958 +4285295253 +4285295169 +4285293075 +4285291080 +4285290894 +4285290453 +4285290240 +4285290009 +4285289994 +4285289280 +4285289028 +4285288833 +4285288350 +4285288029 +4285287228 +4285287219 +4285285863 +4285285665 +4285285653 +4285285284 +4285284975 +4285284018 +4285282539 +4285281525 +4285279110 +4285278885 +4285278648 +4285277799 +4285277718 +4285276953 +4285276938 +4285275909 +4285275540 +4285274154 +4285273158 +4285273014 +4285272018 +4285271514 +4285271298 +4285271184 +4285270374 +4285270323 +4285270053 +4285269495 +4285268868 +4285267698 +4285266870 +4285266579 +4285265214 +4285264878 +4285264263 +4285264089 +4285263423 +4285262565 +4285261473 +4285261329 +4285260465 +4285260390 +4285259625 +4285258185 +4285257000 +4285256613 +4285256529 +4285255905 +4285255623 +4285255425 +4285254378 +4285252620 +4285251579 +4285251495 +4285251255 +4285247538 +4285247175 +4285246854 +4285246539 +4285245768 +4285245504 +4285245378 +4285245228 +4285244985 +4285244559 +4285244523 +4285244040 +4285243458 +4285243269 +4285242249 +4285240734 +4285239678 +4285237188 +4285237185 +4285236945 +4285236840 +4285236033 +4285235559 +4285235535 +4285235514 +4285235055 +4285234959 +4285234854 +4285234758 +4285233630 +4285233144 +4285232733 +4285232160 +4285231650 +4285230864 +4285230705 +4285230474 +4285230204 +4285230039 +4285229583 +4285229268 +4285227723 +4285227120 +4285226580 +4285226049 +4285224438 +4285224405 +4285223370 +4285222938 +4285221654 +4285221549 +4285218810 +4285217088 +4285216638 +4285215738 +4285215063 +4285214358 +4285213449 +4285213014 +4285212615 +4285210230 +4285210200 +4285208178 +4285207890 +4285207755 +4285207419 +4285207200 +4285206465 +4285205823 +4285205643 +4285204713 +4285204008 +4285203879 +4285203714 +4285201353 +4285200795 +4285200345 +4285200213 +4285200069 +4285199343 +4285199043 +4285198644 +4285198533 +4285197798 +4285196760 +4285196730 +4285196634 +4285196550 +4285195848 +4285194780 +4285194564 +4285194339 +4285194255 +4285193664 +4285193463 +4285193205 +4285191798 +4285191363 +4285190775 +4285190703 +4285189215 +4285188648 +4285188159 +4285187355 +4285186848 +4285186620 +4285186218 +4285185918 +4285185528 +4285184835 +4285184574 +4285181559 +4285180845 +4285180725 +4285179783 +4285178700 +4285177308 +4285176285 +4285174113 +4285173144 +4285172148 +4285169574 +4285169490 +4285169343 +4285168350 +4285167444 +4285166640 +4285166334 +4285165443 +4285165380 +4285164225 +4285163568 +4285163508 +4285162770 +4285162323 +4285161699 +4285161600 +4285160580 +4285160514 +4285160463 +4285159884 +4285159614 +4285159515 +4285159305 +4285158438 +4285157574 +4285157274 +4285156218 +4285156170 +4285155519 +4285155255 +4285154079 +4285153500 +4285153218 +4285151970 +4285151304 +4285150575 +4285150089 +4285149288 +4285149120 +4285148943 +4285148580 +4285148040 +4285147560 +4285147005 +4285146474 +4285146423 +4285146348 +4285146204 +4285144863 +4285144755 +4285144248 +4285143819 +4285143630 +4285143210 +4285142559 +4285142484 +4285141554 +4285141530 +4285141068 +4285139844 +4285139820 +4285139760 +4285139454 +4285138605 +4285138569 +4285136850 +4285136409 +4285135485 +4285134960 +4285134918 +4285134738 +4285133580 +4285132818 +4285132215 +4285131213 +4285130274 +4285130223 +4285128618 +4285128060 +4285127364 +4285125528 +4285123314 +4285122813 +4285122693 +4285122420 +4285122303 +4285121265 +4285121043 +4285118970 +4285118955 +4285118829 +4285117500 +4285117278 +4285116780 +4285116060 +4285115460 +4285114449 +4285114035 +4285110894 +4285110399 +4285107615 +4285107513 +4285106775 +4285105320 +4285104759 +4285104465 +4285102038 +4285101339 +4285099728 +4285099080 +4285098495 +4285095933 +4285092315 +4285090383 +4285090353 +4285089264 +4285089033 +4285088799 +4285087320 +4285087308 +4285085388 +4285085259 +4285085103 +4285084929 +4285084749 +4285084518 +4285084509 +4285084503 +4285083405 +4285083348 +4285083318 +4285082985 +4285082373 +4285082130 +4285082004 +4285080294 +4285079865 +4285075899 +4285073163 +4285072725 +4285071570 +4285071414 +4285070223 +4285068348 +4285066863 +4285066773 +4285066563 +4285065639 +4285065498 +4285064379 +4285063413 +4285063410 +4285063359 +4285060254 +4285058373 +4285058280 +4285058085 +4285057953 +4285056663 +4285056369 +4285056198 +4285055958 +4285054875 +4285054674 +4285054254 +4285053993 +4285051905 +4285051074 +4285051008 +4285050153 +4285049910 +4285048908 +4285048725 +4285048200 +4285046979 +4285046724 +4285046349 +4285045803 +4285044930 +4285044048 +4285043748 +4285042839 +4285042698 +4285042035 +4285041990 +4285041768 +4285041633 +4285041369 +4285040955 +4285040625 +4285040619 +4285038798 +4285037415 +4285036845 +4285036059 +4285034538 +4285034085 +4285033530 +4285032618 +4285032543 +4285032270 +4285032129 +4285031565 +4285031553 +4285031145 +4285030608 +4285030029 +4285029798 +4285029075 +4285026900 +4285026858 +4285026273 +4285025304 +4285025004 +4285023714 +4285022808 +4285021938 +4285021893 +4285020690 +4285020318 +4285019754 +4285019268 +4285018680 +4285018020 +4285015404 +4285014735 +4285014420 +4285014240 +4285012884 +4285012203 +4285011573 +4285011150 +4285011108 +4285010673 +4285008780 +4285006428 +4285006020 +4285004619 +4285004394 +4285003743 +4285003023 +4285002345 +4285001808 +4285001643 +4285001484 +4285000263 +4284999474 +4284998943 +4284998805 +4284998418 +4284998298 +4284997755 +4284997650 +4284997485 +4284996990 +4284996534 +4284996390 +4284996285 +4284995769 +4284994650 +4284994125 +4284993489 +4284992745 +4284992193 +4284991140 +4284989694 +4284989100 +4284987483 +4284984873 +4284984870 +4284984555 +4284984354 +4284982953 +4284982299 +4284982173 +4284981669 +4284981618 +4284980388 +4284979989 +4284979863 +4284979788 +4284979305 +4284978918 +4284978774 +4284978615 +4284978510 +4284978174 +4284976863 +4284976785 +4284976344 +4284975990 +4284975708 +4284975183 +4284974394 +4284974100 +4284973260 +4284972405 +4284971523 +4284971010 +4284970590 +4284970518 +4284970233 +4284968085 +4284967884 +4284967713 +4284966024 +4284964578 +4284963549 +4284962463 +4284961569 +4284961329 +4284960588 +4284959535 +4284959325 +4284959283 +4284957453 +4284957003 +4284956205 +4284953655 +4284952560 +4284952278 +4284951879 +4284951579 +4284951165 +4284950568 +4284949530 +4284949359 +4284949200 +4284947859 +4284947253 +4284946590 +4284946245 +4284946134 +4284945768 +4284945339 +4284943899 +4284943158 +4284942144 +4284941925 +4284941055 +4284939933 +4284938178 +4284938148 +4284937683 +4284936573 +4284934785 +4284934284 +4284934155 +4284933873 +4284933849 +4284932088 +4284931938 +4284931413 +4284928845 +4284926358 +4284925665 +4284924693 +4284924213 +4284923964 +4284923538 +4284918900 +4284918669 +4284917670 +4284917463 +4284917064 +4284915438 +4284915288 +4284914583 +4284914490 +4284914394 +4284914280 +4284914025 +4284912870 +4284909123 +4284909039 +4284907368 +4284907194 +4284906888 +4284906573 +4284906183 +4284905313 +4284905148 +4284904365 +4284904158 +4284903315 +4284902700 +4284901929 +4284901479 +4284900765 +4284900630 +4284899844 +4284898623 +4284897798 +4284897030 +4284896763 +4284895845 +4284895755 +4284895275 +4284895173 +4284894579 +4284894060 +4284892650 +4284892470 +4284892239 +4284892074 +4284891390 +4284890760 +4284890568 +4284890559 +4284890280 +4284889218 +4284888294 +4284887949 +4284886149 +4284885870 +4284883020 +4284882378 +4284881634 +4284880764 +4284880503 +4284880335 +4284880290 +4284879060 +4284878193 +4284877539 +4284877113 +4284876975 +4284876909 +4284876648 +4284876120 +4284876009 +4284875514 +4284874494 +4284874434 +4284874284 +4284873900 +4284873678 +4284873528 +4284873015 +4284872808 +4284870624 +4284869928 +4284868878 +4284868785 +4284866664 +4284866115 +4284865374 +4284865089 +4284864750 +4284864615 +4284861978 +4284860604 +4284860460 +4284860238 +4284859995 +4284859083 +4284858228 +4284857025 +4284854754 +4284854358 +4284853938 +4284852423 +4284850809 +4284850464 +4284850113 +4284849813 +4284849348 +4284848895 +4284848763 +4284848193 +4284846450 +4284845349 +4284844590 +4284844059 +4284843759 +4284841830 +4284840945 +4284840633 +4284840609 +4284839313 +4284839160 +4284838599 +4284837270 +4284835065 +4284834543 +4284834510 +4284834048 +4284833895 +4284833388 +4284832674 +4284831783 +4284831270 +4284831228 +4284831090 +4284830460 +4284829569 +4284829479 +4284829044 +4284826140 +4284825840 +4284824340 +4284824124 +4284823968 +4284823038 +4284822939 +4284822555 +4284822384 +4284822045 +4284821889 +4284821400 +4284820518 +4284820485 +4284820308 +4284820119 +4284820074 +4284819978 +4284819063 +4284819033 +4284818100 +4284817545 +4284816060 +4284814920 +4284811725 +4284809850 +4284809310 +4284809088 +4284807798 +4284805659 +4284805230 +4284805119 +4284804999 +4284804954 +4284804720 +4284804630 +4284804273 +4284803934 +4284803244 +4284802425 +4284801015 +4284800283 +4284800280 +4284799683 +4284798819 +4284798384 +4284798195 +4284797343 +4284796029 +4284794925 +4284794493 +4284794370 +4284793605 +4284793245 +4284791754 +4284791574 +4284791430 +4284790749 +4284788460 +4284788430 +4284786708 +4284786675 +4284786450 +4284786225 +4284786024 +4284785769 +4284785223 +4284784395 +4284783408 +4284781668 +4284781269 +4284780639 +4284780120 +4284779628 +4284778323 +4284776643 +4284774600 +4284773130 +4284773085 +4284772920 +4284772899 +4284772269 +4284772065 +4284769620 +4284769590 +4284769578 +4284769425 +4284769419 +4284768108 +4284765210 +4284764400 +4284763344 +4284760815 +4284760500 +4284759429 +4284757104 +4284756789 +4284756330 +4284755229 +4284755109 +4284755100 +4284754773 +4284754383 +4284754278 +4284753240 +4284752430 +4284751995 +4284751449 +4284750933 +4284750405 +4284749715 +4284749313 +4284748278 +4284747048 +4284746928 +4284745290 +4284744693 +4284743298 +4284743130 +4284741825 +4284741198 +4284740295 +4284739635 +4284739389 +4284736743 +4284736314 +4284734154 +4284734100 +4284732375 +4284731508 +4284730743 +4284730455 +4284730374 +4284729534 +4284728694 +4284727770 +4284725865 +4284725355 +4284723789 +4284723549 +4284723243 +4284722205 +4284721773 +4284721383 +4284721218 +4284720978 +4284720285 +4284718794 +4284718500 +4284718170 +4284717723 +4284717324 +4284716679 +4284716523 +4284715809 +4284715674 +4284715224 +4284714993 +4284714549 +4284712704 +4284710835 +4284710478 +4284710055 +4284709269 +4284707973 +4284707064 +4284706800 +4284706359 +4284706194 +4284705435 +4284704868 +4284704850 +4284704598 +4284704535 +4284704109 +4284703893 +4284703353 +4284703125 +4284702504 +4284701085 +4284699420 +4284699090 +4284698964 +4284698493 +4284698484 +4284698334 +4284697245 +4284697158 +4284696405 +4284694764 +4284694470 +4284693399 +4284693384 +4284692868 +4284692763 +4284692100 +4284691815 +4284691368 +4284690579 +4284690360 +4284689160 +4284689043 +4284687378 +4284687174 +4284686364 +4284685215 +4284685110 +4284684939 +4284684780 +4284684204 +4284683355 +4284682638 +4284682464 +4284682389 +4284681933 +4284681630 +4284681423 +4284679983 +4284678450 +4284678348 +4284678153 +4284677445 +4284676338 +4284673755 +4284673680 +4284670983 +4284669045 +4284667350 +4284666648 +4284666585 +4284666459 +4284663843 +4284663414 +4284663348 +4284662604 +4284662019 +4284661974 +4284661533 +4284661143 +4284660714 +4284659643 +4284658350 +4284656565 +4284656400 +4284656334 +4284655185 +4284654648 +4284654225 +4284653988 +4284653850 +4284653784 +4284653355 +4284653253 +4284653208 +4284652923 +4284650979 +4284650793 +4284649620 +4284649563 +4284649473 +4284648480 +4284648180 +4284647895 +4284647790 +4284647325 +4284647085 +4284646854 +4284645960 +4284644790 +4284644619 +4284644229 +4284642594 +4284641670 +4284641178 +4284640764 +4284640515 +4284639654 +4284637104 +4284636690 +4284635355 +4284634128 +4284633960 +4284632658 +4284630645 +4284630024 +4284628239 +4284627954 +4284625353 +4284625350 +4284625035 +4284624498 +4284624279 +4284624099 +4284623805 +4284622893 +4284622350 +4284622044 +4284621873 +4284621645 +4284621549 +4284620745 +4284620019 +4284618639 +4284616668 +4284616623 +4284615774 +4284614970 +4284614895 +4284614724 +4284613935 +4284613584 +4284613458 +4284612045 +4284611673 +4284611460 +4284610953 +4284610854 +4284608739 +4284608439 +4284608049 +4284605904 +4284605688 +4284602565 +4284602433 +4284601095 +4284599604 +4284597978 +4284597120 +4284595554 +4284592959 +4284591069 +4284590658 +4284589869 +4284589305 +4284588840 +4284588603 +4284585723 +4284585144 +4284584220 +4284583950 +4284581514 +4284580788 +4284580590 +4284580209 +4284580035 +4284579678 +4284579438 +4284579144 +4284578640 +4284576960 +4284576105 +4284575580 +4284575049 +4284574530 +4284573303 +4284572820 +4284572718 +4284571959 +4284571758 +4284570195 +4284568968 +4284568548 +4284566700 +4284566310 +4284565155 +4284564795 +4284564600 +4284564318 +4284563304 +4284563205 +4284562014 +4284559485 +4284558915 +4284558789 +4284558564 +4284555210 +4284554715 +4284554568 +4284553404 +4284551955 +4284550464 +4284550329 +4284550095 +4284546573 +4284545649 +4284545550 +4284544620 +4284543408 +4284541980 +4284541668 +4284541425 +4284541323 +4284541068 +4284540639 +4284539589 +4284538098 +4284536868 +4284536850 +4284536124 +4284532293 +4284531849 +4284531585 +4284530865 +4284529095 +4284528699 +4284527064 +4284525843 +4284525474 +4284525180 +4284524889 +4284524685 +4284524259 +4284523344 +4284522579 +4284521853 +4284521685 +4284519945 +4284519048 +4284518559 +4284518334 +4284515130 +4284514920 +4284514128 +4284513348 +4284512949 +4284512574 +4284512055 +4284511749 +4284511218 +4284510174 +4284507543 +4284507510 +4284507408 +4284507108 +4284507045 +4284505560 +4284505380 +4284504960 +4284504624 +4284503124 +4284502374 +4284501453 +4284501420 +4284501165 +4284499350 +4284498993 +4284496335 +4284495678 +4284494040 +4284493995 +4284493965 +4284493503 +4284492708 +4284491790 +4284491364 +4284490899 +4284490329 +4284490140 +4284490014 +4284489219 +4284489144 +4284488943 +4284487893 +4284487299 +4284487143 +4284485085 +4284484653 +4284483444 +4284482739 +4284481524 +4284480585 +4284480198 +4284480165 +4284479913 +4284477000 +4284475944 +4284475719 +4284474855 +4284474129 +4284473829 +4284473499 +4284473325 +4284472344 +4284471588 +4284471528 +4284470874 +4284470574 +4284469278 +4284469050 +4284468705 +4284468303 +4284467799 +4284467139 +4284466689 +4284465963 +4284464793 +4284463713 +4284463443 +4284462729 +4284461643 +4284461328 +4284460383 +4284459525 +4284459129 +4284459123 +4284459060 +4284458388 +4284458325 +4284458265 +4284458199 +4284457770 +4284457695 +4284457350 +4284455850 +4284455199 +4284455034 +4284454383 +4284453654 +4284453360 +4284453279 +4284453129 +4284452808 +4284452613 +4284451989 +4284451290 +4284451269 +4284450780 +4284449433 +4284448035 +4284447168 +4284446088 +4284445539 +4284444900 +4284444633 +4284443625 +4284443238 +4284441825 +4284440580 +4284440208 +4284439290 +4284438774 +4284437505 +4284437349 +4284435714 +4284435429 +4284435078 +4284434889 +4284434799 +4284433989 +4284432750 +4284432438 +4284430209 +4284430029 +4284429423 +4284428904 +4284428244 +4284427899 +4284427665 +4284427635 +4284427545 +4284426300 +4284426219 +4284425958 +4284425094 +4284424905 +4284424464 +4284424233 +4284423324 +4284423144 +4284421590 +4284420423 +4284420279 +4284420150 +4284419490 +4284418080 +4284417588 +4284417213 +4284417168 +4284415113 +4284414783 +4284413973 +4284413343 +4284412998 +4284410964 +4284410889 +4284410553 +4284407853 +4284407784 +4284407340 +4284406398 +4284405798 +4284404970 +4284404865 +4284403254 +4284402930 +4284402585 +4284401313 +4284401070 +4284400884 +4284400809 +4284400704 +4284400218 +4284400140 +4284399945 +4284399789 +4284398688 +4284398643 +4284396654 +4284394443 +4284391398 +4284391245 +4284390705 +4284389868 +4284388935 +4284388470 +4284388113 +4284387138 +4284386169 +4284385734 +4284385575 +4284385530 +4284385479 +4284383988 +4284382998 +4284381993 +4284381948 +4284381540 +4284380523 +4284379464 +4284379389 +4284378969 +4284378570 +4284377940 +4284376029 +4284374769 +4284371733 +4284370644 +4284370560 +4284370095 +4284370065 +4284369405 +4284369318 +4284368940 +4284368724 +4284367359 +4284367053 +4284366945 +4284364764 +4284362370 +4284361719 +4284360888 +4284358518 +4284358014 +4284357804 +4284356289 +4284354849 +4284354453 +4284353214 +4284352194 +4284351603 +4284351093 +4284349770 +4284349524 +4284346335 +4284344160 +4284341580 +4284341250 +4284341160 +4284340893 +4284340434 +4284340263 +4284338484 +4284338355 +4284338073 +4284337968 +4284337914 +4284337413 +4284337014 +4284336960 +4284336264 +4284335298 +4284333084 +4284332928 +4284332919 +4284330630 +4284330204 +4284330165 +4284330015 +4284329610 +4284329124 +4284326340 +4284326004 +4284323559 +4284322875 +4284322824 +4284321039 +4284320610 +4284318765 +4284318519 +4284318108 +4284313089 +4284312660 +4284312180 +4284311214 +4284310689 +4284310485 +4284307728 +4284305709 +4284305124 +4284304839 +4284302085 +4284301815 +4284300720 +4284299769 +4284299349 +4284299073 +4284298950 +4284298569 +4284296163 +4284296049 +4284295974 +4284295218 +4284293595 +4284293340 +4284292320 +4284291129 +4284290349 +4284289530 +4284288723 +4284287775 +4284287544 +4284287154 +4284287103 +4284286989 +4284286704 +4284286509 +4284286305 +4284286083 +4284285510 +4284284658 +4284284364 +4284284289 +4284283605 +4284283368 +4284282198 +4284281058 +4284280785 +4284279909 +4284278943 +4284278193 +4284277914 +4284277788 +4284277023 +4284276828 +4284276603 +4284275730 +4284275553 +4284275190 +4284275085 +4284274890 +4284274779 +4284273069 +4284272979 +4284272685 +4284272088 +4284271095 +4284270570 +4284270210 +4284267963 +4284267270 +4284266208 +4284265935 +4284264918 +4284264909 +4284263754 +4284263100 +4284261990 +4284261588 +4284261378 +4284260550 +4284259848 +4284259470 +4284259365 +4284258873 +4284257319 +4284256920 +4284255573 +4284254334 +4284253755 +4284252399 +4284252045 +4284250788 +4284247833 +4284245883 +4284245799 +4284245370 +4284245220 +4284243804 +4284243165 +4284242655 +4284242289 +4284240864 +4284240633 +4284240393 +4284240033 +4284234270 +4284233448 +4284232869 +4284232533 +4284232029 +4284231873 +4284231528 +4284230544 +4284230214 +4284227883 +4284227619 +4284227013 +4284226890 +4284226695 +4284226143 +4284226113 +4284225753 +4284223944 +4284223698 +4284223665 +4284223410 +4284222789 +4284222783 +4284219834 +4284219810 +4284218559 +4284218385 +4284217458 +4284217428 +4284217098 +4284216744 +4284216600 +4284216315 +4284216285 +4284216048 +4284215559 +4284215514 +4284215124 +4284214623 +4284214284 +4284214164 +4284213669 +4284210810 +4284210603 +4284210594 +4284210078 +4284209994 +4284209175 +4284208773 +4284208419 +4284208083 +4284207390 +4284206844 +4284206769 +4284206289 +4284205530 +4284205299 +4284202683 +4284202173 +4284201549 +4284200460 +4284199545 +4284198744 +4284198135 +4284197940 +4284197874 +4284197760 +4284197529 +4284197295 +4284197169 +4284196983 +4284196224 +4284192540 +4284192024 +4284191505 +4284190854 +4284190434 +4284190413 +4284190239 +4284189714 +4284189525 +4284189255 +4284187935 +4284187803 +4284187230 +4284186768 +4284184098 +4284182943 +4284182934 +4284182859 +4284181914 +4284181854 +4284181824 +4284181683 +4284181293 +4284180705 +4284180570 +4284180375 +4284180363 +4284180279 +4284180195 +4284180165 +4284179745 +4284178920 +4284178449 +4284177105 +4284176244 +4284176154 +4284175155 +4284174930 +4284174639 +4284172644 +4284171849 +4284171783 +4284170280 +4284169653 +4284168399 +4284167538 +4284167313 +4284167160 +4284166878 +4284166458 +4284166260 +4284163050 +4284162450 +4284162354 +4284161295 +4284160734 +4284160434 +4284159705 +4284159528 +4284159075 +4284158028 +4284157743 +4284157359 +4284155643 +4284155595 +4284155328 +4284154950 +4284154773 +4284154290 +4284153894 +4284153705 +4284153333 +4284150498 +4284150024 +4284149988 +4284148254 +4284147048 +4284146973 +4284146835 +4284146520 +4284146475 +4284145899 +4284145860 +4284144759 +4284144600 +4284141963 +4284140769 +4284140349 +4284140265 +4284139695 +4284139518 +4284139053 +4284138333 +4284137928 +4284136734 +4284136095 +4284135915 +4284135294 +4284134895 +4284132723 +4284132123 +4284131589 +4284129738 +4284129618 +4284128859 +4284125928 +4284125340 +4284123885 +4284123144 +4284122034 +4284120933 +4284120489 +4284119853 +4284119283 +4284119229 +4284119169 +4284119148 +4284117495 +4284117444 +4284116133 +4284115008 +4284114780 +4284113940 +4284112005 +4284109068 +4284109059 +4284108849 +4284106959 +4284106749 +4284106344 +4284103248 +4284102360 +4284101820 +4284101505 +4284101394 +4284100995 +4284100245 +4284099783 +4284099708 +4284099543 +4284098430 +4284098124 +4284097068 +4284095604 +4284095154 +4284094578 +4284094494 +4284093759 +4284087864 +4284086808 +4284085638 +4284084414 +4284083424 +4284082218 +4284081945 +4284081660 +4284081543 +4284079890 +4284079593 +4284079380 +4284078114 +4284078039 +4284077055 +4284076425 +4284075234 +4284074205 +4284072780 +4284071274 +4284070539 +4284069483 +4284068229 +4284066963 +4284066723 +4284065544 +4284065394 +4284064254 +4284063129 +4284062355 +4284062079 +4284060000 +4284059979 +4284059274 +4284057669 +4284056973 +4284056649 +4284056160 +4284055788 +4284054978 +4284054753 +4284054663 +4284052254 +4284051054 +4284050583 +4284048924 +4284048699 +4284047634 +4284047055 +4284046983 +4284046959 +4284046059 +4284045633 +4284044400 +4284043995 +4284043860 +4284043695 +4284043149 +4284041553 +4284041415 +4284041214 +4284040293 +4284040125 +4284038613 +4284038205 +4284037938 +4284037308 +4284034935 +4284034539 +4284032310 +4284031575 +4284030498 +4284030054 +4284029238 +4284029154 +4284028824 +4284028038 +4284025899 +4284025275 +4284024195 +4284022995 +4284022839 +4284022029 +4284021738 +4284020943 +4284020910 +4284020544 +4284020355 +4284020103 +4284019305 +4284018315 +4284018213 +4284016803 +4284016500 +4284016260 +4284014805 +4284013548 +4284012183 +4284012108 +4284011994 +4284011565 +4284010995 +4284010584 +4284008949 +4284008718 +4284008364 +4284008190 +4284008058 +4284007713 +4284007428 +4284007059 +4284006438 +4284006384 +4284005694 +4284004425 +4284004194 +4284003720 +4284003585 +4284002679 +4284002148 +4284001818 +4284000378 +4283999718 +4283998545 +4283998515 +4283997744 +4283997465 +4283996799 +4283996220 +4283996205 +4283995890 +4283994210 +4283993358 +4283992383 +4283992275 +4283990565 +4283990058 +4283989725 +4283988870 +4283988483 +4283987613 +4283986509 +4283986269 +4283984544 +4283984319 +4283983704 +4283981274 +4283981208 +4283980659 +4283979558 +4283978538 +4283978019 +4283977830 +4283976954 +4283976873 +4283970393 +4283969178 +4283968800 +4283967885 +4283966310 +4283964495 +4283964399 +4283963640 +4283963109 +4283962770 +4283962089 +4283961918 +4283961408 +4283960955 +4283960880 +4283960535 +4283959434 +4283959008 +4283957940 +4283957895 +4283956470 +4283956230 +4283952378 +4283952345 +4283952084 +4283951595 +4283951490 +4283950944 +4283949318 +4283948670 +4283948340 +4283948205 +4283947815 +4283946210 +4283945679 +4283945043 +4283943954 +4283942598 +4283941593 +4283941389 +4283940564 +4283938749 +4283934795 +4283934135 +4283933295 +4283932983 +4283931855 +4283930484 +4283930055 +4283929245 +4283927934 +4283926344 +4283924928 +4283924490 +4283922483 +4283921379 +4283920563 +4283919444 +4283919018 +4283918943 +4283918154 +4283917869 +4283917539 +4283916339 +4283915685 +4283915403 +4283914890 +4283914830 +4283914230 +4283913933 +4283913504 +4283912304 +4283912253 +4283911575 +4283911458 +4283911029 +4283910603 +4283908053 +4283907333 +4283907204 +4283906889 +4283904570 +4283903130 +4283902818 +4283902224 +4283900853 +4283899059 +4283897670 +4283897493 +4283897358 +4283896938 +4283896674 +4283896275 +4283895819 +4283895045 +4283894055 +4283893083 +4283893008 +4283892534 +4283891928 +4283891583 +4283891463 +4283891064 +4283890050 +4283888643 +4283887695 +4283884734 +4283883873 +4283883528 +4283883180 +4283882655 +4283882610 +4283881929 +4283881329 +4283880963 +4283879865 +4283879409 +4283878320 +4283878170 +4283877165 +4283876544 +4283874189 +4283871744 +4283871318 +4283871228 +4283870739 +4283870475 +4283869353 +4283869350 +4283868939 +4283868828 +4283868708 +4283868480 +4283868090 +4283866818 +4283866719 +4283865213 +4283864739 +4283863143 +4283861433 +4283861259 +4283860140 +4283859585 +4283859420 +4283859018 +4283857164 +4283857068 +4283856183 +4283855625 +4283855364 +4283854569 +4283854104 +4283852625 +4283851515 +4283850303 +4283847804 +4283847720 +4283847459 +4283847408 +4283846598 +4283845908 +4283845188 +4283844474 +4283844369 +4283842779 +4283841420 +4283841219 +4283840349 +4283839950 +4283839923 +4283839815 +4283837250 +4283837199 +4283836788 +4283836455 +4283836140 +4283836044 +4283836005 +4283835465 +4283834544 +4283834094 +4283834073 +4283834040 +4283833728 +4283832693 +4283831160 +4283830878 +4283830383 +4283829948 +4283829054 +4283828559 +4283827773 +4283825469 +4283825034 +4283824329 +4283823960 +4283822460 +4283821905 +4283821023 +4283818563 +4283817315 +4283816190 +4283815599 +4283815590 +4283814414 +4283813985 +4283811804 +4283811189 +4283810868 +4283810184 +4283810175 +4283810058 +4283809410 +4283809389 +4283808474 +4283807823 +4283801259 +4283800275 +4283800083 +4283800074 +4283798949 +4283798370 +4283798274 +4283797815 +4283797335 +4283796615 +4283796525 +4283794800 +4283793360 +4283793204 +4283792910 +4283792859 +4283792229 +4283792070 +4283791524 +4283790939 +4283790414 +4283789985 +4283789874 +4283789583 +4283789523 +4283788869 +4283788530 +4283787879 +4283787663 +4283787495 +4283787465 +4283787378 +4283786844 +4283786739 +4283786340 +4283783784 +4283782719 +4283782695 +4283782263 +4283781753 +4283780775 +4283779860 +4283779458 +4283778240 +4283778198 +4283777910 +4283777895 +4283777373 +4283777289 +4283776869 +4283776290 +4283776134 +4283773824 +4283773515 +4283772858 +4283772825 +4283770830 +4283770053 +4283767020 +4283766228 +4283764464 +4283764434 +4283762328 +4283761284 +4283760249 +4283758479 +4283757618 +4283757540 +4283757423 +4283757285 +4283756520 +4283755488 +4283755239 +4283754840 +4283754618 +4283753118 +4283752794 +4283752599 +4283752269 +4283751723 +4283749875 +4283748918 +4283747580 +4283746764 +4283746290 +4283745000 +4283743869 +4283743770 +4283743644 +4283743485 +4283742309 +4283741964 +4283740614 +4283740593 +4283740314 +4283740254 +4283739984 +4283739915 +4283738229 +4283736945 +4283736675 +4283734995 +4283732988 +4283731698 +4283730933 +4283730780 +4283730669 +4283730120 +4283728329 +4283727969 +4283726595 +4283726028 +4283725983 +4283725710 +4283725494 +4283725050 +4283724849 +4283723640 +4283722689 +4283722155 +4283721384 +4283720820 +4283720580 +4283720280 +4283720064 +4283719188 +4283718909 +4283718348 +4283718165 +4283718045 +4283717775 +4283717130 +4283716899 +4283716260 +4283715729 +4283712210 +4283710374 +4283709918 +4283709603 +4283708595 +4283706978 +4283705160 +4283703618 +4283701089 +4283700975 +4283700693 +4283699763 +4283698203 +4283696520 +4283695374 +4283695275 +4283695053 +4283693988 +4283693070 +4283692425 +4283691543 +4283691144 +4283691048 +4283690628 +4283690298 +4283688543 +4283686944 +4283686833 +4283685300 +4283685180 +4283685108 +4283684418 +4283683365 +4283683323 +4283682243 +4283681403 +4283681139 +4283679834 +4283679825 +4283679303 +4283679135 +4283679060 +4283677884 +4283676495 +4283676360 +4283672679 +4283672625 +4283672595 +4283670594 +4283670159 +4283669193 +4283667555 +4283667375 +4283665923 +4283665284 +4283663628 +4283662689 +4283661420 +4283660160 +4283659530 +4283657649 +4283657379 +4283657253 +4283657118 +4283656434 +4283654619 +4283654220 +4283652378 +4283652273 +4283651823 +4283651163 +4283650194 +4283648904 +4283647233 +4283645553 +4283645514 +4283643729 +4283642244 +4283638278 +4283636829 +4283636160 +4283635005 +4283634645 +4283634225 +4283633658 +4283632443 +4283631939 +4283631588 +4283629203 +4283628324 +4283628054 +4283627664 +4283626689 +4283625774 +4283624850 +4283623998 +4283623590 +4283623344 +4283623293 +4283622663 +4283621229 +4283620923 +4283620419 +4283618328 +4283617005 +4283616693 +4283615574 +4283615553 +4283614875 +4283614515 +4283614434 +4283613393 +4283613213 +4283612850 +4283611284 +4283610885 +4283610834 +4283610288 +4283610000 +4283608425 +4283607033 +4283606778 +4283606523 +4283606325 +4283606244 +4283605983 +4283605980 +4283605179 +4283604249 +4283604153 +4283603208 +4283602968 +4283602860 +4283602464 +4283600658 +4283600055 +4283599545 +4283599335 +4283599269 +4283598744 +4283598735 +4283598525 +4283598483 +4283597793 +4283596968 +4283596029 +4283595204 +4283594385 +4283594325 +4283594070 +4283592993 +4283592879 +4283591973 +4283591688 +4283590653 +4283589948 +4283589939 +4283589828 +4283589660 +4283589003 +4283588673 +4283587848 +4283587635 +4283585493 +4283585160 +4283583150 +4283582880 +4283581179 +4283581095 +4283579769 +4283578713 +4283578200 +4283578134 +4283576568 +4283576340 +4283574669 +4283573988 +4283573958 +4283573049 +4283572275 +4283571354 +4283570808 +4283570754 +4283570670 +4283570094 +4283569974 +4283568309 +4283567919 +4283567820 +4283566938 +4283566665 +4283566650 +4283565180 +4283565060 +4283564859 +4283564724 +4283561778 +4283560923 +4283560158 +4283559948 +4283559834 +4283559210 +4283556444 +4283553879 +4283552475 +4283551944 +4283551884 +4283551725 +4283551569 +4283551413 +4283551398 +4283551089 +4283550309 +4283549784 +4283549514 +4283548104 +4283547933 +4283547849 +4283547630 +4283547408 +4283547135 +4283546994 +4283546538 +4283546100 +4283544600 +4283542119 +4283541108 +4283539884 +4283539254 +4283538579 +4283535789 +4283535774 +4283534979 +4283534529 +4283534160 +4283533995 +4283532975 +4283531784 +4283531589 +4283529453 +4283529369 +4283528943 +4283528430 +4283528058 +4283528040 +4283527734 +4283527689 +4283527575 +4283527374 +4283526579 +4283526234 +4283525550 +4283525319 +4283523189 +4283523138 +4283522229 +4283521695 +4283520549 +4283520450 +4283518023 +4283517663 +4283516205 +4283515650 +4283515545 +4283515458 +4283515095 +4283514840 +4283514633 +4283513520 +4283512695 +4283510670 +4283510253 +4283510148 +4283508864 +4283507808 +4283507340 +4283507004 +4283506209 +4283505855 +4283505798 +4283505198 +4283504538 +4283503518 +4283502810 +4283502165 +4283501934 +4283500200 +4283500089 +4283500035 +4283499855 +4283499570 +4283499144 +4283498490 +4283497905 +4283497875 +4283497833 +4283497743 +4283497428 +4283495499 +4283495433 +4283495088 +4283494788 +4283494605 +4283494440 +4283494230 +4283493390 +4283492748 +4283492079 +4283490414 +4283489673 +4283489379 +4283488503 +4283486844 +4283486625 +4283485545 +4283485494 +4283484840 +4283484609 +4283483673 +4283483484 +4283480208 +4283477640 +4283476428 +4283475843 +4283475273 +4283474403 +4283473848 +4283473515 +4283472600 +4283472465 +4283472453 +4283471190 +4283471073 +4283469744 +4283468304 +4283464860 +4283464395 +4283464050 +4283461938 +4283461488 +4283460753 +4283459349 +4283458599 +4283457963 +4283457765 +4283457618 +4283457594 +4283457270 +4283456520 +4283456100 +4283454195 +4283453754 +4283453415 +4283452965 +4283452323 +4283451894 +4283451570 +4283451534 +4283451423 +4283450094 +4283448540 +4283447250 +4283445333 +4283444280 +4283443680 +4283442765 +4283442225 +4283441760 +4283441148 +4283439915 +4283439723 +4283437518 +4283436444 +4283435724 +4283435103 +4283434335 +4283431515 +4283430249 +4283429748 +4283429304 +4283428959 +4283428059 +4283426430 +4283426145 +4283425803 +4283424474 +4283424150 +4283423619 +4283423574 +4283423028 +4283422983 +4283421108 +4283420973 +4283420868 +4283420865 +4283420799 +4283420553 +4283420475 +4283420373 +4283420223 +4283416200 +4283415540 +4283415093 +4283414703 +4283414205 +4283412978 +4283412759 +4283412468 +4283411958 +4283411205 +4283410899 +4283410785 +4283410740 +4283409135 +4283408940 +4283408853 +4283408610 +4283408034 +4283407974 +4283406993 +4283406030 +4283405523 +4283405244 +4283404503 +4283403915 +4283403474 +4283403384 +4283402829 +4283401344 +4283401020 +4283400528 +4283399679 +4283396463 +4283396355 +4283395968 +4283395905 +4283394780 +4283394774 +4283394678 +4283394603 +4283393658 +4283393463 +4283392044 +4283391315 +4283391003 +4283390658 +4283390655 +4283390385 +4283389734 +4283389473 +4283389419 +4283388129 +4283387349 +4283384895 +4283383119 +4283383035 +4283382363 +4283382159 +4283381760 +4283380974 +4283380473 +4283380395 +4283379189 +4283379135 +4283378763 +4283378760 +4283378475 +4283378148 +4283378034 +4283377974 +4283377164 +4283376273 +4283376240 +4283376039 +4283372289 +4283372133 +4283371515 +4283371503 +4283370264 +4283369853 +4283368578 +4283368113 +4283366634 +4283366583 +4283366199 +4283365968 +4283364615 +4283364540 +4283364210 +4283363304 +4283362599 +4283362083 +4283361960 +4283360889 +4283358684 +4283357184 +4283356953 +4283354253 +4283353500 +4283353224 +4283353164 +4283351289 +4283349495 +4283348964 +4283348649 +4283348109 +4283346129 +4283345613 +4283344983 +4283344518 +4283343585 +4283341788 +4283341623 +4283341443 +4283341293 +4283341158 +4283340624 +4283339595 +4283339499 +4283339175 +4283338113 +4283337483 +4283336919 +4283336805 +4283336685 +4283336274 +4283336043 +4283335344 +4283334684 +4283334609 +4283334483 +4283334198 +4283333085 +4283332119 +4283331645 +4283331603 +4283330625 +4283329149 +4283328678 +4283327985 +4283327970 +4283327163 +4283327160 +4283326074 +4283325420 +4283324988 +4283324820 +4283324520 +4283324430 +4283324244 +4283324043 +4283323848 +4283323779 +4283323464 +4283322240 +4283320119 +4283319759 +4283318550 +4283318208 +4283318028 +4283317389 +4283316390 +4283316255 +4283314605 +4283314575 +4283314440 +4283314398 +4283313723 +4283313588 +4283312064 +4283308623 +4283308380 +4283308224 +4283307708 +4283306655 +4283306508 +4283305860 +4283304924 +4283303718 +4283303655 +4283303109 +4283302395 +4283301618 +4283300769 +4283300580 +4283299758 +4283299629 +4283298645 +4283297718 +4283296710 +4283296620 +4283296503 +4283296305 +4283296008 +4283295843 +4283292795 +4283291250 +4283290803 +4283289993 +4283289699 +4283288784 +4283288094 +4283286675 +4283286399 +4283285280 +4283285079 +4283285028 +4283284683 +4283284584 +4283283984 +4283283138 +4283282769 +4283281575 +4283281539 +4283281035 +4283281029 +4283280753 +4283280468 +4283279748 +4283279274 +4283278950 +4283278608 +4283278518 +4283277165 +4283276613 +4283275248 +4283275128 +4283274519 +4283274264 +4283274198 +4283273295 diff --git a/ot/gpu/cudamat/examples/bench_cudamat.py b/ot/gpu/cudamat/examples/bench_cudamat.py new file mode 100644 index 0000000..b3a5c19 --- /dev/null +++ b/ot/gpu/cudamat/examples/bench_cudamat.py @@ -0,0 +1,97 @@ +from __future__ import print_function, division +import sys +import numpy as np +import cudamat as cmt +import time +import timeit +from inspect import getmodule, getmembers, isfunction +try: from itertools import ifilter as filter +except: pass + +# heat-up time in seconds before starting the benchmark +HEATUP = 2 + +# shapes used for the small and large test matrix +XS_SHAPE = (400, 256) +XL_SHAPE = (4096, 4096) + +# timeit number and repeat parameter +NUM_ITER = 100 +NUM_REPEATS = 5 + +def setup(shape): + """Creates two matrices and corresponding row/column vectors""" + mat = cmt.empty(shape).fill_with_randn() + mat2 = cmt.empty(shape).fill_with_randn() + col = cmt.empty((shape[0], 1)).assign(0) + row = cmt.empty((1, shape[1])).assign(0) + return mat, mat2, col, row + +def bench_dot(X, Y, col, row): + cmt.dot(X.T, Y) + +def bench_add(X, Y, col, row): + X.add(Y) +bench_add.repeats = 5 # 5 times more repetitions than usual + +def bench_mult(X, Y, col, row): + X.mult(Y) + +def bench_sigm(X, Y, col, row): + X.apply_sigmoid() + +def bench_colsum(X, Y, col, row): + X.sum(axis=0, target=row) + +def bench_rowsum(X, Y, col, row): + X.sum(axis=1, target=col) + +def bench_addcolsum(X, Y, col, row): + row.add_sums(X, axis=0, mult=3.2, beta=0.2) + +def bench_addrowsum(X, Y, col, row): + col.add_sums(X, axis=1, mult=3.2, beta=0.2) + +def bench_colmax(X, Y, col, row): + X.max(axis=0, target=row) + +def bench_rowmax(X, Y, col, row): + X.max(axis=1, target=col) + +def bench_addcolmult(X, Y, col, row): + X.add_col_mult(col, mult=3.2) + +def heatup(duration): + """Heat-up the GPU for a while so it enters full-performance mode""" + t1 = time.time() + while time.time() - t1 < duration: + cmt.dot(cmt.empty((200, 200)), cmt.empty((200, 200))) + +def main(): + cmt.init() + cmt.CUDAMatrix.init_random() + if HEATUP: + print("heating up for %g seconds..." % HEATUP, end=' ') + sys.stdout.flush() + heatup(HEATUP) + print("done.") + print("small matrix shape:", XS_SHAPE) + print("large matrix shape:", XL_SHAPE) + for funcname, func in filter(lambda f: f[0].startswith('bench_'), + getmembers(getmodule(main), isfunction)): + print("%-15s" % funcname[len('bench_'):], end=' ') + sys.stdout.flush() + for size, shape, factor in ('small', XS_SHAPE, 10), ('large', XL_SHAPE, 1): + repeat = NUM_REPEATS * getattr(func, 'repeats', 1) + time = min(timeit.repeat(\ + setup="from __main__ import setup, %s\nmats = setup(%s)" % (funcname, shape), + stmt="%s(*mats)" % funcname, repeat=repeat, + number=NUM_ITER * factor)) / (NUM_ITER * factor) + print("%.3es (%s) " % (time, size), end=' ') + sys.stdout.flush() + print() + cmt.shutdown() + +if __name__=="__main__": + main() + diff --git a/ot/gpu/cudamat/examples/nn_cudamat.py b/ot/gpu/cudamat/examples/nn_cudamat.py new file mode 100644 index 0000000..7c56c7d --- /dev/null +++ b/ot/gpu/cudamat/examples/nn_cudamat.py @@ -0,0 +1,133 @@ +# This file shows how to implement a single hidden layer neural network for +# performing binary classification on the GPU using cudamat. + +from __future__ import division +import pdb +import time +import numpy as np +import cudamat as cm +from cudamat import learn as cl +import util + +# initialize CUDA +cm.cublas_init() + +# load data +util.load('mnist49.dat', globals()) + +# Put training data onto the GPU. +dat_train = dat_train/255. +dat_train = dat_train - (np.mean(dat_train, 1)+10**-8)[:, np.newaxis] +dev_train = cm.CUDAMatrix(dat_train) +dev_lbl = cm.CUDAMatrix(lbl_train) + +# training parameters +epsilon = 0.01 +momentum = 0.9 + +num_epochs = 30 +batch_size = 128 +num_batches = dat_train.shape[1]//batch_size + +# model parameters +dim_in = dat_train.shape[0] +dim_out = 1 +num_hid = 1024 + +# initialize weights +w_w1 = cm.CUDAMatrix(dim_in ** -0.5 * np.random.randn(dim_in, num_hid)) +w_b1 = cm.CUDAMatrix(np.zeros((num_hid, 1))) +w_w2 = cm.CUDAMatrix(num_hid ** -0.5 * np.random.randn(num_hid, dim_out)) +w_b2 = cm.CUDAMatrix(np.zeros((dim_out, 1))) + +# initialize weight update matrices +wu_w1 = cm.empty(w_w1.shape).assign(0) +wu_b1 = cm.empty(w_b1.shape).assign(0) +wu_w2 = cm.empty(w_w2.shape).assign(0) +wu_b2 = cm.empty(w_b2.shape).assign(0) + +# initialize temporary storage +h = cm.empty((num_hid, batch_size)) +out = cm.empty((dim_out, batch_size)) +delta = cm.empty((num_hid, batch_size)) + +# Train neural network. +start_time = time.time() +for epoch in range(num_epochs): + print("Epoch %i" % (epoch + 1)) + err = [] + + for batch in range(num_batches): + # get current minibatch + inp = dev_train.slice(batch*batch_size,(batch + 1)*batch_size) + target = dev_lbl.slice(batch*batch_size,(batch + 1)*batch_size) + + # forward pass + cm.dot(w_w1.T, inp, target = h) + + h.add_col_vec(w_b1) + h.apply_sigmoid() + + cm.dot(w_w2.T, h, target = out) + + out.add_col_vec(w_b2) + out.apply_sigmoid() + + # back prop errors + out.subtract(target) # compute error + + # gradients for w_w2 and w_b2 + wu_w2.add_dot(h, out.T, beta = momentum) + wu_b2.add_sums(out, axis = 1, beta = momentum) + + # compute delta + cm.dot(w_w2, out, target = delta) + + # delta = delta * h * (1 - h) + cl.mult_by_sigmoid_deriv(delta, h) + + # gradients for w_w1 and w_b1 + wu_w1.add_dot(inp, delta.T, beta = momentum) + wu_b1.add_sums(delta, axis = 1, beta = momentum) + + # update weights + w_w1.subtract_mult(wu_w1, epsilon/batch_size) + w_b1.subtract_mult(wu_b1, epsilon/batch_size) + w_w2.subtract_mult(wu_w2, epsilon/batch_size) + w_b2.subtract_mult(wu_b2, epsilon/batch_size) + + # calculate error on current minibatch + err.append(np.abs(out.asarray())>0.5) + + print("Training misclassification rate: %f" % np.mean(err)) + print("Time: %f" % (time.time() - start_time)) + +# Evaluate neural network on test data. + +# Load test data onto the GPU. +dat_test = dat_test/255. +dat_test = dat_test - np.mean(dat_test, 1)[:, np.newaxis] +dev_test = cm.CUDAMatrix(dat_test) +dev_lbl = cm.CUDAMatrix(lbl_test) + +# Initalize temporary storage. +h = cm.empty((num_hid, dat_test.shape[1])) +out = cm.empty((dim_out, dat_test.shape[1])) + +# forward pass +cm.dot(w_w1.T, dev_test, target = h) + +h.add_col_vec(w_b1) +h.apply_sigmoid() + +cm.dot(w_w2.T, h, target = out) + +out.add_col_vec(w_b2) +out.apply_sigmoid() + +# compute error +out.subtract(dev_lbl) + +print("Testing misclassification rate: %f" % np.mean(np.abs(out.asarray())>0.5)) + +cm.cublas_shutdown() diff --git a/ot/gpu/cudamat/examples/rbm_cudamat.py b/ot/gpu/cudamat/examples/rbm_cudamat.py new file mode 100644 index 0000000..3f6a900 --- /dev/null +++ b/ot/gpu/cudamat/examples/rbm_cudamat.py @@ -0,0 +1,98 @@ +from __future__ import division +import time +import numpy as np +import cudamat as cm +import util + +# initialize CUDA +cm.cublas_init() +cm.CUDAMatrix.init_random(1) + +# load data +util.load('mnist.dat', globals()) +dev_dat = cm.CUDAMatrix(cm.reformat(dat/255.)) + +# training parameters +epsilon = 0.1 +momentum = 0.9 + +num_epochs = 30 +batch_size = 128 +num_batches = dat.shape[1]//batch_size + +# model parameters +num_vis = dat.shape[0] +num_hid = 4096 + +# initialize weights +w_vh = cm.CUDAMatrix(0.1 * np.random.randn(num_vis, num_hid)) +w_v = cm.CUDAMatrix(np.zeros((num_vis, 1))) +w_h = cm.CUDAMatrix(-4.*np.ones((num_hid, 1))) + +# initialize weight updates +wu_vh = cm.CUDAMatrix(np.zeros((num_vis, num_hid))) +wu_v = cm.CUDAMatrix(np.zeros((num_vis, 1))) +wu_h = cm.CUDAMatrix(np.zeros((num_hid, 1))) + +# initialize temporary storage +v = cm.empty((num_vis, batch_size)) +h = cm.empty((num_hid, batch_size)) +r = cm.empty((num_hid, batch_size)) + +start_time = time.time() +for epoch in range(num_epochs): + print("Epoch %i" % (epoch + 1)) + err = [] + + for batch in range(num_batches): + # get current minibatch + v_true = dev_dat.slice(batch*batch_size,(batch + 1)*batch_size) + v.assign(v_true) + + # apply momentum + wu_vh.mult(momentum) + wu_v.mult(momentum) + wu_h.mult(momentum) + + # positive phase + cm.dot(w_vh.T, v, target = h) + h.add_col_vec(w_h) + h.apply_sigmoid() + + wu_vh.add_dot(v, h.T) + wu_v.add_sums(v, axis = 1) + wu_h.add_sums(h, axis = 1) + + # sample hiddens + r.fill_with_rand() + r.less_than(h, target = h) + + # negative phase + cm.dot(w_vh, h, target = v) + v.add_col_vec(w_v) + v.apply_sigmoid() + + cm.dot(w_vh.T, v, target = h) + h.add_col_vec(w_h) + h.apply_sigmoid() + + wu_vh.subtract_dot(v, h.T) + wu_v.add_sums(v, axis = 1, mult = -1.) + wu_h.add_sums(h, axis = 1, mult = -1.) + + # update weights + w_vh.add_mult(wu_vh, epsilon/batch_size) + w_v.add_mult(wu_v, epsilon/batch_size) + w_h.add_mult(wu_h, epsilon/batch_size) + + # calculate reconstruction error + v.subtract(v_true) + err.append(v.euclid_norm()**2/(num_vis*batch_size)) + + print("Mean squared error: %f" % np.mean(err)) + print("Time: %f" % (time.time() - start_time)) + +w_vh.copy_to_host() +util.save('weights.dat', 'w_vh', {'w_vh': w_vh.numpy_array}) + +cm.cublas_shutdown() diff --git a/ot/gpu/cudamat/examples/rbm_numpy.py b/ot/gpu/cudamat/examples/rbm_numpy.py new file mode 100644 index 0000000..1331566 --- /dev/null +++ b/ot/gpu/cudamat/examples/rbm_numpy.py @@ -0,0 +1,72 @@ +from __future__ import division +import time +import numpy as np +import util + +# load data +util.load('mnist.dat', globals()) +dat = dat/255. + +# training parameters +epsilon = 0.01 +momentum = 0.9 + +num_epochs = 10 +batch_size = 64 +num_batches = dat.shape[1]//batch_size + +# model parameters +num_vis = dat.shape[0] +num_hid = 1024 + +# initialize weights +w_vh = 0.1 * np.random.randn(num_vis, num_hid) +w_v = np.zeros((num_vis, 1)) +w_h = np.zeros((num_hid, 1)) + +# initialize weight updates +wu_vh = np.zeros((num_vis, num_hid)) +wu_v = np.zeros((num_vis, 1)) +wu_h = np.zeros((num_hid, 1)) + +start_time = time.time() +for epoch in range(num_epochs): + print("Epoch %i" % (epoch + 1)) + err = [] + + for batch in range(num_batches): + v_true = dat[:, batch*batch_size:(batch + 1)*batch_size] + v = v_true + + # apply momentum + wu_vh *= momentum + wu_v *= momentum + wu_h *= momentum + + # positive phase + h = 1. / (1 + np.exp(-(np.dot(w_vh.T, v) + w_h))) + + wu_vh += np.dot(v, h.T) + wu_v += v.sum(1)[:, np.newaxis] + wu_h += h.sum(1)[:, np.newaxis] + + # sample hiddens + h = 1. * (h > np.random.rand(num_hid, batch_size)) + + # negative phase + v = 1. / (1 + np.exp(-(np.dot(w_vh, h) + w_v))) + h = 1. / (1 + np.exp(-(np.dot(w_vh.T, v) + w_h))) + + wu_vh -= np.dot(v, h.T) + wu_v -= v.sum(1)[:, np.newaxis] + wu_h -= h.sum(1)[:, np.newaxis] + + # update weights + w_vh += epsilon/batch_size * wu_vh + w_v += epsilon/batch_size * wu_v + w_h += epsilon/batch_size * wu_h + + err.append(np.mean((v - v_true)**2)) + + print("Mean squared error: %f" % np.mean(err)) + print("Time: %f" % (time.time() - start_time)) diff --git a/ot/gpu/cudamat/examples/util.py b/ot/gpu/cudamat/examples/util.py new file mode 100644 index 0000000..79ceead --- /dev/null +++ b/ot/gpu/cudamat/examples/util.py @@ -0,0 +1,22 @@ +from __future__ import division +import gzip +try: import cPickle as pickle +except: import pickle + +def save(fname, var_list, source_dict): + var_list = [var.strip() for var in var_list.split() if len(var.strip())>0] + fo = gzip.GzipFile(fname, 'wb') + pickle.dump(var_list, fo) + for var in var_list: + pickle.dump(source_dict[var], fo, protocol=2) + fo.close() + +def load(fname, target_dict, verbose = True): + fo = gzip.GzipFile(fname, 'rb') + var_list = pickle.load(fo) + if verbose: + print(var_list) + for var in var_list: + target_dict[var] = pickle.load(fo) + fo.close() + diff --git a/ot/gpu/cudamat/setup.py b/ot/gpu/cudamat/setup.py new file mode 100755 index 0000000..ad386d1 --- /dev/null +++ b/ot/gpu/cudamat/setup.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python + +import os +# on Windows, we need the original PATH without Anaconda's compiler in it: +PATH = os.environ.get('PATH') +from distutils.spawn import spawn, find_executable +from setuptools import setup, find_packages, Extension +from setuptools.command.build_ext import build_ext +import sys + +# CUDA specific config +# nvcc is assumed to be in user's PATH +nvcc_compile_args = ['-O', '--ptxas-options=-v', '--compiler-options=-fPIC'] +nvcc_compile_args = os.environ.get('NVCCFLAGS', '').split() + nvcc_compile_args +cuda_libs = ['cublas'] + +cudamat_ext = Extension('cudamat.libcudamat', + sources=['cudamat/cudamat.cu', + 'cudamat/cudamat_kernels.cu'], + libraries=cuda_libs, + extra_compile_args=nvcc_compile_args) +cudalearn_ext = Extension('cudamat.libcudalearn', + sources=['cudamat/learn.cu', + 'cudamat/learn_kernels.cu'], + libraries=cuda_libs, + extra_compile_args=nvcc_compile_args) + + +class CUDA_build_ext(build_ext): + """ + Custom build_ext command that compiles CUDA files. + Note that all extension source files will be processed with this compiler. + """ + def build_extensions(self): + self.compiler.src_extensions.append('.cu') + self.compiler.set_executable('compiler_so', 'nvcc') + self.compiler.set_executable('linker_so', 'nvcc --shared') + if hasattr(self.compiler, '_c_extensions'): + self.compiler._c_extensions.append('.cu') # needed for Windows + self.compiler.spawn = self.spawn + build_ext.build_extensions(self) + + def spawn(self, cmd, search_path=1, verbose=0, dry_run=0): + """ + Perform any CUDA specific customizations before actually launching + compile/link etc. commands. + """ + if (sys.platform == 'darwin' and len(cmd) >= 2 and cmd[0] == 'nvcc' and + cmd[1] == '--shared' and cmd.count('-arch') > 0): + # Versions of distutils on OSX earlier than 2.7.9 inject + # '-arch x86_64' which we need to strip while using nvcc for + # linking + while True: + try: + index = cmd.index('-arch') + del cmd[index:index+2] + except ValueError: + break + elif self.compiler.compiler_type == 'msvc': + # There are several things we need to do to change the commands + # issued by MSVCCompiler into one that works with nvcc. In the end, + # it might have been easier to write our own CCompiler class for + # nvcc, as we're only interested in creating a shared library to + # load with ctypes, not in creating an importable Python extension. + # - First, we replace the cl.exe or link.exe call with an nvcc + # call. In case we're running Anaconda, we search cl.exe in the + # original search path we captured further above -- Anaconda + # inserts a MSVC version into PATH that is too old for nvcc. + cmd[:1] = ['nvcc', '--compiler-bindir', + os.path.dirname(find_executable("cl.exe", PATH)) + or cmd[0]] + # - Secondly, we fix a bunch of command line arguments. + for idx, c in enumerate(cmd): + # create .dll instead of .pyd files + if '.pyd' in c: cmd[idx] = c = c.replace('.pyd', '.dll') + # replace /c by -c + if c == '/c': cmd[idx] = '-c' + # replace /DLL by --shared + elif c == '/DLL': cmd[idx] = '--shared' + # remove --compiler-options=-fPIC + elif '-fPIC' in c: del cmd[idx] + # replace /Tc... by ... + elif c.startswith('/Tc'): cmd[idx] = c[3:] + # replace /Fo... by -o ... + elif c.startswith('/Fo'): cmd[idx:idx+1] = ['-o', c[3:]] + # replace /LIBPATH:... by -L... + elif c.startswith('/LIBPATH:'): cmd[idx] = '-L' + c[9:] + # replace /OUT:... by -o ... + elif c.startswith('/OUT:'): cmd[idx:idx+1] = ['-o', c[5:]] + # remove /EXPORT:initlibcudamat or /EXPORT:initlibcudalearn + elif c.startswith('/EXPORT:'): del cmd[idx] + # replace cublas.lib by -lcublas + elif c == 'cublas.lib': cmd[idx] = '-lcublas' + # - Finally, we pass on all arguments starting with a '/' to the + # compiler or linker, and have nvcc handle all other arguments + if '--shared' in cmd: + pass_on = '--linker-options=' + # we only need MSVCRT for a .dll, remove CMT if it sneaks in: + cmd.append('/NODEFAULTLIB:libcmt.lib') + else: + pass_on = '--compiler-options=' + cmd = ([c for c in cmd if c[0] != '/'] + + [pass_on + ','.join(c for c in cmd if c[0] == '/')]) + # For the future: Apart from the wrongly set PATH by Anaconda, it + # would suffice to run the following for compilation on Windows: + # nvcc -c -O -o .obj .cu + # And the following for linking: + # nvcc --shared -o .dll .obj .obj -lcublas + # This could be done by a NVCCCompiler class for all platforms. + spawn(cmd, search_path, verbose, dry_run) + +setup(name="cudamat", + version="0.3", + description="Performs linear algebra computation on the GPU via CUDA", + ext_modules=[cudamat_ext, cudalearn_ext], + packages=find_packages(exclude=['examples', 'test']), + include_package_data=True, + package_data={'cudamat': ['rnd_multipliers_32bit.txt']}, + author="Volodymyr Mnih", + url="https://github.com/cudamat/cudamat", + cmdclass={'build_ext': CUDA_build_ext}) diff --git a/ot/gpu/cudamat/test/test_cudamat.py b/ot/gpu/cudamat/test/test_cudamat.py new file mode 100644 index 0000000..800243c --- /dev/null +++ b/ot/gpu/cudamat/test/test_cudamat.py @@ -0,0 +1,1217 @@ +import numpy as np +import nose +import cudamat as cm + +def setup(): + cm.cublas_init() + +def teardown(): + cm.cublas_shutdown() + +def test_reshape(): + m = 256 + n = 1 + cm1 = np.array(np.random.rand(n, m)*10, dtype=np.float64, order='F') + cm2 = np.array(np.random.rand(m, n)*10, dtype=np.float64, order='F') + + gm1 = cm.CUDAMatrix(cm1) + gm2 = cm.CUDAMatrix(cm2) + + gm1.reshape((m, n)) + gm2.assign(gm1) + gm1.reshape((n, m)) + + gm1.copy_to_host() + gm2.copy_to_host() + + assert np.max(np.abs(gm1.numpy_array - gm2.numpy_array.T)) < 10**-2, "Error in CUDAMatrix.reshape exceeded threshold" + +def test_T_field(): + m = 256 + n = 128 + cm1 = np.array(np.random.rand(n, m)*10, dtype=np.float64, order='F') + cm2 = np.array(np.random.rand(m, 1)*10, dtype=np.float64, order='F') + gm1 = cm.CUDAMatrix(cm1) + gm2 = cm.CUDAMatrix(cm2) + + # test dot + gm = cm.dot(gm2.T, gm1.T) + c = np.dot(cm2.T, cm1.T) + gm.copy_to_host() + + assert np.max(np.abs(gm.numpy_array - c)) < 10**-2, "Error in CUDAMatrix.dot with TransposedCUDAMatrix exceeded threshold" + + # test add_dot + cm3 = np.array(np.random.rand(1, n)*10, dtype=np.float64, order='F') + gm3 = cm.CUDAMatrix(cm3) + gm3.add_dot(gm2.T, gm1.T) + c = cm3 + np.dot(cm2.T, cm1.T) + gm3.copy_to_host() + + assert np.max(np.abs(gm3.numpy_array - c)) < 10**-2, "Error in CUDAMatrix.add_dot TransposedCUDAMatrix exceeded threshold" + + # test add_sums + gm2.add_sums(gm1.T, axis = 1) + c = cm2 + np.atleast_2d(cm1.sum(0)).T + gm2.copy_to_host() + + assert np.max(np.abs(gm2.numpy_array - c)) < 10**-2, "Error in CUDAMatrix.add_sums TransposedCUDAMatrix exceeded threshold" + +def test_assign(): + m = 256 + n = 128 + a = np.array(np.random.rand(m, n)*10, dtype=np.float64, order='F') + b = np.array(np.random.rand(m, n)*10, dtype=np.float64, order='F') + + m1 = cm.CUDAMatrix(a) + m2 = cm.CUDAMatrix(b) + + m1.assign(m2) + m1.copy_to_host() + + assert np.max(np.abs(m1.numpy_array - m2.numpy_array)) < 10**-4, "Error in CUDAMatrix.assign exceeded threshold" + +def test_assign_scalar(): + m = 256 + n = 128 + a = np.array(np.random.rand(m, n)*10, dtype=np.float64, order='F') + + m1 = cm.CUDAMatrix(a) + + m1.assign(np.pi) + m1.copy_to_host() + + assert np.max(np.abs(m1.numpy_array - np.pi)) < 10**-4, "Error in CUDAMatrix.assign_scalar exceeded threshold" + +def test_get_row_slice(): + m = 256 + n = 128 + start = 11 + end = 54 + + a = np.array(np.random.rand(m, n)*10, dtype=np.float64, order='F') + b = np.array(np.random.rand(end-start, n)*10, dtype=np.float64, order='F') + + c = np.array(a[start:end,:], order='F') + + m1 = cm.CUDAMatrix(a) + m2 = cm.CUDAMatrix(b) + m1.get_row_slice(start, end, target = m2) + m3 = m1.get_row_slice(start, end) + m2.copy_to_host() + m3.copy_to_host() + + assert np.max(np.abs(c - m2.numpy_array)) < 10**-4, "Error in CUDAMatrix.get_row_slice exceeded threshold" + assert np.max(np.abs(c - m3.numpy_array)) < 10**-4, "Error in CUDAMatrix.get_row_slice exceeded threshold" + +def test_set_row_slice(): + m = 256 + n = 128 + start = 11 + end = 54 + + a = np.array(np.random.rand(m, n)*10, dtype=np.float64, order='F') + b = np.array(np.random.rand(end-start, n)*10, dtype=np.float64, order='F') + + c = a.copy() + c[start:end,:] = b + + m1 = cm.CUDAMatrix(a) + m2 = cm.CUDAMatrix(b) + m1.set_row_slice(start, end, m2) + m1.copy_to_host() + + assert np.max(np.abs(c - m1.numpy_array)) < 10**-4, "Error in CUDAMatrix.set_row_slice exceeded threshold" + +def test_transpose(): + m = 6 + n = 128 + + a = np.array(np.random.rand(m, n)*10, dtype=np.float64, order='F') + b = np.array(np.random.rand(n, m), dtype=np.float64, order='F') + + c = a.copy().T + + m = cm.CUDAMatrix(a) + mt1 = cm.CUDAMatrix(b) + m.transpose(target = mt1) + mt2 = m.transpose() + + mt1.copy_to_host() + mt2.copy_to_host() + + assert np.max(np.abs(c - mt1.numpy_array)) < 10**-4, "Error in CUDAMatrix.transpose exceeded threshold" + assert np.max(np.abs(c - mt2.numpy_array)) < 10**-4, "Error in CUDAMatrix.transpose exceeded threshold" + +def test_slice(): + m = 256 + n = 128 + a = np.array(np.random.rand(m, n)*10, dtype=np.float64, order='F') + + c = np.array(a[:,32:64], order='F') + + m1 = cm.CUDAMatrix(a) + m2 = m1.slice(32, 64) + m2.copy_to_host() + + assert np.max(np.abs(c - m2.numpy_array)) < 10**-4, "Error in CUDAMatrix.slice exceeded threshold" + + +def test_add_col_vec(): + m = 250 + n = 120 + a = np.array(np.random.rand(m, n)*10, dtype=np.float64, order='F') + b = np.array(np.random.rand(m, 1)*10, dtype=np.float64, order='F') + t = np.array(np.random.rand(m, n)*10, dtype=np.float64, order='F') + + c = a + b + + m1 = cm.CUDAMatrix(a) + m2 = cm.CUDAMatrix(b) + m3 = cm.CUDAMatrix(t) + + m1.add_col_vec(m2, target = m3) + m1.add_col_vec(m2) + m1.copy_to_host() + m3.copy_to_host() + + assert np.max(np.abs(c - m1.numpy_array)) < 10**-4, "Error in CUDAMatrix.add_col_vec exceeded threshold" + assert np.max(np.abs(c - m3.numpy_array)) < 10**-4, "Error in CUDAMatrix.add_col_vec exceeded threshold" + +def test_add_col_mult(): + m = 256 + n = 128 + mult = np.pi + a = np.array(np.random.rand(m, n)*10, dtype=np.float64, order='F') + b = np.array(np.random.rand(m, 1)*10, dtype=np.float64, order='F') + t = np.array(np.random.rand(m, n)*10, dtype=np.float64, order='F') + + c = a + mult * b + + m1 = cm.CUDAMatrix(a) + m2 = cm.CUDAMatrix(b) + m3 = cm.CUDAMatrix(t) + + m1.add_col_mult(m2, mult, target = m3) + m1.add_col_mult(m2, mult) + m1.copy_to_host() + m3.copy_to_host() + + assert np.max(np.abs(c - m1.numpy_array)) < 10**-4, "Error in CUDAMatrix.add_col_mult exceeded threshold" + assert np.max(np.abs(c - m3.numpy_array)) < 10**-4, "Error in CUDAMatrix.add_col_mult exceeded threshold" + +def test_add_row_vec(): + m = 256 + n = 128 + a = np.array(np.random.rand(m, n)*10, dtype=np.float64, order='F') + b = np.array(np.random.rand(1, n)*10, dtype=np.float64, order='F') + t = np.array(np.random.rand(m, n)*10, dtype=np.float64, order='F') + + c = a + b + + m1 = cm.CUDAMatrix(a) + m2 = cm.CUDAMatrix(b) + m3 = cm.CUDAMatrix(t) + + m1.add_row_vec(m2, target = m3) + m1.add_row_vec(m2) + m1.copy_to_host() + m3.copy_to_host() + + assert np.max(np.abs(c - m1.numpy_array)) < 10**-4, "Error in CUDAMatrix.add_row_vec exceeded threshold" + assert np.max(np.abs(c - m3.numpy_array)) < 10**-4, "Error in CUDAMatrix.add_row_vec exceeded threshold" + +def test_mult_by_col(): + m = 256 + n = 128 + a = np.array(np.random.rand(m, n)*10, dtype=np.float64, order='F') + b = np.array(np.random.rand(m, 1)*10, dtype=np.float64, order='F') + t = np.array(np.random.rand(m, n)*10, dtype=np.float64, order='F') + + c = a * b + + m1 = cm.CUDAMatrix(a) + m2 = cm.CUDAMatrix(b) + m3 = cm.CUDAMatrix(t) + + m1.mult_by_col(m2, target = m3) + m1.mult_by_col(m2) + m1.copy_to_host() + m3.copy_to_host() + + assert np.max(np.abs(c - m1.numpy_array)) < 10**-4, "Error in CUDAMatrix.mult_by_col exceeded threshold" + assert np.max(np.abs(c - m3.numpy_array)) < 10**-4, "Error in CUDAMatrix.mult_by_col exceeded threshold" + +def test_mult_by_row(): + m = 256 + n = 128 + a = np.array(np.random.rand(m, n)*10, dtype=np.float64, order='F') + b = np.array(np.random.rand(1, n)*10, dtype=np.float64, order='F') + t = np.array(np.random.rand(m, n)*10, dtype=np.float64, order='F') + + c = a * b + + m1 = cm.CUDAMatrix(a) + m2 = cm.CUDAMatrix(b) + m3 = cm.CUDAMatrix(t) + + m1.mult_by_row(m2, target = m3) + m1.mult_by_row(m2) + m1.copy_to_host() + m3.copy_to_host() + + assert np.max(np.abs(c - m1.numpy_array)) < 10**-4, "Error in CUDAMatrix.mult_by_row exceeded threshold" + assert np.max(np.abs(c - m3.numpy_array)) < 10**-4, "Error in CUDAMatrix.mult_by_row exceeded threshold" + +def test_div_by_col(): + m = 256 + n = 128 + a = np.array(np.random.rand(m, n)*10, dtype=np.float64, order='F') + b = np.array(np.random.rand(m, 1)*10, dtype=np.float64, order='F') + 0.1 + t = np.array(np.random.rand(m, n)*10, dtype=np.float64, order='F') + + c = a / b + + m1 = cm.CUDAMatrix(a) + m2 = cm.CUDAMatrix(b) + m3 = cm.CUDAMatrix(t) + + m1.div_by_col(m2, target = m3) + m1.div_by_col(m2) + m1.copy_to_host() + m3.copy_to_host() + + assert np.max(np.abs(c - m1.numpy_array)) < 10**-4, "Error in CUDAMatrix.div_by_col exceeded threshold" + assert np.max(np.abs(c - m3.numpy_array)) < 10**-4, "Error in CUDAMatrix.div_by_col exceeded threshold" + +def test_div_by_row(): + m = 256 + n = 128 + a = np.array(np.random.rand(m, n)*10, dtype=np.float64, order='F') + b = np.array(np.random.rand(1, n)*10, dtype=np.float64, order='F') + 0.1 + t = np.array(np.random.rand(m, n)*10, dtype=np.float64, order='F') + + c = a / b + + m1 = cm.CUDAMatrix(a) + m2 = cm.CUDAMatrix(b) + m3 = cm.CUDAMatrix(t) + + m1.div_by_row(m2, target = m3) + m1.div_by_row(m2) + m1.copy_to_host() + m3.copy_to_host() + + assert np.max(np.abs(c - m1.numpy_array)) < 10**-4, "Error in CUDAMatrix.div_by_row exceeded threshold" + assert np.max(np.abs(c - m3.numpy_array)) < 10**-4, "Error in CUDAMatrix.div_by_row exceeded threshold" + +def test_sum(): + m = 256 + n = 128 + a = np.array(np.random.rand(m, n)*10, dtype=np.float64, order='F') + t1 = np.array(np.random.rand(1, n)*10, dtype=np.float64, order='F') + t2 = np.array(np.random.rand(m, 1)*10, dtype=np.float64, order='F') + + mult = 0.8 + c1 = np.atleast_2d(a.sum(0)) * mult + c2 = np.atleast_2d(a.sum(1)).T + + m = cm.CUDAMatrix(a) + mt1 = cm.CUDAMatrix(t1) + mt2 = cm.CUDAMatrix(t2) + + m.sum(axis = 0, target = mt1, mult = mult) + mt1r = m.sum(axis = 0, mult = mult) + + m.sum(axis = 1, target = mt2) + mt2r = m.sum(axis = 1) + + mt1.copy_to_host() + mt1r.copy_to_host() + mt2.copy_to_host() + mt2r.copy_to_host() + + assert np.max(np.abs(c1 - mt1.numpy_array)) < 10**-3, "Error in CUDAMatrix.sum exceeded threshold" + assert np.max(np.abs(c1 - mt1r.numpy_array)) < 10**-3, "Error in CUDAMatrix.sum exceeded threshold" + assert np.max(np.abs(c2 - mt2.numpy_array)) < 10**-3, "Error in CUDAMatrix.sum exceeded threshold" + assert np.max(np.abs(c2 - mt2r.numpy_array)) < 10**-3, "Error in CUDAMatrix.sum exceeded threshold" + +def test_sum_trans(): + m = 256 + n = 128 + a = np.array(np.random.rand(m, n)*10, dtype=np.float64, order='F') + t1 = np.array(np.random.rand(1, m)*10, dtype=np.float64, order='F') + t2 = np.array(np.random.rand(n, 1)*10, dtype=np.float64, order='F') + + c1 = np.atleast_2d(a.T.sum(0)) + c2 = np.atleast_2d(a.T.sum(1)).T + + m = cm.CUDAMatrix(a) + m.set_trans(True) + mt1 = cm.CUDAMatrix(t1) + mt2 = cm.CUDAMatrix(t2) + + m.sum(axis = 0, target = mt1) + mt1r = m.sum(axis = 0) + + m.sum(axis = 1, target = mt2) + mt2r = m.sum(axis = 1) + + mt1.copy_to_host() + mt1r.copy_to_host() + mt2.copy_to_host() + mt2r.copy_to_host() + + assert np.max(np.abs(c1 - mt1.numpy_array)) < 10**-3, "Error in CUDAMatrix.sum exceeded threshold" + assert np.max(np.abs(c1 - mt1r.numpy_array)) < 10**-3, "Error in CUDAMatrix.sum exceeded threshold" + assert np.max(np.abs(c2 - mt2.numpy_array)) < 10**-3, "Error in CUDAMatrix.sum exceeded threshold" + assert np.max(np.abs(c2 - mt2r.numpy_array)) < 10**-3, "Error in CUDAMatrix.sum exceeded threshold" + +def test_mean(): + m = 256 + n = 128 + a = np.array(np.random.rand(m, n)*10, dtype=np.float64, order='F') + t1 = np.array(np.random.rand(1, n)*10, dtype=np.float64, order='F') + t2 = np.array(np.random.rand(m, 1)*10, dtype=np.float64, order='F') + + c1 = np.atleast_2d(a.mean(0)) + c2 = np.atleast_2d(a.mean(1)).T + + m = cm.CUDAMatrix(a) + mt1 = cm.CUDAMatrix(t1) + mt2 = cm.CUDAMatrix(t2) + + m.mean(axis = 0, target = mt1) + mt1r = m.mean(axis = 0) + + m.mean(axis = 1, target = mt2) + mt2r = m.mean(axis = 1) + + mt1.copy_to_host() + mt1r.copy_to_host() + mt2.copy_to_host() + mt2r.copy_to_host() + + assert np.max(np.abs(c1 - mt1.numpy_array)) < 10**-3, "Error in CUDAMatrix.sum exceeded threshold" + assert np.max(np.abs(c1 - mt1r.numpy_array)) < 10**-3, "Error in CUDAMatrix.sum exceeded threshold" + assert np.max(np.abs(c2 - mt2.numpy_array)) < 10**-3, "Error in CUDAMatrix.sum exceeded threshold" + assert np.max(np.abs(c2 - mt2r.numpy_array)) < 10**-3, "Error in CUDAMatrix.sum exceeded threshold" + +def test_add_sums(): + m = 256 + n = 128 + + a = np.array(np.random.rand(m, n)*10, dtype=np.float64, order='F') + t1 = np.array(np.random.rand(m, 1)*10, dtype=np.float64, order='F') + t2 = np.array(np.random.rand(1, n)*10, dtype=np.float64, order='F') + + mult = np.pi + beta = 0.7 + + c1 = beta * t1 + mult * np.atleast_2d(a.sum(1)).T + c2 = t2 + np.atleast_2d(a.sum(0)) + + m = cm.CUDAMatrix(a) + mt1 = cm.CUDAMatrix(t1) + mt2 = cm.CUDAMatrix(t2) + + mt1.add_sums(m, axis = 1, mult = np.pi, beta = beta) + mt2.add_sums(m, axis = 0) + + mt1.copy_to_host() + mt2.copy_to_host() + + assert np.max(np.abs(c1 - mt1.numpy_array)) < 10**-3, "Error in CUDAMatrix.add_sums exceeded threshold" + assert np.max(np.abs(c2 - mt2.numpy_array)) < 10**-3, "Error in CUDAMatrix.add_sums exceeded threshold" + + +def test_less_than(): + m = 256 + n = 128 + a = np.array(np.random.randn(m, n)*10, dtype=np.float64, order='F') + b = np.array(np.random.randn(m, n)*10, dtype=np.float64, order='F') + t1 = np.array(np.random.randn(m, n)*10, dtype=np.float64, order='F') + t2 = np.array(np.random.randn(m, n)*10, dtype=np.float64, order='F') + v = 0.1 + + r1 = 1 * (a < b) + r2 = 1 * (a < v) + + da = cm.CUDAMatrix(a) + db = cm.CUDAMatrix(b) + dt1 = cm.CUDAMatrix(t1) + dt2 = cm.CUDAMatrix(t2) + + da.less_than(db, target = dt1) + da.less_than(v, target = dt2) + da.less_than(db) + + da.copy_to_host() + dt1.copy_to_host() + dt2.copy_to_host() + + assert np.max(np.abs(r1 - da.numpy_array)) < 10**-4, "Error in CUDAMatrix.less_than exceeded threshold" + assert np.max(np.abs(r1 - dt1.numpy_array)) < 10**-4, "Error in CUDAMatrix.less_than exceeded threshold" + assert np.max(np.abs(r2 - dt2.numpy_array)) < 10**-4, "Error in CUDAMatrix.less_than exceeded threshold" + +def test_greater_than(): + m = 256 + n = 128 + a = np.array(np.random.randn(m, n)*10, dtype=np.float64, order='F') + b = np.array(np.random.randn(m, n)*10, dtype=np.float64, order='F') + t1 = np.array(np.random.randn(m, n)*10, dtype=np.float64, order='F') + t2 = np.array(np.random.randn(m, n)*10, dtype=np.float64, order='F') + v = 0.1 + + r1 = 1 * (a > b) + r2 = 1 * (a > v) + + da = cm.CUDAMatrix(a) + db = cm.CUDAMatrix(b) + dt1 = cm.CUDAMatrix(t1) + dt2 = cm.CUDAMatrix(t2) + + da.greater_than(db, target = dt1) + da.greater_than(v, target = dt2) + da.greater_than(db) + + da.copy_to_host() + dt1.copy_to_host() + dt2.copy_to_host() + + assert np.max(np.abs(r1 - da.numpy_array)) < 10**-4, "Error in CUDAMatrix.greater_than exceeded threshold" + assert np.max(np.abs(r1 - dt1.numpy_array)) < 10**-4, "Error in CUDAMatrix.greater_than exceeded threshold" + assert np.max(np.abs(r2 - dt2.numpy_array)) < 10**-4, "Error in CUDAMatrix.greater_than exceeded threshold" + +def test_minimum(): + m = 256 + n = 128 + a = np.array(np.random.randn(m, n)*10, dtype=np.float64, order='F') + b = np.array(np.random.randn(m, n)*10, dtype=np.float64, order='F') + t1 = np.array(np.random.randn(m, n)*10, dtype=np.float64, order='F') + t2 = np.array(np.random.randn(m, n)*10, dtype=np.float64, order='F') + v = 0.1 + + r1 = np.minimum(a, b) + r2 = np.minimum(a, v) + + da = cm.CUDAMatrix(a) + db = cm.CUDAMatrix(b) + dt1 = cm.CUDAMatrix(t1) + dt2 = cm.CUDAMatrix(t2) + + da.minimum(db, target = dt1) + da.minimum(v, target = dt2) + da.minimum(db) + + da.copy_to_host() + dt1.copy_to_host() + dt2.copy_to_host() + + assert np.max(np.abs(r1 - da.numpy_array)) < 10**-4, "Error in CUDAMatrix.minimum exceeded threshold" + assert np.max(np.abs(r1 - dt1.numpy_array)) < 10**-4, "Error in CUDAMatrix.minimum exceeded threshold" + assert np.max(np.abs(r2 - dt2.numpy_array)) < 10**-4, "Error in CUDAMatrix.minimum exceeded threshold" + +def test_maximum(): + m = 256 + n = 128 + a = np.array(np.random.randn(m, n)*10, dtype=np.float64, order='F') + b = np.array(np.random.randn(m, n)*10, dtype=np.float64, order='F') + t1 = np.array(np.random.randn(m, n)*10, dtype=np.float64, order='F') + t2 = np.array(np.random.randn(m, n)*10, dtype=np.float64, order='F') + v = 0.1 + + r1 = np.maximum(a, b) + r2 = np.maximum(a, v) + + da = cm.CUDAMatrix(a) + db = cm.CUDAMatrix(b) + dt1 = cm.CUDAMatrix(t1) + dt2 = cm.CUDAMatrix(t2) + + da.maximum(db, target = dt1) + da.maximum(v, target = dt2) + da.maximum(db) + + da.copy_to_host() + dt1.copy_to_host() + dt2.copy_to_host() + + assert np.max(np.abs(r1 - da.numpy_array)) < 10**-4, "Error in CUDAMatrix.maximum exceeded threshold" + assert np.max(np.abs(r1 - dt1.numpy_array)) < 10**-4, "Error in CUDAMatrix.maximum exceeded threshold" + assert np.max(np.abs(r2 - dt2.numpy_array)) < 10**-4, "Error in CUDAMatrix.maximum exceeded threshold" + +def test_minmax(): + m = 256 + n = 128 + for op in 'min', 'max', 'argmin', 'argmax': + for sign in (1, -1): + a = np.array(np.random.randn(m, n)*10*sign, dtype=np.float64, order='F') + t0 = np.array(np.random.rand(1, n)*10, dtype=np.float64, order='F') + t1 = np.array(np.random.rand(m, 1)*10, dtype=np.float64, order='F') + + r0 = np.atleast_2d(getattr(a, op)(0)) + r1 = np.atleast_2d(getattr(a, op)(1)) + + da = cm.CUDAMatrix(a) + dr10 = cm.CUDAMatrix(t0) + dr11 = cm.CUDAMatrix(t1) + + getattr(da, op)(axis = 0, target = dr10) + getattr(da, op)(axis = 1, target = dr11) + dr20 = getattr(da, op)(axis = 0) + dr21 = getattr(da, op)(axis = 1) + + dr10.copy_to_host() + dr11.copy_to_host() + dr20.copy_to_host() + dr21.copy_to_host() + + assert np.max(np.abs(r0 - dr10.numpy_array)) < 10**-4, "Error in CUDAMatrix.%s exceeded threshold" % op + assert np.max(np.abs(r1 - dr11.numpy_array.T)) < 10**-4, "Error in CUDAMatrix.%s exceeded threshold" % op + assert np.max(np.abs(r0 - dr20.numpy_array)) < 10**-4, "Error in CUDAMatrix.%s exceeded threshold" % op + assert np.max(np.abs(r1 - dr21.numpy_array.T)) < 10**-4, "Error in CUDAMatrix.%s exceeded threshold" % op + +def test_sign(): + m = 256 + n = 128 + a = np.array(np.random.randn(m, n)*10, dtype=np.float64, order='F') + a[0,0] = 0. + a[0,1] = -0. + t = np.array(np.random.randn(m, n)*10, dtype=np.float64, order='F') + + c = np.sign(a) + + m1 = cm.CUDAMatrix(a) + m3 = cm.CUDAMatrix(t) + + m2 = m1.sign() + m1.sign(target = m3) + + m2.copy_to_host() + m3.copy_to_host() + + assert np.max(np.abs(c - m2.numpy_array)) < 10**-4, "Error in CUDAMatrix.sign exceeded threshold" + assert np.max(np.abs(c - m3.numpy_array)) < 10**-4, "Error in CUDAMatrix.sign exceeded threshold" + +def test_sigmoid(): + m = 256 + n = 128 + a = np.array(np.random.randn(m, n)*10, dtype=np.float64, order='F') + b = np.array(np.random.randn(m, n)*10, dtype=np.float64, order='F') + + c = 1. / (1. + np.exp(-a)) + + m1 = cm.CUDAMatrix(a) + m2 = cm.CUDAMatrix(b) + m1.apply_sigmoid(target = m2) + m1.apply_sigmoid() + + m1.copy_to_host() + m2.copy_to_host() + + assert np.max(np.abs(c - m1.numpy_array)) < 10**-4, "Error in CUDAMatrix.apply_sigmoid exceeded threshold" + assert np.max(np.abs(c - m2.numpy_array)) < 10**-4, "Error in CUDAMatrix.apply_sigmoid exceeded threshold" + +def test_tanh(): + m = 256 + n = 128 + a = np.array(np.random.randn(m, n)*10, dtype=np.float64, order='F') + b = np.array(np.random.randn(m, n)*10, dtype=np.float64, order='F') + + c = np.tanh(a) + + m1 = cm.CUDAMatrix(a) + m2 = cm.CUDAMatrix(b) + m1.apply_tanh(target = m2) + m1.apply_tanh() + + m1.copy_to_host() + m2.copy_to_host() + + assert np.max(np.abs(c - m1.numpy_array)) < 10**-4, "Error in CUDAMatrix.apply_tanh exceeded threshold" + assert np.max(np.abs(c - m2.numpy_array)) < 10**-4, "Error in CUDAMatrix.apply_tanh exceeded threshold" + +def test_soft_threshold(): + m = 256 + n = 128 + a = np.array(np.random.randn(m, n)*10, dtype=np.float64, order='F') + b = np.array(np.random.randn(m, n)*10, dtype=np.float64, order='F') + + alpha = 0.5 + c = np.sign(a) * np.maximum(0, np.abs(a) - alpha) + + m1 = cm.CUDAMatrix(a) + m2 = cm.CUDAMatrix(b) + m1.apply_soft_threshold(alpha, target = m2) + m1.apply_soft_threshold(alpha) + + m1.copy_to_host() + m2.copy_to_host() + + assert np.max(np.abs(c - m1.numpy_array)) < 10**-4, "Error in CUDAMatrix.apply_soft_threshold exceeded threshold" + assert np.max(np.abs(c - m2.numpy_array)) < 10**-4, "Error in CUDAMatrix.apply_soft_threshold exceeded threshold" + +def test_log(): + m = 256 + n = 128 + a = np.array(np.random.rand(m, n)*10+0.1, dtype=np.float64, order='F') + b = np.array(np.random.rand(m, n)*10+0.1, dtype=np.float64, order='F') + + c = np.log(a) + + m1 = cm.CUDAMatrix(a) + m2 = cm.CUDAMatrix(b) + cm.log(m1, target = m2) + cm.log(m1) + + m1.copy_to_host() + m2.copy_to_host() + + assert np.max(np.abs(c - m1.numpy_array)) < 10**-4, "Error in cudamat.log exceeded threshold" + assert np.max(np.abs(c - m2.numpy_array)) < 10**-4, "Error in cudamat.log exceeded threshold" + +def test_exp(): + m = 256 + n = 128 + a = np.array(np.random.randn(m, n), dtype=np.float64, order='F') + b = np.array(np.random.randn(m, n), dtype=np.float64, order='F') + + c = np.exp(a) + + m1 = cm.CUDAMatrix(a) + m2 = cm.CUDAMatrix(b) + cm.exp(m1, target = m2) + cm.exp(m1) + + m1.copy_to_host() + m2.copy_to_host() + + assert np.max(np.abs(c - m1.numpy_array)) < 10**-4, "Error in cudamat.exp exceeded threshold" + assert np.max(np.abs(c - m2.numpy_array)) < 10**-4, "Error in cudamat.exp exceeded threshold" + +def test_gamma(): + m = 256 + n = 128 + a = np.array(np.random.rand(m, n)*5, dtype=np.float64, order='F') + b = np.array(np.random.rand(m, n)*5, dtype=np.float64, order='F') + + from scipy.special import gamma + c = gamma(a) + + m1 = cm.CUDAMatrix(a) + m2 = cm.CUDAMatrix(b) + cm.gamma(m1, target = m2) + cm.gamma(m1) + + m1.copy_to_host() + m2.copy_to_host() + + assert np.max(np.abs(c - m1.numpy_array)) < 10**-2, "Error in cudamat.gamma exceeded threshold" + assert np.max(np.abs(c - m2.numpy_array)) < 10**-2, "Error in cudamat.gamma exceeded threshold" + +def test_lgamma(): + m = 256 + n = 128 + a = np.array(np.random.rand(m, n)*10, dtype=np.float64, order='F') + b = np.array(np.random.rand(m, n)*10, dtype=np.float64, order='F') + + from scipy.special import gammaln + c = gammaln(a) + + m1 = cm.CUDAMatrix(a) + m2 = cm.CUDAMatrix(b) + cm.lgamma(m1, target = m2) + cm.lgamma(m1) + + m1.copy_to_host() + m2.copy_to_host() + + assert np.max(np.abs(c - m1.numpy_array)) < 10**-2, "Error in cudamat.lgamma exceeded threshold " + str(np.max(np.abs(c - m1.numpy_array))) + assert np.max(np.abs(c - m2.numpy_array)) < 10**-2, "Error in cudamat.lgamma exceeded threshold" + +def test_sqrt(): + m = 256 + n = 128 + a = np.array(np.random.rand(m, n)*20, dtype=np.float64, order='F') + b = np.array(np.random.rand(m, n), dtype=np.float64, order='F') + + c = np.sqrt(a) + + m1 = cm.CUDAMatrix(a) + m2 = cm.CUDAMatrix(b) + cm.sqrt(m1, target = m2) + cm.sqrt(m1) + + m1.copy_to_host() + m2.copy_to_host() + + assert np.max(np.abs(c - m1.numpy_array)) < 10**-4, "Error in cudamat.sqrt exceeded threshold" + assert np.max(np.abs(c - m2.numpy_array)) < 10**-4, "Error in cudamat.sqrt exceeded threshold" + +def test_pow(): + m = 256 + n = 128 + a = np.array(np.random.randn(m, n)*20, dtype=np.float64, order='F') + b = np.array(np.random.rand(m, n), dtype=np.float64, order='F') + p = 2 + + c = a**p + + m1 = cm.CUDAMatrix(a) + m2 = cm.CUDAMatrix(b) + cm.pow(m1, p, target = m2) + cm.pow(m1, p) + + m1.copy_to_host() + m2.copy_to_host() + + assert np.max(np.abs(c - m1.numpy_array)) < 10**-3, "Error in cudamat.pow exceeded threshold" + assert np.max(np.abs(c - m2.numpy_array)) < 10**-3, "Error in cudamat.pow exceeded threshold" + +def test_pow_matrix(): + m = 256 + n = 128 + a = np.array(np.random.rand(m, n)*20, dtype=np.float64, order='F') + b = np.array(np.random.rand(m, n), dtype=np.float64, order='F') + p = np.array(np.random.randn(m, n), dtype=np.float64, order='F') + + + c = a**p + + m1 = cm.CUDAMatrix(a) + m2 = cm.CUDAMatrix(b) + mp = cm.CUDAMatrix(p) + cm.pow(m1, mp, target = m2) + cm.pow(m1, mp) + + m1.copy_to_host() + m2.copy_to_host() + + assert np.max(np.abs(c - m1.numpy_array)) < 10**-2, "Error in cudamat.pow exceeded threshold" + assert np.max(np.abs(c - m2.numpy_array)) < 10**-2, "Error in cudamat.pow exceeded threshold" + +def test_reciprocal(): + m = 256 + n = 128 + a = np.array(np.random.rand(m, n)*10+10**-3, dtype=np.float64, order='F') + b = np.array(np.random.rand(m, n)*10, dtype=np.float64, order='F') + + c = 1. / a + + m1 = cm.CUDAMatrix(a) + m2 = cm.CUDAMatrix(b) + m1.reciprocal(target = m2) + m1.reciprocal() + + m1.copy_to_host() + m2.copy_to_host() + + assert np.max(np.abs(c - m1.numpy_array)) < 10**-4, "Error in CUDAMatrix.reciprocal exceeded threshold" + assert np.max(np.abs(c - m2.numpy_array)) < 10**-4, "Error in CUDAMatrix.reciprocal exceeded threshold" + +def test_add_mult(): + m = 256 + n = 128 + alpha = np.pi + a = np.array(np.random.randn(m, n)*10, dtype=np.float64, order='F') + b = np.array(np.random.randn(m, n)*10, dtype=np.float64, order='F') + + c = a + np.pi * b + + m1 = cm.CUDAMatrix(a) + m2 = cm.CUDAMatrix(b) + m1.add_mult(m2, np.pi) + m1.copy_to_host() + + assert np.max(np.abs(c - m1.numpy_array)) < 10**-4, "Error in CUDAMatrix.add_mult exceeded threshold" + +def test_subtract_mult(): + m = 256 + n = 128 + alpha = np.pi + a = np.array(np.random.randn(m, n)*10, dtype=np.float64, order='F') + b = np.array(np.random.randn(m, n)*10, dtype=np.float64, order='F') + + c = a - np.pi * b + + m1 = cm.CUDAMatrix(a) + m2 = cm.CUDAMatrix(b) + m1.subtract_mult(m2, np.pi) + m1.copy_to_host() + + assert np.max(np.abs(c - m1.numpy_array)) < 10**-4, "Error in CUDAMatrix.subtract_mult exceeded threshold" + +def test_add(): + m = 256 + n = 128 + a = np.array(np.random.randn(m, n)*10, dtype=np.float64, order='F') + b = np.array(1.+np.random.rand(m, n)*10, dtype=np.float64, order='F') + t = np.array(np.empty((m, n)), dtype=np.float64, order='F') + + c = a + b + + m1 = cm.CUDAMatrix(a) + m2 = cm.CUDAMatrix(b) + m3 = cm.CUDAMatrix(t) + + m1.add(m2, target = m3) + m1.add(m2) + + m3.copy_to_host() + m1.copy_to_host() + + assert np.max(np.abs(c - m3.numpy_array)) < 10**-4, "Error in CUDAMatrix.add exceeded threshold" + assert np.max(np.abs(c - m1.numpy_array)) < 10**-4, "Error in CUDAMatrix.add exceeded threshold" + +def test_subtract(): + m = 256 + n = 128 + a = np.array(np.random.randn(m, n)*10, dtype=np.float64, order='F') + b = np.array(1.+np.random.rand(m, n)*10, dtype=np.float64, order='F') + t = np.array(np.empty((m, n)), dtype=np.float64, order='F') + + c = a - b + + m1 = cm.CUDAMatrix(a) + m2 = cm.CUDAMatrix(b) + m3 = cm.CUDAMatrix(t) + + m1.subtract(m2, target = m3) + m1.subtract(m2) + + m3.copy_to_host() + m1.copy_to_host() + + assert np.max(np.abs(c - m3.numpy_array)) < 10**-4, "Error in CUDAMatrix.subtract exceeded threshold" + assert np.max(np.abs(c - m1.numpy_array)) < 10**-4, "Error in CUDAMatrix.subtract exceeded threshold" + +def test_divide(): + m = 256 + n = 128 + a = np.array(np.random.randn(m, n)*10, dtype=np.float64, order='F') + b = np.array(1.+np.random.rand(m, n)*10, dtype=np.float64, order='F') + t = np.array(np.empty((m, n)), dtype=np.float64, order='F') + + c = a / b + + m1 = cm.CUDAMatrix(a) + m2 = cm.CUDAMatrix(b) + m3 = cm.CUDAMatrix(t) + + m1.divide(m2, target = m3) + m1.divide(m2) + + m3.copy_to_host() + m1.copy_to_host() + + assert np.max(np.abs(c - m3.numpy_array)) < 10**-4, "Error in CUDAMatrix.div exceeded threshold" + assert np.max(np.abs(c - m1.numpy_array)) < 10**-4, "Error in CUDAMatrix.div exceeded threshold" + +def test_mult(): + m = 256 + n = 128 + a = np.array(np.random.randn(m, n)*10, dtype=np.float64, order='F') + b = np.array(np.random.randn(m, n)*10, dtype=np.float64, order='F') + t = np.array(np.empty((m, n)), dtype=np.float64, order='F') + + c = a * b + + m1 = cm.CUDAMatrix(a) + m2 = cm.CUDAMatrix(b) + m3 = cm.CUDAMatrix(t) + + m1.mult(m2, target = m3) + m1.mult(m2) + + m3.copy_to_host() + m1.copy_to_host() + + assert np.max(np.abs(c - m3.numpy_array)) < 10**-4, "Error in CUDAMatrix.multiply exceeded threshold" + assert np.max(np.abs(c - m1.numpy_array)) < 10**-4, "Error in CUDAMatrix.multiply exceeded threshold" + +def test_scalar_mult(): + m = 256 + n = 128 + alpha = np.pi + a = np.array(np.random.randn(m, n)*10, dtype=np.float64, order='F') + t = np.array(np.empty((m, n)), dtype=np.float64, order='F') + + c = a * alpha + + m1 = cm.CUDAMatrix(a) + m2 = cm.CUDAMatrix(t) + + m1.mult(alpha, target = m2) + m1.mult(alpha) + + m1.copy_to_host() + m2.copy_to_host() + + assert np.max(np.abs(c - m1.numpy_array)) < 10**-4, "Error in CUDAMatrix.mult exceeded threshold" + assert np.max(np.abs(c - m2.numpy_array)) < 10**-4, "Error in CUDAMatrix.mult exceeded threshold" + +def test_scalar_div(): + m = 256 + n = 128 + alpha = np.pi + a = np.array(np.random.randn(m, n)*10, dtype=np.float64, order='F') + t = np.array(np.empty((m, n)), dtype=np.float64, order='F') + + c = a / alpha + + m1 = cm.CUDAMatrix(a) + m2 = cm.CUDAMatrix(t) + + m1.divide(alpha, target = m2) + m1.divide(alpha) + + m1.copy_to_host() + m2.copy_to_host() + + assert np.max(np.abs(c - m1.numpy_array)) < 10**-4, "Error in CUDAMatrix.divide exceeded threshold" + assert np.max(np.abs(c - m2.numpy_array)) < 10**-4, "Error in CUDAMatrix.divide exceeded threshold" + +def test_add_scalar(): + m = 256 + n = 128 + alpha = np.pi + a = np.array(np.random.randn(m, n)*10, dtype=np.float64, order='F') + t = np.array(np.empty((m, n)), dtype=np.float64, order='F') + + c = a + alpha + + m1 = cm.CUDAMatrix(a) + m2 = cm.CUDAMatrix(t) + + m1.add(alpha, target = m2) + m1.add(alpha) + + m1.copy_to_host() + m2.copy_to_host() + + assert np.max(np.abs(c - m1.numpy_array)) < 10**-4, "Error in CUDAMatrix.add_scalar exceeded threshold" + assert np.max(np.abs(c - m2.numpy_array)) < 10**-4, "Error in CUDAMatrix.add_scalar exceeded threshold" + +def test_dot(): + m = 128 + k = 256 + n = 64 + a = np.array(np.random.randn(m, k)*10, dtype=np.float64, order='F') + b = np.array(np.random.randn(k, n)*10, dtype=np.float64, order='F') + c = np.array(np.random.randn(m, n)*10, dtype=np.float64, order='F') + + alpha = 2. + beta = 0.3 + r = beta * c + alpha * np.dot(a, b) + + m1 = cm.CUDAMatrix(a) + m2 = cm.CUDAMatrix(b) + m3 = cm.CUDAMatrix(c) + m3 = cm.dot(m1, m2, target = m3, alpha = alpha, beta = beta) + m3.copy_to_host() + + assert np.max(np.abs(r - m3.numpy_array)) < 10**-2, "Error in CUDAMatrix.dot exceeded threshold" + +def test_dot_trans(): + m = 128 + k = 256 + n = 64 + a = np.array(np.random.randn(k, m)*10, dtype=np.float64, order='F') + b = np.array(np.random.randn(k, n)*10, dtype=np.float64, order='F') + + c = np.dot(a.T, b) + + m1 = cm.CUDAMatrix(a) + m2 = cm.CUDAMatrix(b) + m1.set_trans(True); + m3 = cm.dot(m1, m2) + m3.copy_to_host() + + assert np.max(np.abs(c - m3.numpy_array)) < 10**-2, "Error in CUDAMatrix.dot exceeded threshold" + +def test_dot_vect(): + m = 128 + k = 256 + n = 1 + a = np.array(np.random.randn(m, k)*10, dtype=np.float64, order='F') + b = np.array(np.random.randn(k, n)*10, dtype=np.float64, order='F') + A = cm.CUDAMatrix(a) + B = cm.CUDAMatrix(b) + + c = np.dot(a, b) + C = cm.dot(A, B) + assert np.max(np.abs(c - C.asarray())) < 10**-2, "Error in CUDAMatrix.dot exceeded threshold" + + c = np.dot(a.T, b[:m]) + C = cm.dot(A.T, B.slice(0, m)) + assert np.max(np.abs(c - C.asarray())) < 10**-2, "Error in CUDAMatrix.dot exceeded threshold" + + c = np.dot(b.T, a.T) + C = cm.dot(B.T, A.T) + assert np.max(np.abs(c - C.asarray())) < 10**-2, "Error in CUDAMatrix.dot exceeded threshold" + + c = np.dot(b[:m].T, a) + C = cm.dot(B.slice(0, m).reshape((1, m)), A) + assert np.max(np.abs(c - C.asarray())) < 10**-2, "Error in CUDAMatrix.dot exceeded threshold" + +def test_add_dot(): + m = 128 + k = 256 + n = 64 + a = np.array(np.random.randn(m, k)*10, dtype=np.float64, order='F') + b = np.array(np.random.randn(k, n)*10, dtype=np.float64, order='F') + c = np.array(np.random.randn(m, n)*10, dtype=np.float64, order='F') + + mult = 2.1 + beta = 0.8 + res = beta * c + mult * np.dot(a, b) + + m1 = cm.CUDAMatrix(a) + m2 = cm.CUDAMatrix(b) + m3 = cm.CUDAMatrix(c) + m3.add_dot(m1, m2, mult = mult, beta = beta) + + m3.copy_to_host() + + assert np.max(np.abs(res - m3.numpy_array)) < 10**-2, "Error in CUDAMatrix.add_dot exceeded threshold" + +def test_vdot(): + m = 64 + n = 64 + a = np.array(np.random.randn(m, n), dtype=np.float64, order='F') + b = np.array(np.random.randn(m, n), dtype=np.float64, order='F') + + true_res = np.vdot(a, b) + + m1 = cm.CUDAMatrix(a) + m2 = cm.CUDAMatrix(b) + + res = cm.vdot(m1, m2) + + assert np.abs(res - true_res) < 10**-2, "Error in CUDAMatrix.vdot exceeded threshold" + +def test_subtract_dot(): + m = 128 + k = 256 + n = 64 + a = np.array(np.random.randn(m, k)*10, dtype=np.float64, order='F') + b = np.array(np.random.randn(k, n)*10, dtype=np.float64, order='F') + c = np.array(np.random.randn(m, n)*10, dtype=np.float64, order='F') + + res = c - np.dot(a, b) + + m1 = cm.CUDAMatrix(a) + m2 = cm.CUDAMatrix(b) + m3 = cm.CUDAMatrix(c) + m3.subtract_dot(m1, m2) + + m3.copy_to_host() + + assert np.max(np.abs(res - m3.numpy_array)) < 10**-2, "Error in CUDAMatrix.subtract_dot exceeded threshold" + +def test_random(): + cm.CUDAMatrix.init_random(1) + m1 = cm.CUDAMatrix(np.array(np.empty((128,256)), dtype=np.float64, order='F')) + m2 = cm.CUDAMatrix(np.array(np.empty((128,256)), dtype=np.float64, order='F')) + + m1.fill_with_rand() + m1.copy_to_host() + m2.fill_with_randn() + m2.copy_to_host() + + assert np.abs(np.mean(m1.numpy_array) - 0.5) < 10**-2, "Error in CUDAMatrix.fill_with_rand threshold" + assert np.abs(np.mean(m2.numpy_array)) < 10**-2, "Error in CUDAMatrix.fill_with_randn threshold" + +def test_euclid_norm(): + m = 256 + n = 128 + a = np.array(np.random.rand(m, n)*10, dtype=np.float64, order='F') + + m = cm.CUDAMatrix(a) + + n1 = np.sqrt(np.sum(a**2)) + n2 = m.euclid_norm() + + assert np.abs(n1-n2) < 10**-2, "Error in CUDAMatrix.euclid_norm exceeded threshold" + +def test_manhattan_norm(): + m = 256 + n = 128 + a = np.array(np.random.rand(m, n)*10, dtype=np.float64, order='F') + + m = cm.CUDAMatrix(a) + + n1 = np.sum(np.abs(a), dtype=np.double) + n2 = m.manhattan_norm() + + assert np.abs(n1-n2) < 2e-2, "Error in CUDAMatrix.manhattan_norm exceeded threshold (%f != %f)" % (n1, n2) + +def test_allfinite(): + a = cm.empty((10, 20)).assign(1).divide(0) # NaN + b = cm.empty((10, 20)).assign(1e20).mult(1e20) # Inf + c = cm.empty((10, 20)).assign(1) # 1.0 + + assert (not a.allfinite()) and (not b.allfinite()) and c.allfinite(), "CUDAMatrix.allfinite does not work" + +def test_select_columns(): + m = 256 + n = 128 + k = 8 + + s = np.array(np.random.randn(m, n), dtype=np.float64, order='F') + i_l = [0, 1, 2, 3, 5, 10, 12, 20] + i = np.array(i_l).T[np.newaxis, :] + t = np.empty((m, k)) + + s_d = cm.CUDAMatrix(s) + i_d = cm.CUDAMatrix(i) + t_d = cm.CUDAMatrix(t) + + s_d.select_columns(i_d, t_d) + res = s[:,i_l] + + assert np.max(np.abs(res - t_d.asarray())) < 10**-4, "Error in CUDAMatrix.select_columns exceeded threshold" + + +def test_where(): + m = 256 + n = 128 + a = np.array(np.random.randn(m, n)*10, dtype=np.float64, order='F') + z = np.zeros_like(a) + res = np.where(a > 0, a, z); + + a_d = cm.CUDAMatrix(a) + z_d = cm.CUDAMatrix(z) + res_d = cm.empty(a_d.shape) + a_d.greater_than(0, res_d) + cm.where(res_d, a_d, z_d) + assert np.abs(res-res_d.asarray()).max() < 1e-2, "Error in cudamat.where" + + +def test_correlate(): + m = 64 + n = 32 + km = 17 + kn = 11 + + a = np.array(np.random.randn(m, n)*10, dtype=np.float64, order='F') + k = np.array(np.random.randn(km, kn)*10, dtype=np.float64, order='F') + + res = np.zeros_like(a) + for i in range(len(a)): + for j in range(len(a[0])): + for h in range(-(km/2), km/2 + 1): + for w in range(-(kn/2), kn/2 + 1): + if i+h >= 0 and i+h < m and j+w >= 0 and j+w < n: + res[i][j] += a[i + h][j + w] * k[km/2 + h][kn/2 + w] + + a_d = cm.CUDAMatrix(a) + k_d = cm.CUDAMatrix(k) + + res_d = cm.correlate(a_d, k_d) + assert np.abs(res-res_d.asarray()).max() < 1e-2, "Error in cudamat.correlate" + + +if __name__ == '__main__': + nose.runmodule() diff --git a/ot/gpu/cudamat/test/test_learn.py b/ot/gpu/cudamat/test/test_learn.py new file mode 100644 index 0000000..954757a --- /dev/null +++ b/ot/gpu/cudamat/test/test_learn.py @@ -0,0 +1,27 @@ +import pdb +import numpy as np +import nose +import cudamat as cm + +def setup(): + cm.cublas_init() + +def teardown(): + cm.cublas_shutdown() + +def test_mult_by_sigmoid_deriv(): + m = 256 + n = 128 + c_targets = np.array(np.random.randn(m, n)*10, dtype=np.float32, order='F') + c_acts = np.array(np.random.rand(m, n), dtype=np.float32, order='F') + + g_targets = cm.CUDAMatrix(c_targets) + g_acts = cm.CUDAMatrix(c_acts) + + c_targets = c_targets * c_acts * (1. - c_acts) + cm.learn.mult_by_sigmoid_deriv(g_targets, g_acts) + + assert np.max(np.abs(c_acts - g_acts.asarray())) < 10**-2, "Error in cudamat.learn.mult_by_sigmoid_deriv exceeded threshold" + +if __name__ == '__main__': + nose.runmodule() diff --git a/ot/gpu/da.py b/ot/gpu/da.py new file mode 100644 index 0000000..7d04b5b --- /dev/null +++ b/ot/gpu/da.py @@ -0,0 +1,81 @@ +# -*- coding: utf-8 -*- +""" +Domain adaptation with optimal transport and GPU +""" + +import numpy as np +from ..utils import unif +from ..da import OTDA +from .bregman import sinkhornGPU + + +def pairwiseEuclideanGPU(a, b, returnAsGPU=False, squared=False, cudamat=None): + # a is shape (n, f) and b shape (m, f). Return matrix c of shape (n, m). + # First compute in c_GPU the squared euclidean distance. And return its + # square root. At each cell [i,j] of c, we want to have + # sum{k in range(f)} ( (a[i,k] - b[j,k])^2 ). We know that + # (a-b)^2 = a^2 -2ab +b^2. Thus we want to have in each cell of c: + # sum{k in range(f)} ( a[i,k]^2 -2a[i,k]b[j,k] +b[j,k]^2). + + a_GPU = cudamat.CUDAMatrix(a) + b_GPU = cudamat.CUDAMatrix(b) + + # Multiply a by b transpose to obtain in each cell [i,j] of c the + # value sum{k in range(f)} ( a[i,k]b[j,k] ) + c_GPU = cudamat.dot(a_GPU, b_GPU.transpose()) + # multiply by -2 to have sum{k in range(f)} ( -2a[i,k]b[j,k] ) + c_GPU.mult(-2) + + # Compute the vectors of the sum of squared elements. + a_GPU = cudamat.pow(a_GPU, 2).sum(axis=1) + b_GPU = cudamat.pow(b_GPU, 2).sum(axis=1) + + # Add the vectors in each columns (respectivly rows) of c. + # sum{k in range(f)} ( a[i,k]^2 -2a[i,k]b[j,k] ) + c_GPU.add_col_vec(a_GPU) + # sum{k in range(f)} ( a[i,k]^2 -2a[i,k]b[j,k] +b[j,k]^2) + c_GPU.add_row_vec(b_GPU.transpose()) + + if not squared: + c_GPU = cudamat.sqrt(c_GPU) + + if returnAsGPU: + return c_GPU + else: + return c_GPU.asarray() + + +class OTDA_sinkhorn_GPU(OTDA): + def fit(self, xs, xt, reg=1, ws=None, wt=None, norm=None): + import cudamat + cudamat.init() + xs = np.asarray(xs, dtype=np.float64) + xt = np.asarray(xt, dtype=np.float64) + + self.xs = xs + self.xt = xt + + if wt is None: + wt = unif(xt.shape[0]) + if ws is None: + ws = unif(xs.shape[0]) + + self.ws = ws + self.wt = wt + + self.M_GPU = pairwiseEuclideanGPU(xs, xt, returnAsGPU=True, + squared=True, cudamat=cudamat) + + if norm == "median": + self.M_GPU.divide(float(np.median(self.M_GPU.asarray()))) + elif norm == "max": + self.M_GPU.divide(float(np.max(self.M_GPU.asarray()))) + elif norm == "log": + M = np.log(1 + self.M_GPU.asarray()) + self.M_GPU = cudamat.CUDAMatrix(M) + elif norm == "loglog": + M = np.log(1 + np.log(1 + self.M_GPU.asarray())) + self.M_GPU = cudamat.CUDAMatrix(M) + + self.G = sinkhornGPU(ws, wt, self.M_GPU, reg, cudamat=cudamat) + self.computed = True -- cgit v1.2.3 From 2eaee96a2a927476cbd8ca31754a89afd0825916 Mon Sep 17 00:00:00 2001 From: Rémi Flamary Date: Fri, 9 Jun 2017 13:18:04 +0200 Subject: add doc and correct encoding --- ot/__init__.py | 6 +++++- ot/bregman.py | 4 ++-- ot/utils.py | 8 ++++---- 3 files changed, 11 insertions(+), 7 deletions(-) (limited to 'ot/bregman.py') diff --git a/ot/__init__.py b/ot/__init__.py index 13abcda..b2af88b 100644 --- a/ot/__init__.py +++ b/ot/__init__.py @@ -1,4 +1,8 @@ -"""Python Optimal Transport toolbox""" +"""Python Optimal Transport toolbox + + + +""" # All submodules and packages diff --git a/ot/bregman.py b/ot/bregman.py index c46e5dc..00dca88 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -508,11 +508,11 @@ def geometricMean(alldistribT): return np.exp(np.mean(np.log(alldistribT),axis=1)) def projR(gamma,p): - #return np.dot(np.diag(p/np.maximum(np.sum(gamma,axis=1),1e-10)),gamma) + """return the KL projection on the row constrints """ return np.multiply(gamma.T,p/np.maximum(np.sum(gamma,axis=1),1e-10)).T def projC(gamma,q): - #return (np.dot(np.diag(q/np.maximum(np.sum(gamma,axis=0),1e-10)),gamma.T)).T + """return the KL projection on the column constrints """ return np.multiply(gamma,q/np.maximum(np.sum(gamma,axis=0),1e-10)) diff --git a/ot/utils.py b/ot/utils.py index e5cd6c0..fc6b0d2 100644 --- a/ot/utils.py +++ b/ot/utils.py @@ -63,10 +63,10 @@ def dist(x1,x2=None,metric='sqeuclidean'): matrix with n2 samples of size d (if None then x2=x1) metric : str, fun, optional name of the metric to be computed (full list in the doc of scipy), If a string, - the distance function can be ‘braycurtis’, ‘canberra’, ‘chebyshev’, ‘cityblock’, - ‘correlation’, ‘cosine’, ‘dice’, ‘euclidean’, ‘hamming’, ‘jaccard’, ‘kulsinski’, - ‘mahalanobis’, ‘matching’, ‘minkowski’, ‘rogerstanimoto’, ‘russellrao’, ‘seuclidean’, - ‘sokalmichener’, ‘sokalsneath’, ‘sqeuclidean’, ‘wminkowski’, ‘yule’. + the distance function can be 'braycurtis', 'canberra', 'chebyshev', 'cityblock', + 'correlation', 'cosine', 'dice', 'euclidean', 'hamming', 'jaccard', 'kulsinski', + 'mahalanobis', 'matching', 'minkowski', 'rogerstanimoto', 'russellrao', 'seuclidean', + 'sokalmichener', 'sokalsneath', 'sqeuclidean', 'wminkowski', 'yule'. Returns -- cgit v1.2.3 From a632c40b97c51534bfe65fee8b493780df118039 Mon Sep 17 00:00:00 2001 From: Rémi Flamary Date: Tue, 13 Jun 2017 14:39:59 +0200 Subject: make sinkhorn more general with method selection --- ot/bregman.py | 110 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 107 insertions(+), 3 deletions(-) (limited to 'ot/bregman.py') diff --git a/ot/bregman.py b/ot/bregman.py index 00dca88..6b3c68b 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -5,8 +5,112 @@ Bregman projections for regularized OT import numpy as np +def sinkhorn(a,b, M, reg,method='sinkhorn', numItermax = 1000, stopThr=1e-9, verbose=False, log=False,**kwargs): + u""" + Solve the entropic regularization optimal transport problem + + The function solves the following optimization problem: + + .. math:: + \gamma = arg\min_\gamma <\gamma,M>_F + reg\cdot\Omega(\gamma) + + s.t. \gamma 1 = a + + \gamma^T 1= b + + \gamma\geq 0 + where : + + - M is the (ns,nt) metric cost matrix + - :math:`\Omega` is the entropic regularization term :math:`\Omega(\gamma)=\sum_{i,j} \gamma_{i,j}\log(\gamma_{i,j})` + - a and b are source and target weights (sum to 1) + + The algorithm used for solving the problem is the Sinkhorn-Knopp matrix scaling algorithm as proposed in [2]_ + + + Parameters + ---------- + a : np.ndarray (ns,) + samples weights in the source domain + b : np.ndarray (nt,) + samples in the target domain + M : np.ndarray (ns,nt) + loss matrix + reg : float + Regularization term >0 + method : str + method used for the solver either 'sinkhorn', 'sinkhorn_stabilized' or + 'sinkhorn_epsilon_scaling', see those function for specific parameters + numItermax : int, optional + Max number of iterations + stopThr : float, optional + Stop threshol on error (>0) + verbose : bool, optional + Print information along iterations + log : bool, optional + record log if True + + + Returns + ------- + gamma : (ns x nt) ndarray + Optimal transportation matrix for the given parameters + log : dict + log dictionary return only if log==True in parameters + + Examples + -------- + + >>> import ot + >>> a=[.5,.5] + >>> b=[.5,.5] + >>> M=[[0.,1.],[1.,0.]] + >>> ot.sinkhorn(a,b,M,1) + array([[ 0.36552929, 0.13447071], + [ 0.13447071, 0.36552929]]) + + + References + ---------- + + .. [2] M. Cuturi, Sinkhorn Distances : Lightspeed Computation of Optimal Transport, Advances in Neural Information Processing Systems (NIPS) 26, 2013 + + .. [9] Schmitzer, B. (2016). Stabilized Sparse Scaling Algorithms for Entropy Regularized Transport Problems. arXiv preprint arXiv:1610.06519. + + .. [10] Chizat, L., Peyré, G., Schmitzer, B., & Vialard, F. X. (2016). Scaling algorithms for unbalanced transport problems. arXiv preprint arXiv:1607.05816. + + + + See Also + -------- + ot.lp.emd : Unregularized OT + ot.optim.cg : General regularized OT + ot.bregman.sinkhorn_knopp : Classic Sinkhorn [2] + ot.bregman.sinkhorn_stabilized: Stabilized sinkhorn [9][10] + ot.bregman.sinkhorn_epsilon_scaling: Sinkhorn with epslilon scaling [9][10] + + """ + + if method.lower()=='sinkhorn': + sink= lambda: sinkhorn_knopp(a,b, M, reg,numItermax=numItermax, + stopThr=stopThr, verbose=verbose, log=log,**kwargs) + elif method.lower()=='sinkhorn_stabilized': + sink= lambda: sinkhorn_stabilized(a,b, M, reg,numItermax=numItermax, + stopThr=stopThr, verbose=verbose, log=log, **kwargs) + elif method.lower()=='sinkhorn_epsilon_scaling': + sink= lambda: sinkhorn_epsilon_scaling(a,b, M, reg,numItermax=numItermax, + stopThr=stopThr, verbose=verbose, log=log, **kwargs) + else: + print('Warning : unknown method using classic Sinkhorn Knopp') + sink= lambda: sinkhorn_knopp(a,b, M, reg, **kwargs) + + return sink() + + + + -def sinkhorn(a,b, M, reg, numItermax = 1000, stopThr=1e-9, verbose=False, log=False): +def sinkhorn_knopp(a,b, M, reg, numItermax = 1000, stopThr=1e-9, verbose=False, log=False,**kwargs): """ Solve the entropic regularization optimal transport problem @@ -147,7 +251,7 @@ def sinkhorn(a,b, M, reg, numItermax = 1000, stopThr=1e-9, verbose=False, log=Fa else: return u.reshape((-1,1))*K*v.reshape((1,-1)) -def sinkhorn_stabilized(a,b, M, reg, numItermax = 1000,tau=1e3, stopThr=1e-9,warmstart=None, verbose=False,print_period=20, log=False): +def sinkhorn_stabilized(a,b, M, reg, numItermax = 1000,tau=1e3, stopThr=1e-9,warmstart=None, verbose=False,print_period=20, log=False,**kwargs): """ Solve the entropic regularization OT problem with log stabilization @@ -331,7 +435,7 @@ def sinkhorn_stabilized(a,b, M, reg, numItermax = 1000,tau=1e3, stopThr=1e-9,war else: return get_Gamma(alpha,beta,u,v) -def sinkhorn_epsilon_scaling(a,b, M, reg, numItermax = 100, epsilon0=1e4, numInnerItermax = 100,tau=1e3, stopThr=1e-9,warmstart=None, verbose=False,print_period=10, log=False): +def sinkhorn_epsilon_scaling(a,b, M, reg, numItermax = 100, epsilon0=1e4, numInnerItermax = 100,tau=1e3, stopThr=1e-9,warmstart=None, verbose=False,print_period=10, log=False,**kwargs): """ Solve the entropic regularization optimal transport problem with log stabilization and epsilon scaling. -- cgit v1.2.3 From a7bed093f91922e18fa5902c4d1d63b9712d5794 Mon Sep 17 00:00:00 2001 From: Rémi Flamary Date: Tue, 13 Jun 2017 15:14:45 +0200 Subject: implement paralell sinkhorn --- examples/plot_OT_1D.py | 2 +- examples/plot_compute_emd.py | 31 +++++++++++++++++++------ ot/bregman.py | 54 ++++++++++++++++++++++++++++++++------------ 3 files changed, 64 insertions(+), 23 deletions(-) (limited to 'ot/bregman.py') diff --git a/examples/plot_OT_1D.py b/examples/plot_OT_1D.py index e5719eb..6661aa3 100644 --- a/examples/plot_OT_1D.py +++ b/examples/plot_OT_1D.py @@ -50,7 +50,7 @@ ot.plot.plot1D_mat(a,b,G0,'OT matrix G0') #%% Sinkhorn lambd=1e-3 -Gs=ot.sinkhorn(a,b,M,lambd) +Gs=ot.sinkhorn(a,b,M,lambd,verbose=True) pl.figure(4) ot.plot.plot1D_mat(a,b,Gs,'OT matrix Sinkhorn') diff --git a/examples/plot_compute_emd.py b/examples/plot_compute_emd.py index 87b39a6..226bc97 100644 --- a/examples/plot_compute_emd.py +++ b/examples/plot_compute_emd.py @@ -32,10 +32,11 @@ B=np.zeros((n,n_target)) for i,m in enumerate(lst_m): B[:,i]=gauss(n,m=m,s=5) -# loss matrix +# loss matrix and normalization M=ot.dist(x.reshape((n,1)),x.reshape((n,1)),'euclidean') +M/=M.max() M2=ot.dist(x.reshape((n,1)),x.reshape((n,1)),'sqeuclidean') - +M2/=M2.max() #%% plot the distributions pl.figure(1) @@ -46,12 +47,28 @@ pl.subplot(2,1,2) pl.plot(x,B,label='Target distributions') pl.title('Target distributions') -#%% plot distributions and loss matrix +#%% Compute and plot distributions and loss matrix + +d_emd=ot.emd2(a,B,M) # direct computation of EMD +d_emd2=ot.emd2(a,B,M2) # direct computation of EMD with loss M3 + -emd=ot.emd2(a,B,M) -emd2=ot.emd2(a,B,M2) pl.figure(2) -pl.plot(emd,label='Euclidean loss') -pl.plot(emd,label='Squared Euclidean loss') +pl.plot(d_emd,label='Euclidean EMD') +pl.plot(d_emd2,label='Squared Euclidean EMD') +pl.title('EMD distances') pl.legend() +#%% +reg=1e-2 +d_sinkhorn=ot.sinkhorn(a,B,M,reg) +d_sinkhorn2=ot.sinkhorn(a,B,M2,reg) + +pl.figure(2) +pl.clf() +pl.plot(d_emd,label='Euclidean EMD') +pl.plot(d_emd2,label='Squared Euclidean EMD') +pl.plot(d_sinkhorn,label='Euclidean Sinkhorn') +pl.plot(d_emd2,label='Squared Euclidean Sinkhorn') +pl.title('EMD distances') +pl.legend() \ No newline at end of file diff --git a/ot/bregman.py b/ot/bregman.py index 6b3c68b..3a9b15f 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -7,7 +7,7 @@ import numpy as np def sinkhorn(a,b, M, reg,method='sinkhorn', numItermax = 1000, stopThr=1e-9, verbose=False, log=False,**kwargs): u""" - Solve the entropic regularization optimal transport problem + Solve the entropic regularization optimal transport problem and return the OT matrix The function solves the following optimization problem: @@ -107,12 +107,9 @@ def sinkhorn(a,b, M, reg,method='sinkhorn', numItermax = 1000, stopThr=1e-9, ver return sink() - - - def sinkhorn_knopp(a,b, M, reg, numItermax = 1000, stopThr=1e-9, verbose=False, log=False,**kwargs): """ - Solve the entropic regularization optimal transport problem + Solve the entropic regularization optimal transport problem and return the OT matrix The function solves the following optimization problem: @@ -188,22 +185,35 @@ def sinkhorn_knopp(a,b, M, reg, numItermax = 1000, stopThr=1e-9, verbose=False, a=np.asarray(a,dtype=np.float64) b=np.asarray(b,dtype=np.float64) M=np.asarray(M,dtype=np.float64) + if len(a)==0: a=np.ones((M.shape[0],),dtype=np.float64)/M.shape[0] if len(b)==0: b=np.ones((M.shape[1],),dtype=np.float64)/M.shape[1] + # init data Nini = len(a) Nfin = len(b) + + if len(b.shape)>1: + nbb=b.shape[1] + else: + nbb=0 + if log: log={'err':[]} # we assume that no distances are null except those of the diagonal of distances - u = np.ones(Nini)/Nini - v = np.ones(Nfin)/Nfin + if nbb: + u = np.ones((Nini,nbb))/Nini + v = np.ones((Nfin,nbb))/Nfin + else: + u = np.ones(Nini)/Nini + v = np.ones(Nfin)/Nfin + #print(reg) @@ -231,8 +241,11 @@ def sinkhorn_knopp(a,b, M, reg, numItermax = 1000, stopThr=1e-9, verbose=False, break if cpt%10==0: # we can speed up the process by checking for the error only all the 10th iterations - transp = u.reshape(-1, 1) * (K * v) - err = np.linalg.norm((np.sum(transp,axis=0)-b))**2 + if nbb: + err = np.sum((u-uprev)**2)/np.sum((u)**2)+np.sum((v-vprev)**2)/np.sum((v)**2) + else: + transp = u.reshape(-1, 1) * (K * v) + err = np.linalg.norm((np.sum(transp,axis=0)-b))**2 if log: log['err'].append(err) @@ -244,12 +257,23 @@ def sinkhorn_knopp(a,b, M, reg, numItermax = 1000, stopThr=1e-9, verbose=False, if log: log['u']=u log['v']=v - - #print('err=',err,' cpt=',cpt) - if log: - return u.reshape((-1,1))*K*v.reshape((1,-1)),log - else: - return u.reshape((-1,1))*K*v.reshape((1,-1)) + + if nbb: #return only loss + res=np.zeros((nbb)) + for i in range(nbb): + res[i]=np.sum(u[:,i].reshape((-1,1))*K*v[:,i].reshape((1,-1))*M) + if log: + return res,log + else: + return res + + else: # return OT matrix + + if log: + return u.reshape((-1,1))*K*v.reshape((1,-1)),log + else: + return u.reshape((-1,1))*K*v.reshape((1,-1)) + def sinkhorn_stabilized(a,b, M, reg, numItermax = 1000,tau=1e3, stopThr=1e-9,warmstart=None, verbose=False,print_period=20, log=False,**kwargs): """ -- cgit v1.2.3 From 38e96f88eb520b9fa8333686565b082d2921e131 Mon Sep 17 00:00:00 2001 From: Rémi Flamary Date: Tue, 13 Jun 2017 15:50:11 +0200 Subject: implement paralell sinkhorn stabilized --- examples/plot_OT_2D_samples.py | 4 +-- examples/plot_compute_emd.py | 9 +++--- ot/bregman.py | 68 +++++++++++++++++++++++++++++++----------- 3 files changed, 58 insertions(+), 23 deletions(-) (limited to 'ot/bregman.py') diff --git a/examples/plot_OT_2D_samples.py b/examples/plot_OT_2D_samples.py index 3b95083..edfb781 100644 --- a/examples/plot_OT_2D_samples.py +++ b/examples/plot_OT_2D_samples.py @@ -13,7 +13,7 @@ import ot #%% parameters and data generation -n=2 # nb samples +n=50 # nb samples mu_s=np.array([0,0]) cov_s=np.array([[1,0],[0,1]]) @@ -62,7 +62,7 @@ pl.title('OT matrix with samples') #%% sinkhorn # reg term -lambd=5e-3 +lambd=5e-4 Gs=ot.sinkhorn(a,b,M,lambd) diff --git a/examples/plot_compute_emd.py b/examples/plot_compute_emd.py index 226bc97..08de6ee 100644 --- a/examples/plot_compute_emd.py +++ b/examples/plot_compute_emd.py @@ -16,7 +16,7 @@ from ot.datasets import get_1D_gauss as gauss #%% parameters n=100 # nb bins -n_target=10 # nb target distributions +n_target=50 # nb target distributions # bin positions @@ -61,14 +61,15 @@ pl.legend() #%% reg=1e-2 -d_sinkhorn=ot.sinkhorn(a,B,M,reg) +d_sinkhorn=ot.sinkhorn(a,B,M,reg,method='sinkhorn_stabilized') +d_sinkhorn0=ot.sinkhorn(a,B,M,reg) d_sinkhorn2=ot.sinkhorn(a,B,M2,reg) pl.figure(2) pl.clf() pl.plot(d_emd,label='Euclidean EMD') pl.plot(d_emd2,label='Squared Euclidean EMD') -pl.plot(d_sinkhorn,label='Euclidean Sinkhorn') -pl.plot(d_emd2,label='Squared Euclidean Sinkhorn') +pl.plot(d_sinkhorn,'+',label='Euclidean Sinkhorn') +pl.plot(d_sinkhorn2,'+',label='Squared Euclidean Sinkhorn') pl.title('EMD distances') pl.legend() \ No newline at end of file diff --git a/ot/bregman.py b/ot/bregman.py index 3a9b15f..e847f24 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -32,8 +32,9 @@ def sinkhorn(a,b, M, reg,method='sinkhorn', numItermax = 1000, stopThr=1e-9, ver ---------- a : np.ndarray (ns,) samples weights in the source domain - b : np.ndarray (nt,) - samples in the target domain + b : np.ndarray (nt,) or np.ndarray (nt,nbb) + samples in the target domain, compute sinkhorn with multiple targets + and fixed M if b is a matrix (return OT loss + dual variables in log) M : np.ndarray (ns,nt) loss matrix reg : float @@ -134,8 +135,9 @@ def sinkhorn_knopp(a,b, M, reg, numItermax = 1000, stopThr=1e-9, verbose=False, ---------- a : np.ndarray (ns,) samples weights in the source domain - b : np.ndarray (nt,) - samples in the target domain + b : np.ndarray (nt,) or np.ndarray (nt,nbb) + samples in the target domain, compute sinkhorn with multiple targets + and fixed M if b is a matrix (return OT loss + dual variables in log) M : np.ndarray (ns,nt) loss matrix reg : float @@ -369,11 +371,17 @@ def sinkhorn_stabilized(a,b, M, reg, numItermax = 1000,tau=1e3, stopThr=1e-9,war if len(b)==0: b=np.ones((M.shape[1],),dtype=np.float64)/M.shape[1] + # test if multiple target + if len(b.shape)>1: + nbb=b.shape[1] + a=a[:,np.newaxis] + else: + nbb=0 + # init data na = len(a) nb = len(b) - cpt = 0 if log: log={'err':[]} @@ -383,10 +391,11 @@ def sinkhorn_stabilized(a,b, M, reg, numItermax = 1000,tau=1e3, stopThr=1e-9,war alpha,beta=np.zeros(na),np.zeros(nb) else: alpha,beta=warmstart - u,v = np.ones(na)/na,np.ones(nb)/nb - - #print(reg) - + + if nbb: + u,v = np.ones((na,nbb))/na,np.ones((nb,nbb))/nb + else: + u,v = np.ones(na)/na,np.ones(nb)/nb def get_K(alpha,beta): """log space computation""" @@ -405,22 +414,32 @@ def sinkhorn_stabilized(a,b, M, reg, numItermax = 1000,tau=1e3, stopThr=1e-9,war err=1 while loop: + # remove numerical problems and store them in K if np.abs(u).max()>tau or np.abs(v).max()>tau: - alpha,beta=alpha+reg*np.log(u),beta+reg*np.log(v) - u,v = np.ones(na)/na,np.ones(nb)/nb + if nbb: + alpha,beta=alpha+reg*np.max(np.log(u),1),beta+reg*np.max(np.log(v)) + else: + alpha,beta=alpha+reg*np.log(u),beta+reg*np.log(v) + if nbb: + u,v = np.ones((na,nbb))/na,np.ones((nb,nbb))/nb + else: + u,v = np.ones(na)/na,np.ones(nb)/nb K=get_K(alpha,beta) uprev = u vprev = v + + # sinkhorn update v = b/np.dot(K.T,u) u = a/np.dot(K,v) - - if cpt%print_period==0: # we can speed up the process by checking for the error only all the 10th iterations - transp = get_Gamma(alpha,beta,u,v) - err = np.linalg.norm((np.sum(transp,axis=0)-b))**2 + if nbb: + err = np.sum((u-uprev)**2)/np.sum((u)**2)+np.sum((v-vprev)**2)/np.sum((v)**2) + else: + transp = get_Gamma(alpha,beta,u,v) + err = np.linalg.norm((np.sum(transp,axis=0)-b))**2 if log: log['err'].append(err) @@ -448,6 +467,8 @@ def sinkhorn_stabilized(a,b, M, reg, numItermax = 1000,tau=1e3, stopThr=1e-9,war break cpt = cpt +1 + + #print('err=',err,' cpt=',cpt) if log: log['logu']=alpha/reg+np.log(u) @@ -455,9 +476,22 @@ def sinkhorn_stabilized(a,b, M, reg, numItermax = 1000,tau=1e3, stopThr=1e-9,war log['alpha']=alpha+reg*np.log(u) log['beta']=beta+reg*np.log(v) log['warmstart']=(log['alpha'],log['beta']) - return get_Gamma(alpha,beta,u,v),log + if nbb: + res=np.zeros((nbb)) + for i in range(nbb): + res[i]=np.sum(get_Gamma(alpha,beta,u[:,i],v[:,i])*M) + return res,log + + else: + return get_Gamma(alpha,beta,u,v),log else: - return get_Gamma(alpha,beta,u,v) + if nbb: + res=np.zeros((nbb)) + for i in range(nbb): + res[i]=np.sum(get_Gamma(alpha,beta,u[:,i],v[:,i])*M) + return res + else: + return get_Gamma(alpha,beta,u,v) def sinkhorn_epsilon_scaling(a,b, M, reg, numItermax = 100, epsilon0=1e4, numInnerItermax = 100,tau=1e3, stopThr=1e-9,warmstart=None, verbose=False,print_period=10, log=False,**kwargs): """ -- cgit v1.2.3 From 77bcf836272faf1a40fdea97131b85390e935be3 Mon Sep 17 00:00:00 2001 From: Rémi Flamary Date: Tue, 20 Jun 2017 14:51:12 +0200 Subject: add clean zeros function for sparse distributions --- ot/bregman.py | 2 ++ ot/utils.py | 7 +++++++ 2 files changed, 9 insertions(+) (limited to 'ot/bregman.py') diff --git a/ot/bregman.py b/ot/bregman.py index e847f24..a13345d 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -108,6 +108,8 @@ def sinkhorn(a,b, M, reg,method='sinkhorn', numItermax = 1000, stopThr=1e-9, ver return sink() + + def sinkhorn_knopp(a,b, M, reg, numItermax = 1000, stopThr=1e-9, verbose=False, log=False,**kwargs): """ Solve the entropic regularization optimal transport problem and return the OT matrix diff --git a/ot/utils.py b/ot/utils.py index fc6b0d2..7ad7637 100644 --- a/ot/utils.py +++ b/ot/utils.py @@ -50,6 +50,13 @@ def unif(n): """ return np.ones((n,))/n +def clean_zeros(a,b,M): + """ Remove all components with zeros weights in a and b + """ + M2=M[a>0,:][:,b>0].copy() # copy force c style matrix (froemd) + a2=a[a>0] + b2=b[b>0] + return a2,b2,M2 def dist(x1,x2=None,metric='sqeuclidean'): """Compute distance between samples in x1 and x2 using function scipy.spatial.distance.cdist -- cgit v1.2.3 From 4fba2c9c479e8f410a23ef24458effc29fc3f7f0 Mon Sep 17 00:00:00 2001 From: Rémi Flamary Date: Mon, 3 Jul 2017 14:51:47 +0200 Subject: debug bregman stabilized --- ot/bregman.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) (limited to 'ot/bregman.py') diff --git a/ot/bregman.py b/ot/bregman.py index a13345d..68be01c 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -416,6 +416,16 @@ def sinkhorn_stabilized(a,b, M, reg, numItermax = 1000,tau=1e3, stopThr=1e-9,war err=1 while loop: + + + uprev = u + vprev = v + + # sinkhorn update + v = b/(np.dot(K.T,u)+1e-16) + u = a/(np.dot(K,v)+1e-16) + + # remove numerical problems and store them in K if np.abs(u).max()>tau or np.abs(v).max()>tau: if nbb: @@ -428,12 +438,6 @@ def sinkhorn_stabilized(a,b, M, reg, numItermax = 1000,tau=1e3, stopThr=1e-9,war u,v = np.ones(na)/na,np.ones(nb)/nb K=get_K(alpha,beta) - uprev = u - vprev = v - - # sinkhorn update - v = b/np.dot(K.T,u) - u = a/np.dot(K,v) if cpt%print_period==0: # we can speed up the process by checking for the error only all the 10th iterations @@ -458,9 +462,7 @@ def sinkhorn_stabilized(a,b, M, reg, numItermax = 1000,tau=1e3, stopThr=1e-9,war loop=False - if (np.any(np.dot(K.T,u)==0) or - np.any(np.isnan(u)) or np.any(np.isnan(v)) or - np.any(np.isinf(u)) or np.any(np.isinf(v))): + if np.any(np.isnan(u)) or np.any(np.isnan(v)): # we have reached the machine precision # come back to previous solution and quit loop print('Warning: numerical errors at iteration', cpt) -- cgit v1.2.3 From 47477c5782f87570de590ea423a082b71dd63241 Mon Sep 17 00:00:00 2001 From: Rémi Flamary Date: Fri, 7 Jul 2017 08:51:45 +0200 Subject: add sinkhorbn2 +v3 --- README.md | 5 +- examples/plot_compute_emd.py | 4 +- ot/__init__.py | 8 +-- ot/bregman.py | 144 +++++++++++++++++++++++++++++++++++++------ 4 files changed, 133 insertions(+), 28 deletions(-) (limited to 'ot/bregman.py') diff --git a/README.md b/README.md index 4aa4cc5..d53387b 100644 --- a/README.md +++ b/README.md @@ -83,14 +83,15 @@ import ot # a,b are 1D histograms (sum to 1 and positive) # M is the ground cost matrix Wd=ot.emd2(a,b,M) # exact linear program +Wd_reg=ot.sinkhorn2(a,b,M,reg) # entropic regularized OT # if b is a matrix compute all distances to a and return a vector ``` * Compute OT matrix ```python # a,b are 1D histograms (sum to 1 and positive) # M is the ground cost matrix -Totp=ot.emd(a,b,M) # exact linear program -Totp_reg=ot.sinkhorn(a,b,M,reg) # entropic regularized OT +T=ot.emd(a,b,M) # exact linear program +T_reg=ot.sinkhorn(a,b,M,reg) # entropic regularized OT ``` * Compute Wasserstein barycenter ```python diff --git a/examples/plot_compute_emd.py b/examples/plot_compute_emd.py index c7063e8..f2cdc35 100644 --- a/examples/plot_compute_emd.py +++ b/examples/plot_compute_emd.py @@ -61,8 +61,8 @@ pl.legend() #%% reg=1e-2 -d_sinkhorn=ot.sinkhorn(a,B,M,reg) -d_sinkhorn2=ot.sinkhorn(a,B,M2,reg) +d_sinkhorn=ot.sinkhorn2(a,B,M,reg) +d_sinkhorn2=ot.sinkhorn2(a,B,M2,reg) pl.figure(2) pl.clf() diff --git a/ot/__init__.py b/ot/__init__.py index b2af88b..4220148 100644 --- a/ot/__init__.py +++ b/ot/__init__.py @@ -16,14 +16,14 @@ from . import da # OT functions from .lp import emd, emd2 -from .bregman import sinkhorn, barycenter +from .bregman import sinkhorn, sinkhorn2, barycenter from .da import sinkhorn_lpl1_mm # utils functions from .utils import dist, unif, tic, toc, toq -__version__ = "0.2" +__version__ = "0.3" -__all__ = ["emd", "emd2", "sinkhorn", "utils", 'datasets', 'bregman', 'lp', - 'plot', 'tic', 'toc', 'toq', +__all__ = ["emd", "emd2", "sinkhorn","sinkhorn2", "utils", 'datasets', + 'bregman', 'lp', 'plot', 'tic', 'toc', 'toq', 'dist', 'unif', 'barycenter', 'sinkhorn_lpl1_mm', 'da', 'optim'] diff --git a/ot/bregman.py b/ot/bregman.py index 68be01c..0d68602 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -41,7 +41,7 @@ def sinkhorn(a,b, M, reg,method='sinkhorn', numItermax = 1000, stopThr=1e-9, ver Regularization term >0 method : str method used for the solver either 'sinkhorn', 'sinkhorn_stabilized' or - 'sinkhorn_epsilon_scaling', see those function for specific parameters + 'sinkhorn_epsilon_scaling', see those function for specific parameters numItermax : int, optional Max number of iterations stopThr : float, optional @@ -91,7 +91,7 @@ def sinkhorn(a,b, M, reg,method='sinkhorn', numItermax = 1000, stopThr=1e-9, ver ot.bregman.sinkhorn_epsilon_scaling: Sinkhorn with epslilon scaling [9][10] """ - + if method.lower()=='sinkhorn': sink= lambda: sinkhorn_knopp(a,b, M, reg,numItermax=numItermax, stopThr=stopThr, verbose=verbose, log=log,**kwargs) @@ -100,15 +100,119 @@ def sinkhorn(a,b, M, reg,method='sinkhorn', numItermax = 1000, stopThr=1e-9, ver stopThr=stopThr, verbose=verbose, log=log, **kwargs) elif method.lower()=='sinkhorn_epsilon_scaling': sink= lambda: sinkhorn_epsilon_scaling(a,b, M, reg,numItermax=numItermax, - stopThr=stopThr, verbose=verbose, log=log, **kwargs) + stopThr=stopThr, verbose=verbose, log=log, **kwargs) else: print('Warning : unknown method using classic Sinkhorn Knopp') sink= lambda: sinkhorn_knopp(a,b, M, reg, **kwargs) - + return sink() + +def sinkhorn2(a,b, M, reg,method='sinkhorn', numItermax = 1000, stopThr=1e-9, verbose=False, log=False,**kwargs): + u""" + Solve the entropic regularization optimal transport problem and return the loss + + The function solves the following optimization problem: + + .. math:: + W = \min_\gamma <\gamma,M>_F + reg\cdot\Omega(\gamma) + + s.t. \gamma 1 = a + + \gamma^T 1= b + + \gamma\geq 0 + where : + + - M is the (ns,nt) metric cost matrix + - :math:`\Omega` is the entropic regularization term :math:`\Omega(\gamma)=\sum_{i,j} \gamma_{i,j}\log(\gamma_{i,j})` + - a and b are source and target weights (sum to 1) + + The algorithm used for solving the problem is the Sinkhorn-Knopp matrix scaling algorithm as proposed in [2]_ + + + Parameters + ---------- + a : np.ndarray (ns,) + samples weights in the source domain + b : np.ndarray (nt,) or np.ndarray (nt,nbb) + samples in the target domain, compute sinkhorn with multiple targets + and fixed M if b is a matrix (return OT loss + dual variables in log) + M : np.ndarray (ns,nt) + loss matrix + reg : float + Regularization term >0 + method : str + method used for the solver either 'sinkhorn', 'sinkhorn_stabilized' or + 'sinkhorn_epsilon_scaling', see those function for specific parameters + numItermax : int, optional + Max number of iterations + stopThr : float, optional + Stop threshol on error (>0) + verbose : bool, optional + Print information along iterations + log : bool, optional + record log if True + + + Returns + ------- + W : (nt) ndarray or float + Optimal transportation matrix for the given parameters + log : dict + log dictionary return only if log==True in parameters + + Examples + -------- + + >>> import ot + >>> a=[.5,.5] + >>> b=[.5,.5] + >>> M=[[0.,1.],[1.,0.]] + >>> ot.sinkhorn2(a,b,M,1) + array([ 0.26894142]) + References + ---------- + + .. [2] M. Cuturi, Sinkhorn Distances : Lightspeed Computation of Optimal Transport, Advances in Neural Information Processing Systems (NIPS) 26, 2013 + + .. [9] Schmitzer, B. (2016). Stabilized Sparse Scaling Algorithms for Entropy Regularized Transport Problems. arXiv preprint arXiv:1610.06519. + + .. [10] Chizat, L., Peyré, G., Schmitzer, B., & Vialard, F. X. (2016). Scaling algorithms for unbalanced transport problems. arXiv preprint arXiv:1607.05816. + + + + See Also + -------- + ot.lp.emd : Unregularized OT + ot.optim.cg : General regularized OT + ot.bregman.sinkhorn_knopp : Classic Sinkhorn [2] + ot.bregman.sinkhorn_stabilized: Stabilized sinkhorn [9][10] + ot.bregman.sinkhorn_epsilon_scaling: Sinkhorn with epslilon scaling [9][10] + + """ + + if method.lower()=='sinkhorn': + sink= lambda: sinkhorn_knopp(a,b, M, reg,numItermax=numItermax, + stopThr=stopThr, verbose=verbose, log=log,**kwargs) + elif method.lower()=='sinkhorn_stabilized': + sink= lambda: sinkhorn_stabilized(a,b, M, reg,numItermax=numItermax, + stopThr=stopThr, verbose=verbose, log=log, **kwargs) + elif method.lower()=='sinkhorn_epsilon_scaling': + sink= lambda: sinkhorn_epsilon_scaling(a,b, M, reg,numItermax=numItermax, + stopThr=stopThr, verbose=verbose, log=log, **kwargs) + else: + print('Warning : unknown method using classic Sinkhorn Knopp') + sink= lambda: sinkhorn_knopp(a,b, M, reg, **kwargs) + + b=np.asarray(b,dtype=np.float64) + if len(b.shape)<2: + b=b.reshape((-1,1)) + + return sink() + def sinkhorn_knopp(a,b, M, reg, numItermax = 1000, stopThr=1e-9, verbose=False, log=False,**kwargs): """ @@ -189,23 +293,23 @@ def sinkhorn_knopp(a,b, M, reg, numItermax = 1000, stopThr=1e-9, verbose=False, a=np.asarray(a,dtype=np.float64) b=np.asarray(b,dtype=np.float64) M=np.asarray(M,dtype=np.float64) - + if len(a)==0: a=np.ones((M.shape[0],),dtype=np.float64)/M.shape[0] if len(b)==0: b=np.ones((M.shape[1],),dtype=np.float64)/M.shape[1] - + # init data Nini = len(a) Nfin = len(b) - + if len(b.shape)>1: nbb=b.shape[1] else: nbb=0 - + if log: log={'err':[]} @@ -217,7 +321,7 @@ def sinkhorn_knopp(a,b, M, reg, numItermax = 1000, stopThr=1e-9, verbose=False, else: u = np.ones(Nini)/Nini v = np.ones(Nfin)/Nfin - + #print(reg) @@ -261,23 +365,23 @@ def sinkhorn_knopp(a,b, M, reg, numItermax = 1000, stopThr=1e-9, verbose=False, if log: log['u']=u log['v']=v - - if nbb: #return only loss + + if nbb: #return only loss res=np.zeros((nbb)) for i in range(nbb): res[i]=np.sum(u[:,i].reshape((-1,1))*K*v[:,i].reshape((1,-1))*M) if log: return res,log else: - return res - + return res + else: # return OT matrix - + if log: return u.reshape((-1,1))*K*v.reshape((1,-1)),log else: return u.reshape((-1,1))*K*v.reshape((1,-1)) - + def sinkhorn_stabilized(a,b, M, reg, numItermax = 1000,tau=1e3, stopThr=1e-9,warmstart=None, verbose=False,print_period=20, log=False,**kwargs): """ @@ -393,7 +497,7 @@ def sinkhorn_stabilized(a,b, M, reg, numItermax = 1000,tau=1e3, stopThr=1e-9,war alpha,beta=np.zeros(na),np.zeros(nb) else: alpha,beta=warmstart - + if nbb: u,v = np.ones((na,nbb))/na,np.ones((nb,nbb))/nb else: @@ -420,7 +524,7 @@ def sinkhorn_stabilized(a,b, M, reg, numItermax = 1000,tau=1e3, stopThr=1e-9,war uprev = u vprev = v - + # sinkhorn update v = b/(np.dot(K.T,u)+1e-16) u = a/(np.dot(K,v)+1e-16) @@ -471,8 +575,8 @@ def sinkhorn_stabilized(a,b, M, reg, numItermax = 1000,tau=1e3, stopThr=1e-9,war break cpt = cpt +1 - - + + #print('err=',err,' cpt=',cpt) if log: log['logu']=alpha/reg+np.log(u) @@ -493,7 +597,7 @@ def sinkhorn_stabilized(a,b, M, reg, numItermax = 1000,tau=1e3, stopThr=1e-9,war res=np.zeros((nbb)) for i in range(nbb): res[i]=np.sum(get_Gamma(alpha,beta,u[:,i],v[:,i])*M) - return res + return res else: return get_Gamma(alpha,beta,u,v) -- cgit v1.2.3 From 37ca3142a4c8c808382f5eb1c23bf198c3e4610e Mon Sep 17 00:00:00 2001 From: Alexandre Gramfort Date: Wed, 12 Jul 2017 23:52:57 +0200 Subject: pep8 --- ot/bregman.py | 495 +++++++++++++++++++++++++++++--------------------------- ot/da.py | 504 +++++++++++++++++++++++++++++---------------------------- ot/datasets.py | 107 ++++++------ ot/dr.py | 197 +++++++++++----------- ot/optim.py | 133 +++++++-------- 5 files changed, 735 insertions(+), 701 deletions(-) (limited to 'ot/bregman.py') diff --git a/ot/bregman.py b/ot/bregman.py index 0d68602..fe10880 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -5,7 +5,8 @@ Bregman projections for regularized OT import numpy as np -def sinkhorn(a,b, M, reg,method='sinkhorn', numItermax = 1000, stopThr=1e-9, verbose=False, log=False,**kwargs): + +def sinkhorn(a, b, M, reg, method='sinkhorn', numItermax=1000, stopThr=1e-9, verbose=False, log=False, **kwargs): u""" Solve the entropic regularization optimal transport problem and return the OT matrix @@ -92,22 +93,28 @@ def sinkhorn(a,b, M, reg,method='sinkhorn', numItermax = 1000, stopThr=1e-9, ver """ - if method.lower()=='sinkhorn': - sink= lambda: sinkhorn_knopp(a,b, M, reg,numItermax=numItermax, - stopThr=stopThr, verbose=verbose, log=log,**kwargs) - elif method.lower()=='sinkhorn_stabilized': - sink= lambda: sinkhorn_stabilized(a,b, M, reg,numItermax=numItermax, - stopThr=stopThr, verbose=verbose, log=log, **kwargs) - elif method.lower()=='sinkhorn_epsilon_scaling': - sink= lambda: sinkhorn_epsilon_scaling(a,b, M, reg,numItermax=numItermax, - stopThr=stopThr, verbose=verbose, log=log, **kwargs) + if method.lower() == 'sinkhorn': + def sink(): + return sinkhorn_knopp(a, b, M, reg, numItermax=numItermax, + stopThr=stopThr, verbose=verbose, log=log, **kwargs) + elif method.lower() == 'sinkhorn_stabilized': + def sink(): + return sinkhorn_stabilized(a, b, M, reg, numItermax=numItermax, + stopThr=stopThr, verbose=verbose, log=log, **kwargs) + elif method.lower() == 'sinkhorn_epsilon_scaling': + def sink(): + return sinkhorn_epsilon_scaling(a, b, M, reg, numItermax=numItermax, + stopThr=stopThr, verbose=verbose, log=log, **kwargs) else: print('Warning : unknown method using classic Sinkhorn Knopp') - sink= lambda: sinkhorn_knopp(a,b, M, reg, **kwargs) + + def sink(): + return sinkhorn_knopp(a, b, M, reg, **kwargs) return sink() -def sinkhorn2(a,b, M, reg,method='sinkhorn', numItermax = 1000, stopThr=1e-9, verbose=False, log=False,**kwargs): + +def sinkhorn2(a, b, M, reg, method='sinkhorn', numItermax=1000, stopThr=1e-9, verbose=False, log=False, **kwargs): u""" Solve the entropic regularization optimal transport problem and return the loss @@ -170,7 +177,7 @@ def sinkhorn2(a,b, M, reg,method='sinkhorn', numItermax = 1000, stopThr=1e-9, ve >>> M=[[0.,1.],[1.,0.]] >>> ot.sinkhorn2(a,b,M,1) array([ 0.26894142]) - + References @@ -194,27 +201,32 @@ def sinkhorn2(a,b, M, reg,method='sinkhorn', numItermax = 1000, stopThr=1e-9, ve """ - if method.lower()=='sinkhorn': - sink= lambda: sinkhorn_knopp(a,b, M, reg,numItermax=numItermax, - stopThr=stopThr, verbose=verbose, log=log,**kwargs) - elif method.lower()=='sinkhorn_stabilized': - sink= lambda: sinkhorn_stabilized(a,b, M, reg,numItermax=numItermax, - stopThr=stopThr, verbose=verbose, log=log, **kwargs) - elif method.lower()=='sinkhorn_epsilon_scaling': - sink= lambda: sinkhorn_epsilon_scaling(a,b, M, reg,numItermax=numItermax, - stopThr=stopThr, verbose=verbose, log=log, **kwargs) + if method.lower() == 'sinkhorn': + def sink(): + return sinkhorn_knopp(a, b, M, reg, numItermax=numItermax, + stopThr=stopThr, verbose=verbose, log=log, **kwargs) + elif method.lower() == 'sinkhorn_stabilized': + def sink(): + return sinkhorn_stabilized(a, b, M, reg, numItermax=numItermax, + stopThr=stopThr, verbose=verbose, log=log, **kwargs) + elif method.lower() == 'sinkhorn_epsilon_scaling': + def sink(): + return sinkhorn_epsilon_scaling(a, b, M, reg, numItermax=numItermax, + stopThr=stopThr, verbose=verbose, log=log, **kwargs) else: print('Warning : unknown method using classic Sinkhorn Knopp') - sink= lambda: sinkhorn_knopp(a,b, M, reg, **kwargs) - - b=np.asarray(b,dtype=np.float64) - if len(b.shape)<2: - b=b.reshape((-1,1)) + + def sink(): + return sinkhorn_knopp(a, b, M, reg, **kwargs) + + b = np.asarray(b, dtype=np.float64) + if len(b.shape) < 2: + b = b.reshape((-1, 1)) return sink() -def sinkhorn_knopp(a,b, M, reg, numItermax = 1000, stopThr=1e-9, verbose=False, log=False,**kwargs): +def sinkhorn_knopp(a, b, M, reg, numItermax=1000, stopThr=1e-9, verbose=False, log=False, **kwargs): """ Solve the entropic regularization optimal transport problem and return the OT matrix @@ -290,100 +302,101 @@ def sinkhorn_knopp(a,b, M, reg, numItermax = 1000, stopThr=1e-9, verbose=False, """ - a=np.asarray(a,dtype=np.float64) - b=np.asarray(b,dtype=np.float64) - M=np.asarray(M,dtype=np.float64) - - - if len(a)==0: - a=np.ones((M.shape[0],),dtype=np.float64)/M.shape[0] - if len(b)==0: - b=np.ones((M.shape[1],),dtype=np.float64)/M.shape[1] + a = np.asarray(a, dtype=np.float64) + b = np.asarray(b, dtype=np.float64) + M = np.asarray(M, dtype=np.float64) + if len(a) == 0: + a = np.ones((M.shape[0],), dtype=np.float64) / M.shape[0] + if len(b) == 0: + b = np.ones((M.shape[1],), dtype=np.float64) / M.shape[1] # init data Nini = len(a) Nfin = len(b) - if len(b.shape)>1: - nbb=b.shape[1] + if len(b.shape) > 1: + nbb = b.shape[1] else: - nbb=0 - + nbb = 0 if log: - log={'err':[]} + log = {'err': []} - # we assume that no distances are null except those of the diagonal of distances + # we assume that no distances are null except those of the diagonal of + # distances if nbb: - u = np.ones((Nini,nbb))/Nini - v = np.ones((Nfin,nbb))/Nfin + u = np.ones((Nini, nbb)) / Nini + v = np.ones((Nfin, nbb)) / Nfin else: - u = np.ones(Nini)/Nini - v = np.ones(Nfin)/Nfin + u = np.ones(Nini) / Nini + v = np.ones(Nfin) / Nfin + # print(reg) - #print(reg) + K = np.exp(-M / reg) + # print(np.min(K)) - K = np.exp(-M/reg) - #print(np.min(K)) - - Kp = (1/a).reshape(-1, 1) * K + Kp = (1 / a).reshape(-1, 1) * K cpt = 0 - err=1 - while (err>stopThr and cpt stopThr and cpt < numItermax): uprev = u vprev = v KtransposeU = np.dot(K.T, u) v = np.divide(b, KtransposeU) - u = 1./np.dot(Kp,v) + u = 1. / np.dot(Kp, v) - if (np.any(KtransposeU==0) or - np.any(np.isnan(u)) or np.any(np.isnan(v)) or - np.any(np.isinf(u)) or np.any(np.isinf(v))): + if (np.any(KtransposeU == 0) or + np.any(np.isnan(u)) or np.any(np.isnan(v)) or + np.any(np.isinf(u)) or np.any(np.isinf(v))): # we have reached the machine precision # come back to previous solution and quit loop print('Warning: numerical errors at iteration', cpt) u = uprev v = vprev break - if cpt%10==0: - # we can speed up the process by checking for the error only all the 10th iterations + if cpt % 10 == 0: + # we can speed up the process by checking for the error only all + # the 10th iterations if nbb: - err = np.sum((u-uprev)**2)/np.sum((u)**2)+np.sum((v-vprev)**2)/np.sum((v)**2) + err = np.sum((u - uprev)**2) / np.sum((u)**2) + \ + np.sum((v - vprev)**2) / np.sum((v)**2) else: transp = u.reshape(-1, 1) * (K * v) - err = np.linalg.norm((np.sum(transp,axis=0)-b))**2 + err = np.linalg.norm((np.sum(transp, axis=0) - b))**2 if log: log['err'].append(err) if verbose: - if cpt%200 ==0: - print('{:5s}|{:12s}'.format('It.','Err')+'\n'+'-'*19) - print('{:5d}|{:8e}|'.format(cpt,err)) - cpt = cpt +1 + if cpt % 200 == 0: + print( + '{:5s}|{:12s}'.format('It.', 'Err') + '\n' + '-' * 19) + print('{:5d}|{:8e}|'.format(cpt, err)) + cpt = cpt + 1 if log: - log['u']=u - log['v']=v + log['u'] = u + log['v'] = v - if nbb: #return only loss - res=np.zeros((nbb)) + if nbb: # return only loss + res = np.zeros((nbb)) for i in range(nbb): - res[i]=np.sum(u[:,i].reshape((-1,1))*K*v[:,i].reshape((1,-1))*M) + res[i] = np.sum( + u[:, i].reshape((-1, 1)) * K * v[:, i].reshape((1, -1)) * M) if log: - return res,log + return res, log else: return res - else: # return OT matrix + else: # return OT matrix if log: - return u.reshape((-1,1))*K*v.reshape((1,-1)),log + return u.reshape((-1, 1)) * K * v.reshape((1, -1)), log else: - return u.reshape((-1,1))*K*v.reshape((1,-1)) + return u.reshape((-1, 1)) * K * v.reshape((1, -1)) -def sinkhorn_stabilized(a,b, M, reg, numItermax = 1000,tau=1e3, stopThr=1e-9,warmstart=None, verbose=False,print_period=20, log=False,**kwargs): +def sinkhorn_stabilized(a, b, M, reg, numItermax=1000, tau=1e3, stopThr=1e-9, warmstart=None, verbose=False, print_period=20, log=False, **kwargs): """ Solve the entropic regularization OT problem with log stabilization @@ -468,21 +481,21 @@ def sinkhorn_stabilized(a,b, M, reg, numItermax = 1000,tau=1e3, stopThr=1e-9,war """ - a=np.asarray(a,dtype=np.float64) - b=np.asarray(b,dtype=np.float64) - M=np.asarray(M,dtype=np.float64) + a = np.asarray(a, dtype=np.float64) + b = np.asarray(b, dtype=np.float64) + M = np.asarray(M, dtype=np.float64) - if len(a)==0: - a=np.ones((M.shape[0],),dtype=np.float64)/M.shape[0] - if len(b)==0: - b=np.ones((M.shape[1],),dtype=np.float64)/M.shape[1] + if len(a) == 0: + a = np.ones((M.shape[0],), dtype=np.float64) / M.shape[0] + if len(b) == 0: + b = np.ones((M.shape[1],), dtype=np.float64) / M.shape[1] # test if multiple target - if len(b.shape)>1: - nbb=b.shape[1] - a=a[:,np.newaxis] + if len(b.shape) > 1: + nbb = b.shape[1] + a = a[:, np.newaxis] else: - nbb=0 + nbb = 0 # init data na = len(a) @@ -490,81 +503,80 @@ def sinkhorn_stabilized(a,b, M, reg, numItermax = 1000,tau=1e3, stopThr=1e-9,war cpt = 0 if log: - log={'err':[]} + log = {'err': []} - # we assume that no distances are null except those of the diagonal of distances + # we assume that no distances are null except those of the diagonal of + # distances if warmstart is None: - alpha,beta=np.zeros(na),np.zeros(nb) + alpha, beta = np.zeros(na), np.zeros(nb) else: - alpha,beta=warmstart + alpha, beta = warmstart if nbb: - u,v = np.ones((na,nbb))/na,np.ones((nb,nbb))/nb + u, v = np.ones((na, nbb)) / na, np.ones((nb, nbb)) / nb else: - u,v = np.ones(na)/na,np.ones(nb)/nb + u, v = np.ones(na) / na, np.ones(nb) / nb - def get_K(alpha,beta): + def get_K(alpha, beta): """log space computation""" - return np.exp(-(M-alpha.reshape((na,1))-beta.reshape((1,nb)))/reg) + return np.exp(-(M - alpha.reshape((na, 1)) - beta.reshape((1, nb))) / reg) - def get_Gamma(alpha,beta,u,v): + def get_Gamma(alpha, beta, u, v): """log space gamma computation""" - return np.exp(-(M-alpha.reshape((na,1))-beta.reshape((1,nb)))/reg+np.log(u.reshape((na,1)))+np.log(v.reshape((1,nb)))) + return np.exp(-(M - alpha.reshape((na, 1)) - beta.reshape((1, nb))) / reg + np.log(u.reshape((na, 1))) + np.log(v.reshape((1, nb)))) - #print(np.min(K)) + # print(np.min(K)) - K=get_K(alpha,beta) + K = get_K(alpha, beta) transp = K - loop=1 + loop = 1 cpt = 0 - err=1 + err = 1 while loop: - - uprev = u vprev = v # sinkhorn update - v = b/(np.dot(K.T,u)+1e-16) - u = a/(np.dot(K,v)+1e-16) - + v = b / (np.dot(K.T, u) + 1e-16) + u = a / (np.dot(K, v) + 1e-16) # remove numerical problems and store them in K - if np.abs(u).max()>tau or np.abs(v).max()>tau: + if np.abs(u).max() > tau or np.abs(v).max() > tau: if nbb: - alpha,beta=alpha+reg*np.max(np.log(u),1),beta+reg*np.max(np.log(v)) + alpha, beta = alpha + reg * \ + np.max(np.log(u), 1), beta + reg * np.max(np.log(v)) else: - alpha,beta=alpha+reg*np.log(u),beta+reg*np.log(v) + alpha, beta = alpha + reg * np.log(u), beta + reg * np.log(v) if nbb: - u,v = np.ones((na,nbb))/na,np.ones((nb,nbb))/nb + u, v = np.ones((na, nbb)) / na, np.ones((nb, nbb)) / nb else: - u,v = np.ones(na)/na,np.ones(nb)/nb - K=get_K(alpha,beta) - + u, v = np.ones(na) / na, np.ones(nb) / nb + K = get_K(alpha, beta) - if cpt%print_period==0: - # we can speed up the process by checking for the error only all the 10th iterations + if cpt % print_period == 0: + # we can speed up the process by checking for the error only all + # the 10th iterations if nbb: - err = np.sum((u-uprev)**2)/np.sum((u)**2)+np.sum((v-vprev)**2)/np.sum((v)**2) + err = np.sum((u - uprev)**2) / np.sum((u)**2) + \ + np.sum((v - vprev)**2) / np.sum((v)**2) else: - transp = get_Gamma(alpha,beta,u,v) - err = np.linalg.norm((np.sum(transp,axis=0)-b))**2 + transp = get_Gamma(alpha, beta, u, v) + err = np.linalg.norm((np.sum(transp, axis=0) - b))**2 if log: log['err'].append(err) if verbose: - if cpt%(print_period*20) ==0: - print('{:5s}|{:12s}'.format('It.','Err')+'\n'+'-'*19) - print('{:5d}|{:8e}|'.format(cpt,err)) + if cpt % (print_period * 20) == 0: + print( + '{:5s}|{:12s}'.format('It.', 'Err') + '\n' + '-' * 19) + print('{:5d}|{:8e}|'.format(cpt, err)) + if err <= stopThr: + loop = False - if err<=stopThr: - loop=False - - if cpt>=numItermax: - loop=False - + if cpt >= numItermax: + loop = False if np.any(np.isnan(u)) or np.any(np.isnan(v)): # we have reached the machine precision @@ -574,34 +586,34 @@ def sinkhorn_stabilized(a,b, M, reg, numItermax = 1000,tau=1e3, stopThr=1e-9,war v = vprev break - cpt = cpt +1 - + cpt = cpt + 1 #print('err=',err,' cpt=',cpt) if log: - log['logu']=alpha/reg+np.log(u) - log['logv']=beta/reg+np.log(v) - log['alpha']=alpha+reg*np.log(u) - log['beta']=beta+reg*np.log(v) - log['warmstart']=(log['alpha'],log['beta']) + log['logu'] = alpha / reg + np.log(u) + log['logv'] = beta / reg + np.log(v) + log['alpha'] = alpha + reg * np.log(u) + log['beta'] = beta + reg * np.log(v) + log['warmstart'] = (log['alpha'], log['beta']) if nbb: - res=np.zeros((nbb)) + res = np.zeros((nbb)) for i in range(nbb): - res[i]=np.sum(get_Gamma(alpha,beta,u[:,i],v[:,i])*M) - return res,log + res[i] = np.sum(get_Gamma(alpha, beta, u[:, i], v[:, i]) * M) + return res, log else: - return get_Gamma(alpha,beta,u,v),log + return get_Gamma(alpha, beta, u, v), log else: if nbb: - res=np.zeros((nbb)) + res = np.zeros((nbb)) for i in range(nbb): - res[i]=np.sum(get_Gamma(alpha,beta,u[:,i],v[:,i])*M) + res[i] = np.sum(get_Gamma(alpha, beta, u[:, i], v[:, i]) * M) return res else: - return get_Gamma(alpha,beta,u,v) + return get_Gamma(alpha, beta, u, v) + -def sinkhorn_epsilon_scaling(a,b, M, reg, numItermax = 100, epsilon0=1e4, numInnerItermax = 100,tau=1e3, stopThr=1e-9,warmstart=None, verbose=False,print_period=10, log=False,**kwargs): +def sinkhorn_epsilon_scaling(a, b, M, reg, numItermax=100, epsilon0=1e4, numInnerItermax=100, tau=1e3, stopThr=1e-9, warmstart=None, verbose=False, print_period=10, log=False, **kwargs): """ Solve the entropic regularization optimal transport problem with log stabilization and epsilon scaling. @@ -690,14 +702,14 @@ def sinkhorn_epsilon_scaling(a,b, M, reg, numItermax = 100, epsilon0=1e4, numInn """ - a=np.asarray(a,dtype=np.float64) - b=np.asarray(b,dtype=np.float64) - M=np.asarray(M,dtype=np.float64) + a = np.asarray(a, dtype=np.float64) + b = np.asarray(b, dtype=np.float64) + M = np.asarray(M, dtype=np.float64) - if len(a)==0: - a=np.ones((M.shape[0],),dtype=np.float64)/M.shape[0] - if len(b)==0: - b=np.ones((M.shape[1],),dtype=np.float64)/M.shape[1] + if len(a) == 0: + a = np.ones((M.shape[0],), dtype=np.float64) / M.shape[0] + if len(b) == 0: + b = np.ones((M.shape[1],), dtype=np.float64) / M.shape[1] # init data na = len(a) @@ -705,88 +717,94 @@ def sinkhorn_epsilon_scaling(a,b, M, reg, numItermax = 100, epsilon0=1e4, numInn # nrelative umerical precision with 64 bits numItermin = 35 - numItermax=max(numItermin,numItermax) # ensure that last velue is exact - + numItermax = max(numItermin, numItermax) # ensure that last velue is exact cpt = 0 if log: - log={'err':[]} + log = {'err': []} - # we assume that no distances are null except those of the diagonal of distances + # we assume that no distances are null except those of the diagonal of + # distances if warmstart is None: - alpha,beta=np.zeros(na),np.zeros(nb) + alpha, beta = np.zeros(na), np.zeros(nb) else: - alpha,beta=warmstart - + alpha, beta = warmstart - def get_K(alpha,beta): + def get_K(alpha, beta): """log space computation""" - return np.exp(-(M-alpha.reshape((na,1))-beta.reshape((1,nb)))/reg) + return np.exp(-(M - alpha.reshape((na, 1)) - beta.reshape((1, nb))) / reg) - #print(np.min(K)) - def get_reg(n): # exponential decreasing - return (epsilon0-reg)*np.exp(-n)+reg + # print(np.min(K)) + def get_reg(n): # exponential decreasing + return (epsilon0 - reg) * np.exp(-n) + reg - loop=1 + loop = 1 cpt = 0 - err=1 + err = 1 while loop: - regi=get_reg(cpt) + regi = get_reg(cpt) - G,logi=sinkhorn_stabilized(a,b, M, regi, numItermax = numInnerItermax, stopThr=1e-9,warmstart=(alpha,beta), verbose=False,print_period=20,tau=tau, log=True) + G, logi = sinkhorn_stabilized(a, b, M, regi, numItermax=numInnerItermax, stopThr=1e-9, warmstart=( + alpha, beta), verbose=False, print_period=20, tau=tau, log=True) - alpha=logi['alpha'] - beta=logi['beta'] + alpha = logi['alpha'] + beta = logi['beta'] - if cpt>=numItermax: - loop=False + if cpt >= numItermax: + loop = False - if cpt%(print_period)==0: # spsion nearly converged - # we can speed up the process by checking for the error only all the 10th iterations + if cpt % (print_period) == 0: # spsion nearly converged + # we can speed up the process by checking for the error only all + # the 10th iterations transp = G - err = np.linalg.norm((np.sum(transp,axis=0)-b))**2+np.linalg.norm((np.sum(transp,axis=1)-a))**2 + err = np.linalg.norm( + (np.sum(transp, axis=0) - b))**2 + np.linalg.norm((np.sum(transp, axis=1) - a))**2 if log: log['err'].append(err) if verbose: - if cpt%(print_period*10) ==0: - print('{:5s}|{:12s}'.format('It.','Err')+'\n'+'-'*19) - print('{:5d}|{:8e}|'.format(cpt,err)) + if cpt % (print_period * 10) == 0: + print( + '{:5s}|{:12s}'.format('It.', 'Err') + '\n' + '-' * 19) + print('{:5d}|{:8e}|'.format(cpt, err)) - if err<=stopThr and cpt>numItermin: - loop=False + if err <= stopThr and cpt > numItermin: + loop = False - cpt = cpt +1 + cpt = cpt + 1 #print('err=',err,' cpt=',cpt) if log: - log['alpha']=alpha - log['beta']=beta - log['warmstart']=(log['alpha'],log['beta']) - return G,log + log['alpha'] = alpha + log['beta'] = beta + log['warmstart'] = (log['alpha'], log['beta']) + return G, log else: return G -def geometricBar(weights,alldistribT): +def geometricBar(weights, alldistribT): """return the weighted geometric mean of distributions""" - assert(len(weights)==alldistribT.shape[1]) - return np.exp(np.dot(np.log(alldistribT),weights.T)) + assert(len(weights) == alldistribT.shape[1]) + return np.exp(np.dot(np.log(alldistribT), weights.T)) + def geometricMean(alldistribT): """return the geometric mean of distributions""" - return np.exp(np.mean(np.log(alldistribT),axis=1)) + return np.exp(np.mean(np.log(alldistribT), axis=1)) -def projR(gamma,p): + +def projR(gamma, p): """return the KL projection on the row constrints """ - return np.multiply(gamma.T,p/np.maximum(np.sum(gamma,axis=1),1e-10)).T + return np.multiply(gamma.T, p / np.maximum(np.sum(gamma, axis=1), 1e-10)).T + -def projC(gamma,q): +def projC(gamma, q): """return the KL projection on the column constrints """ - return np.multiply(gamma,q/np.maximum(np.sum(gamma,axis=0),1e-10)) + return np.multiply(gamma, q / np.maximum(np.sum(gamma, axis=0), 1e-10)) -def barycenter(A,M,reg, weights=None, numItermax = 1000, stopThr=1e-4,verbose=False,log=False): +def barycenter(A, M, reg, weights=None, numItermax=1000, stopThr=1e-4, verbose=False, log=False): """Compute the entropic regularized wasserstein barycenter of distributions A The function solves the following optimization problem: @@ -837,49 +855,49 @@ def barycenter(A,M,reg, weights=None, numItermax = 1000, stopThr=1e-4,verbose=Fa """ - if weights is None: - weights=np.ones(A.shape[1])/A.shape[1] + weights = np.ones(A.shape[1]) / A.shape[1] else: - assert(len(weights)==A.shape[1]) + assert(len(weights) == A.shape[1]) if log: - log={'err':[]} + log = {'err': []} - #M = M/np.median(M) # suggested by G. Peyre - K = np.exp(-M/reg) + # M = M/np.median(M) # suggested by G. Peyre + K = np.exp(-M / reg) cpt = 0 - err=1 + err = 1 - UKv=np.dot(K,np.divide(A.T,np.sum(K,axis=0)).T) - u = (geometricMean(UKv)/UKv.T).T + UKv = np.dot(K, np.divide(A.T, np.sum(K, axis=0)).T) + u = (geometricMean(UKv) / UKv.T).T - while (err>stopThr and cpt stopThr and cpt < numItermax): + cpt = cpt + 1 + UKv = u * np.dot(K, np.divide(A, np.dot(K, u))) + u = (u.T * geometricBar(weights, UKv)).T / UKv - if cpt%10==1: - err=np.sum(np.std(UKv,axis=1)) + if cpt % 10 == 1: + err = np.sum(np.std(UKv, axis=1)) # log and verbose print if log: log['err'].append(err) if verbose: - if cpt%200 ==0: - print('{:5s}|{:12s}'.format('It.','Err')+'\n'+'-'*19) - print('{:5d}|{:8e}|'.format(cpt,err)) + if cpt % 200 == 0: + print( + '{:5s}|{:12s}'.format('It.', 'Err') + '\n' + '-' * 19) + print('{:5d}|{:8e}|'.format(cpt, err)) if log: - log['niter']=cpt - return geometricBar(weights,UKv),log + log['niter'] = cpt + return geometricBar(weights, UKv), log else: - return geometricBar(weights,UKv) + return geometricBar(weights, UKv) -def unmix(a,D,M,M0,h0,reg,reg0,alpha,numItermax = 1000, stopThr=1e-3,verbose=False,log=False): +def unmix(a, D, M, M0, h0, reg, reg0, alpha, numItermax=1000, stopThr=1e-3, verbose=False, log=False): """ Compute the unmixing of an observation with a given dictionary using Wasserstein distance @@ -943,43 +961,44 @@ def unmix(a,D,M,M0,h0,reg,reg0,alpha,numItermax = 1000, stopThr=1e-3,verbose=Fal """ #M = M/np.median(M) - K = np.exp(-M/reg) + K = np.exp(-M / reg) #M0 = M0/np.median(M0) - K0 = np.exp(-M0/reg0) + K0 = np.exp(-M0 / reg0) old = h0 - err=1 - cpt=0 + err = 1 + cpt = 0 #log = {'niter':0, 'all_err':[]} if log: - log={'err':[]} - - - while (err>stopThr and cpt stopThr and cpt < numItermax): + K = projC(K, a) + K0 = projC(K0, h0) + new = np.sum(K0, axis=1) + # we recombine the current selection from dictionnary + inv_new = np.dot(D, new) + other = np.sum(K, axis=1) + # geometric interpolation + delta = np.exp(alpha * np.log(other) + (1 - alpha) * np.log(inv_new)) + K = projR(K, delta) + K0 = np.dot(np.diag(np.dot(D.T, delta / inv_new)), K0) + + err = np.linalg.norm(np.sum(K0, axis=1) - old) old = new if log: log['err'].append(err) if verbose: - if cpt%200 ==0: - print('{:5s}|{:12s}'.format('It.','Err')+'\n'+'-'*19) - print('{:5d}|{:8e}|'.format(cpt,err)) + if cpt % 200 == 0: + print('{:5s}|{:12s}'.format('It.', 'Err') + '\n' + '-' * 19) + print('{:5d}|{:8e}|'.format(cpt, err)) - cpt = cpt+1 + cpt = cpt + 1 if log: - log['niter']=cpt - return np.sum(K0,axis=1),log + log['niter'] = cpt + return np.sum(K0, axis=1), log else: - return np.sum(K0,axis=1) + return np.sum(K0, axis=1) diff --git a/ot/da.py b/ot/da.py index ddf1c60..5039fbd 100644 --- a/ot/da.py +++ b/ot/da.py @@ -6,12 +6,12 @@ Domain adaptation with optimal transport import numpy as np from .bregman import sinkhorn from .lp import emd -from .utils import unif,dist,kernel +from .utils import unif, dist, kernel from .optim import cg from .optim import gcg -def sinkhorn_lpl1_mm(a,labels_a, b, M, reg, eta=0.1,numItermax = 10,numInnerItermax = 200,stopInnerThr=1e-9,verbose=False,log=False): +def sinkhorn_lpl1_mm(a, labels_a, b, M, reg, eta=0.1, numItermax=10, numInnerItermax=200, stopInnerThr=1e-9, verbose=False, log=False): """ Solve the entropic regularization optimal transport problem with nonconvex group lasso regularization @@ -94,7 +94,7 @@ def sinkhorn_lpl1_mm(a,labels_a, b, M, reg, eta=0.1,numItermax = 10,numInnerIter W = np.zeros(M.shape) for cpt in range(numItermax): - Mreg = M + eta*W + Mreg = M + eta * W transp = sinkhorn(a, b, Mreg, reg, numItermax=numInnerItermax, stopThr=stopInnerThr) # the transport has been computed. Check if classes are really @@ -102,12 +102,13 @@ def sinkhorn_lpl1_mm(a,labels_a, b, M, reg, eta=0.1,numItermax = 10,numInnerIter W = np.ones(M.shape) for (i, c) in enumerate(classes): majs = np.sum(transp[indices_labels[i]], axis=0) - majs = p*((majs+epsilon)**(p-1)) + majs = p * ((majs + epsilon)**(p - 1)) W[indices_labels[i]] = majs return transp -def sinkhorn_l1l2_gl(a,labels_a, b, M, reg, eta=0.1,numItermax = 10,numInnerItermax = 200,stopInnerThr=1e-9,verbose=False,log=False): + +def sinkhorn_l1l2_gl(a, labels_a, b, M, reg, eta=0.1, numItermax=10, numInnerItermax=200, stopInnerThr=1e-9, verbose=False, log=False): """ Solve the entropic regularization optimal transport problem with group lasso regularization @@ -176,32 +177,30 @@ def sinkhorn_l1l2_gl(a,labels_a, b, M, reg, eta=0.1,numItermax = 10,numInnerIter ot.optim.gcg : Generalized conditional gradient for OT problems """ - lstlab=np.unique(labels_a) + lstlab = np.unique(labels_a) def f(G): - res=0 + res = 0 for i in range(G.shape[1]): for lab in lstlab: - temp=G[labels_a==lab,i] - res+=np.linalg.norm(temp) + temp = G[labels_a == lab, i] + res += np.linalg.norm(temp) return res def df(G): - W=np.zeros(G.shape) + W = np.zeros(G.shape) for i in range(G.shape[1]): for lab in lstlab: - temp=G[labels_a==lab,i] - n=np.linalg.norm(temp) + temp = G[labels_a == lab, i] + n = np.linalg.norm(temp) if n: - W[labels_a==lab,i]=temp/n + W[labels_a == lab, i] = temp / n return W - - return gcg(a,b,M,reg,eta,f,df,G0=None,numItermax = numItermax,numInnerItermax=numInnerItermax, stopThr=stopInnerThr,verbose=verbose,log=log) - + return gcg(a, b, M, reg, eta, f, df, G0=None, numItermax=numItermax, numInnerItermax=numInnerItermax, stopThr=stopInnerThr, verbose=verbose, log=log) -def joint_OT_mapping_linear(xs,xt,mu=1,eta=0.001,bias=False,verbose=False,verbose2=False,numItermax = 100,numInnerItermax = 10,stopInnerThr=1e-6,stopThr=1e-5,log=False,**kwargs): +def joint_OT_mapping_linear(xs, xt, mu=1, eta=0.001, bias=False, verbose=False, verbose2=False, numItermax=100, numInnerItermax=10, stopInnerThr=1e-6, stopThr=1e-5, log=False, **kwargs): """Joint OT and linear mapping estimation as proposed in [8] The function solves the following optimization problem: @@ -281,97 +280,105 @@ def joint_OT_mapping_linear(xs,xt,mu=1,eta=0.001,bias=False,verbose=False,verbos """ - ns,nt,d=xs.shape[0],xt.shape[0],xt.shape[1] + ns, nt, d = xs.shape[0], xt.shape[0], xt.shape[1] if bias: - xs1=np.hstack((xs,np.ones((ns,1)))) - xstxs=xs1.T.dot(xs1) - I=np.eye(d+1) - I[-1]=0 - I0=I[:,:-1] - sel=lambda x : x[:-1,:] + xs1 = np.hstack((xs, np.ones((ns, 1)))) + xstxs = xs1.T.dot(xs1) + I = np.eye(d + 1) + I[-1] = 0 + I0 = I[:, :-1] + + def sel(x): + return x[:-1, :] else: - xs1=xs - xstxs=xs1.T.dot(xs1) - I=np.eye(d) - I0=I - sel=lambda x : x + xs1 = xs + xstxs = xs1.T.dot(xs1) + I = np.eye(d) + I0 = I + + def sel(x): + return x if log: - log={'err':[]} + log = {'err': []} - a,b=unif(ns),unif(nt) - M=dist(xs,xt)*ns - G=emd(a,b,M) + a, b = unif(ns), unif(nt) + M = dist(xs, xt) * ns + G = emd(a, b, M) - vloss=[] + vloss = [] - def loss(L,G): + def loss(L, G): """Compute full loss""" - return np.sum((xs1.dot(L)-ns*G.dot(xt))**2)+mu*np.sum(G*M)+eta*np.sum(sel(L-I0)**2) + return np.sum((xs1.dot(L) - ns * G.dot(xt))**2) + mu * np.sum(G * M) + eta * np.sum(sel(L - I0)**2) def solve_L(G): """ solve L problem with fixed G (least square)""" - xst=ns*G.dot(xt) - return np.linalg.solve(xstxs+eta*I,xs1.T.dot(xst)+eta*I0) + xst = ns * G.dot(xt) + return np.linalg.solve(xstxs + eta * I, xs1.T.dot(xst) + eta * I0) - def solve_G(L,G0): + def solve_G(L, G0): """Update G with CG algorithm""" - xsi=xs1.dot(L) + xsi = xs1.dot(L) + def f(G): - return np.sum((xsi-ns*G.dot(xt))**2) + return np.sum((xsi - ns * G.dot(xt))**2) + def df(G): - return -2*ns*(xsi-ns*G.dot(xt)).dot(xt.T) - G=cg(a,b,M,1.0/mu,f,df,G0=G0,numItermax=numInnerItermax,stopThr=stopInnerThr) + return -2 * ns * (xsi - ns * G.dot(xt)).dot(xt.T) + G = cg(a, b, M, 1.0 / mu, f, df, G0=G0, + numItermax=numInnerItermax, stopThr=stopInnerThr) return G + L = solve_L(G) - L=solve_L(G) - - vloss.append(loss(L,G)) + vloss.append(loss(L, G)) if verbose: - print('{:5s}|{:12s}|{:8s}'.format('It.','Loss','Delta loss')+'\n'+'-'*32) - print('{:5d}|{:8e}|{:8e}'.format(0,vloss[-1],0)) - + print('{:5s}|{:12s}|{:8s}'.format( + 'It.', 'Loss', 'Delta loss') + '\n' + '-' * 32) + print('{:5d}|{:8e}|{:8e}'.format(0, vloss[-1], 0)) # init loop - if numItermax>0: - loop=1 + if numItermax > 0: + loop = 1 else: - loop=0 - it=0 + loop = 0 + it = 0 while loop: - it+=1 + it += 1 # update G - G=solve_G(L,G) + G = solve_G(L, G) - #update L - L=solve_L(G) + # update L + L = solve_L(G) - vloss.append(loss(L,G)) + vloss.append(loss(L, G)) - if it>=numItermax: - loop=0 + if it >= numItermax: + loop = 0 - if abs(vloss[-1]-vloss[-2])/abs(vloss[-2])0: - loop=1 + if numItermax > 0: + loop = 1 else: - loop=0 - it=0 + loop = 0 + it = 0 while loop: - it+=1 + it += 1 # update G - G=solve_G(L,G) + G = solve_G(L, G) - #update L - L=solve_L(G) + # update L + L = solve_L(G) - vloss.append(loss(L,G)) + vloss.append(loss(L, G)) - if it>=numItermax: - loop=0 + if it >= numItermax: + loop = 0 - if abs(vloss[-1]-vloss[-2])/abs(vloss[-2])0: # >0 then source to target - G=self.G - w=self.ws.reshape((self.xs.shape[0],1)) - x=self.xt + if direction > 0: # >0 then source to target + G = self.G + w = self.ws.reshape((self.xs.shape[0], 1)) + x = self.xt else: - G=self.G.T - w=self.wt.reshape((self.xt.shape[0],1)) - x=self.xs + G = self.G.T + w = self.wt.reshape((self.xt.shape[0], 1)) + x = self.xs if self.computed: - if self.metric=='sqeuclidean': - return np.dot(G/w,x) # weighted mean + if self.metric == 'sqeuclidean': + return np.dot(G / w, x) # weighted mean else: - print("Warning, metric not handled yet, using weighted average") - return np.dot(G/w,x) # weighted mean + print( + "Warning, metric not handled yet, using weighted average") + return np.dot(G / w, x) # weighted mean return None else: print("Warning, model not fitted yet, returning None") return None - - def predict(self,x,direction=1): + def predict(self, x, direction=1): """ Out of sample mapping using the formulation from [6] For each sample x to map, it finds the nearest source sample xs and @@ -657,29 +666,30 @@ class OTDA(object): .. [6] Ferradans, S., Papadakis, N., Peyré, G., & Aujol, J. F. (2014). Regularized discrete optimal transport. SIAM Journal on Imaging Sciences, 7(3), 1853-1882. """ - if direction>0: # >0 then source to target - xf=self.xt - x0=self.xs + if direction > 0: # >0 then source to target + xf = self.xt + x0 = self.xs else: - xf=self.xs - x0=self.xt + xf = self.xs + x0 = self.xt - D0=dist(x,x0) # dist netween new samples an source - idx=np.argmin(D0,1) # closest one - xf=self.interp(direction)# interp the source samples - return xf[idx,:]+x-x0[idx,:] # aply the delta to the interpolation + D0 = dist(x, x0) # dist netween new samples an source + idx = np.argmin(D0, 1) # closest one + xf = self.interp(direction) # interp the source samples + # aply the delta to the interpolation + return xf[idx, :] + x - x0[idx, :] def normalizeM(self, norm): """ Apply normalization to the loss matrix - - + + Parameters ---------- norm : str type of normalization from 'median','max','log','loglog' - + """ - + if norm == "median": self.M /= float(np.median(self.M)) elif norm == "max": @@ -688,149 +698,149 @@ class OTDA(object): self.M = np.log(1 + self.M) elif norm == "loglog": self.M = np.log(1 + np.log(1 + self.M)) - class OTDA_sinkhorn(OTDA): + """Class for domain adaptation with optimal transport with entropic regularization""" - def fit(self,xs,xt,reg=1,ws=None,wt=None,norm=None,**kwargs): + def fit(self, xs, xt, reg=1, ws=None, wt=None, norm=None, **kwargs): """ Fit regularized domain adaptation between samples is xs and xt (with optional weights)""" - self.xs=xs - self.xt=xt + self.xs = xs + self.xt = xt if wt is None: - wt=unif(xt.shape[0]) + wt = unif(xt.shape[0]) if ws is None: - ws=unif(xs.shape[0]) + ws = unif(xs.shape[0]) - self.ws=ws - self.wt=wt + self.ws = ws + self.wt = wt - self.M=dist(xs,xt,metric=self.metric) + self.M = dist(xs, xt, metric=self.metric) self.normalizeM(norm) - self.G=sinkhorn(ws,wt,self.M,reg,**kwargs) - self.computed=True + self.G = sinkhorn(ws, wt, self.M, reg, **kwargs) + self.computed = True class OTDA_lpl1(OTDA): - """Class for domain adaptation with optimal transport with entropic and group regularization""" + """Class for domain adaptation with optimal transport with entropic and group regularization""" - def fit(self,xs,ys,xt,reg=1,eta=1,ws=None,wt=None,norm=None,**kwargs): + def fit(self, xs, ys, xt, reg=1, eta=1, ws=None, wt=None, norm=None, **kwargs): """ Fit regularized domain adaptation between samples is xs and xt (with optional weights), See ot.da.sinkhorn_lpl1_mm for fit parameters""" - self.xs=xs - self.xt=xt + self.xs = xs + self.xt = xt if wt is None: - wt=unif(xt.shape[0]) + wt = unif(xt.shape[0]) if ws is None: - ws=unif(xs.shape[0]) + ws = unif(xs.shape[0]) - self.ws=ws - self.wt=wt + self.ws = ws + self.wt = wt - self.M=dist(xs,xt,metric=self.metric) + self.M = dist(xs, xt, metric=self.metric) self.normalizeM(norm) - self.G=sinkhorn_lpl1_mm(ws,ys,wt,self.M,reg,eta,**kwargs) - self.computed=True + self.G = sinkhorn_lpl1_mm(ws, ys, wt, self.M, reg, eta, **kwargs) + self.computed = True + class OTDA_l1l2(OTDA): - """Class for domain adaptation with optimal transport with entropic and group lasso regularization""" + """Class for domain adaptation with optimal transport with entropic and group lasso regularization""" - def fit(self,xs,ys,xt,reg=1,eta=1,ws=None,wt=None,norm=None,**kwargs): + def fit(self, xs, ys, xt, reg=1, eta=1, ws=None, wt=None, norm=None, **kwargs): """ Fit regularized domain adaptation between samples is xs and xt (with optional weights), See ot.da.sinkhorn_lpl1_gl for fit parameters""" - self.xs=xs - self.xt=xt + self.xs = xs + self.xt = xt if wt is None: - wt=unif(xt.shape[0]) + wt = unif(xt.shape[0]) if ws is None: - ws=unif(xs.shape[0]) + ws = unif(xs.shape[0]) - self.ws=ws - self.wt=wt + self.ws = ws + self.wt = wt - self.M=dist(xs,xt,metric=self.metric) + self.M = dist(xs, xt, metric=self.metric) self.normalizeM(norm) - self.G=sinkhorn_l1l2_gl(ws,ys,wt,self.M,reg,eta,**kwargs) - self.computed=True + self.G = sinkhorn_l1l2_gl(ws, ys, wt, self.M, reg, eta, **kwargs) + self.computed = True + class OTDA_mapping_linear(OTDA): - """Class for optimal transport with joint linear mapping estimation as in [8]""" + """Class for optimal transport with joint linear mapping estimation as in [8]""" def __init__(self): """ Class initialization""" + self.xs = 0 + self.xt = 0 + self.G = 0 + self.L = 0 + self.bias = False + self.computed = False + self.metric = 'sqeuclidean' - self.xs=0 - self.xt=0 - self.G=0 - self.L=0 - self.bias=False - self.computed=False - self.metric='sqeuclidean' - - def fit(self,xs,xt,mu=1,eta=1,bias=False,**kwargs): + def fit(self, xs, xt, mu=1, eta=1, bias=False, **kwargs): """ Fit domain adaptation between samples is xs and xt (with optional weights)""" - self.xs=xs - self.xt=xt - self.bias=bias - + self.xs = xs + self.xt = xt + self.bias = bias - self.ws=unif(xs.shape[0]) - self.wt=unif(xt.shape[0]) + self.ws = unif(xs.shape[0]) + self.wt = unif(xt.shape[0]) - self.G,self.L=joint_OT_mapping_linear(xs,xt,mu=mu,eta=eta,bias=bias,**kwargs) - self.computed=True + self.G, self.L = joint_OT_mapping_linear( + xs, xt, mu=mu, eta=eta, bias=bias, **kwargs) + self.computed = True def mapping(self): return lambda x: self.predict(x) - - def predict(self,x): + def predict(self, x): """ Out of sample mapping estimated during the call to fit""" if self.computed: if self.bias: - x=np.hstack((x,np.ones((x.shape[0],1)))) - return x.dot(self.L) # aply the delta to the interpolation + x = np.hstack((x, np.ones((x.shape[0], 1)))) + return x.dot(self.L) # aply the delta to the interpolation else: print("Warning, model not fitted yet, returning None") return None -class OTDA_mapping_kernel(OTDA_mapping_linear): - """Class for optimal transport with joint nonlinear mapping estimation as in [8]""" +class OTDA_mapping_kernel(OTDA_mapping_linear): + """Class for optimal transport with joint nonlinear mapping estimation as in [8]""" - def fit(self,xs,xt,mu=1,eta=1,bias=False,kerneltype='gaussian',sigma=1,**kwargs): + def fit(self, xs, xt, mu=1, eta=1, bias=False, kerneltype='gaussian', sigma=1, **kwargs): """ Fit domain adaptation between samples is xs and xt """ - self.xs=xs - self.xt=xt - self.bias=bias - - self.ws=unif(xs.shape[0]) - self.wt=unif(xt.shape[0]) - self.kernel=kerneltype - self.sigma=sigma - self.kwargs=kwargs + self.xs = xs + self.xt = xt + self.bias = bias + self.ws = unif(xs.shape[0]) + self.wt = unif(xt.shape[0]) + self.kernel = kerneltype + self.sigma = sigma + self.kwargs = kwargs - self.G,self.L=joint_OT_mapping_kernel(xs,xt,mu=mu,eta=eta,bias=bias,**kwargs) - self.computed=True + self.G, self.L = joint_OT_mapping_kernel( + xs, xt, mu=mu, eta=eta, bias=bias, **kwargs) + self.computed = True - - def predict(self,x): + def predict(self, x): """ Out of sample mapping estimated during the call to fit""" if self.computed: - K=kernel(x,self.xs,method=self.kernel,sigma=self.sigma,**self.kwargs) + K = kernel( + x, self.xs, method=self.kernel, sigma=self.sigma, **self.kwargs) if self.bias: - K=np.hstack((K,np.ones((x.shape[0],1)))) + K = np.hstack((K, np.ones((x.shape[0], 1)))) return K.dot(self.L) else: print("Warning, model not fitted yet, returning None") - return None \ No newline at end of file + return None diff --git a/ot/datasets.py b/ot/datasets.py index 7816833..4371a23 100644 --- a/ot/datasets.py +++ b/ot/datasets.py @@ -7,7 +7,7 @@ import numpy as np import scipy as sp -def get_1D_gauss(n,m,s): +def get_1D_gauss(n, m, s): """return a 1D histogram for a gaussian distribution (n bins, mean m and std s) Parameters @@ -27,12 +27,12 @@ def get_1D_gauss(n,m,s): 1D histogram for a gaussian distribution """ - x=np.arange(n,dtype=np.float64) - h=np.exp(-(x-m)**2/(2*s**2)) - return h/h.sum() + x = np.arange(n, dtype=np.float64) + h = np.exp(-(x - m)**2 / (2 * s**2)) + return h / h.sum() -def get_2D_samples_gauss(n,m,sigma): +def get_2D_samples_gauss(n, m, sigma): """return n samples drawn from 2D gaussian N(m,sigma) Parameters @@ -52,17 +52,17 @@ def get_2D_samples_gauss(n,m,sigma): n samples drawn from N(m,sigma) """ - if np.isscalar(sigma): - sigma=np.array([sigma,]) - if len(sigma)>1: - P=sp.linalg.sqrtm(sigma) - res= np.random.randn(n,2).dot(P)+m + if np.isscalar(sigma): + sigma = np.array([sigma, ]) + if len(sigma) > 1: + P = sp.linalg.sqrtm(sigma) + res = np.random.randn(n, 2).dot(P) + m else: - res= np.random.randn(n,2)*np.sqrt(sigma)+m + res = np.random.randn(n, 2) * np.sqrt(sigma) + m return res -def get_data_classif(dataset,n,nz=.5,theta=0,**kwargs): +def get_data_classif(dataset, n, nz=.5, theta=0, **kwargs): """ dataset generation for classification problems Parameters @@ -84,48 +84,53 @@ def get_data_classif(dataset,n,nz=.5,theta=0,**kwargs): labels of the samples """ - if dataset.lower()=='3gauss': - y=np.floor((np.arange(n)*1.0/n*3))+1 - x=np.zeros((n,2)) + if dataset.lower() == '3gauss': + y = np.floor((np.arange(n) * 1.0 / n * 3)) + 1 + x = np.zeros((n, 2)) # class 1 - x[y==1,0]=-1.; x[y==1,1]=-1. - x[y==2,0]=-1.; x[y==2,1]=1. - x[y==3,0]=1. ; x[y==3,1]=0 - - x[y!=3,:]+=1.5*nz*np.random.randn(sum(y!=3),2) - x[y==3,:]+=2*nz*np.random.randn(sum(y==3),2) - - elif dataset.lower()=='3gauss2': - y=np.floor((np.arange(n)*1.0/n*3))+1 - x=np.zeros((n,2)) - y[y==4]=3 + x[y == 1, 0] = -1. + x[y == 1, 1] = -1. + x[y == 2, 0] = -1. + x[y == 2, 1] = 1. + x[y == 3, 0] = 1. + x[y == 3, 1] = 0 + + x[y != 3, :] += 1.5 * nz * np.random.randn(sum(y != 3), 2) + x[y == 3, :] += 2 * nz * np.random.randn(sum(y == 3), 2) + + elif dataset.lower() == '3gauss2': + y = np.floor((np.arange(n) * 1.0 / n * 3)) + 1 + x = np.zeros((n, 2)) + y[y == 4] = 3 # class 1 - x[y==1,0]=-2.; x[y==1,1]=-2. - x[y==2,0]=-2.; x[y==2,1]=2. - x[y==3,0]=2. ; x[y==3,1]=0 - - x[y!=3,:]+=nz*np.random.randn(sum(y!=3),2) - x[y==3,:]+=2*nz*np.random.randn(sum(y==3),2) - - elif dataset.lower()=='gaussrot' : - rot=np.array([[np.cos(theta),np.sin(theta)],[-np.sin(theta),np.cos(theta)]]) - m1=np.array([-1,1]) - m2=np.array([1,-1]) - y=np.floor((np.arange(n)*1.0/n*2))+1 - n1=np.sum(y==1) - n2=np.sum(y==2) - x=np.zeros((n,2)) - - x[y==1,:]=get_2D_samples_gauss(n1,m1,nz) - x[y==2,:]=get_2D_samples_gauss(n2,m2,nz) - - x=x.dot(rot) - - + x[y == 1, 0] = -2. + x[y == 1, 1] = -2. + x[y == 2, 0] = -2. + x[y == 2, 1] = 2. + x[y == 3, 0] = 2. + x[y == 3, 1] = 0 + + x[y != 3, :] += nz * np.random.randn(sum(y != 3), 2) + x[y == 3, :] += 2 * nz * np.random.randn(sum(y == 3), 2) + + elif dataset.lower() == 'gaussrot': + rot = np.array( + [[np.cos(theta), np.sin(theta)], [-np.sin(theta), np.cos(theta)]]) + m1 = np.array([-1, 1]) + m2 = np.array([1, -1]) + y = np.floor((np.arange(n) * 1.0 / n * 2)) + 1 + n1 = np.sum(y == 1) + n2 = np.sum(y == 2) + x = np.zeros((n, 2)) + + x[y == 1, :] = get_2D_samples_gauss(n1, m1, nz) + x[y == 2, :] = get_2D_samples_gauss(n2, m2, nz) + + x = x.dot(rot) else: - x=np.array(0) - y=np.array(0) + x = np.array(0) + y = np.array(0) print("unknown dataset") - return x,y.astype(int) \ No newline at end of file + return x, y.astype(int) diff --git a/ot/dr.py b/ot/dr.py index 763ce35..77cbae2 100644 --- a/ot/dr.py +++ b/ot/dr.py @@ -3,43 +3,46 @@ Dimension reduction with optimal transport """ +from scipy import linalg import autograd.numpy as np from pymanopt.manifolds import Stiefel from pymanopt import Problem from pymanopt.solvers import SteepestDescent, TrustRegions -import scipy.linalg as la -def dist(x1,x2): + +def dist(x1, x2): """ Compute squared euclidean distance between samples (autograd) """ - x1p2=np.sum(np.square(x1),1) - x2p2=np.sum(np.square(x2),1) - return x1p2.reshape((-1,1))+x2p2.reshape((1,-1))-2*np.dot(x1,x2.T) + x1p2 = np.sum(np.square(x1), 1) + x2p2 = np.sum(np.square(x2), 1) + return x1p2.reshape((-1, 1)) + x2p2.reshape((1, -1)) - 2 * np.dot(x1, x2.T) + -def sinkhorn(w1,w2,M,reg,k): +def sinkhorn(w1, w2, M, reg, k): """Sinkhorn algorithm with fixed number of iteration (autograd) """ - K=np.exp(-M/reg) - ui=np.ones((M.shape[0],)) - vi=np.ones((M.shape[1],)) + K = np.exp(-M / reg) + ui = np.ones((M.shape[0],)) + vi = np.ones((M.shape[1],)) for i in range(k): - vi=w2/(np.dot(K.T,ui)) - ui=w1/(np.dot(K,vi)) - G=ui.reshape((M.shape[0],1))*K*vi.reshape((1,M.shape[1])) + vi = w2 / (np.dot(K.T, ui)) + ui = w1 / (np.dot(K, vi)) + G = ui.reshape((M.shape[0], 1)) * K * vi.reshape((1, M.shape[1])) return G -def split_classes(X,y): + +def split_classes(X, y): """split samples in X by classes in y """ - lstsclass=np.unique(y) - return [X[y==i,:].astype(np.float32) for i in lstsclass] + lstsclass = np.unique(y) + return [X[y == i, :].astype(np.float32) for i in lstsclass] + + +def fda(X, y, p=2, reg=1e-16): + """ + Fisher Discriminant Analysis -def fda(X,y,p=2,reg=1e-16): - """ - Fisher Discriminant Analysis - - Parameters ---------- X : numpy.ndarray (n,d) @@ -59,62 +62,62 @@ def fda(X,y,p=2,reg=1e-16): proj : fun projection function including mean centering - - """ - - mx=np.mean(X) - X-=mx.reshape((1,-1)) - + + """ + + mx = np.mean(X) + X -= mx.reshape((1, -1)) + # data split between classes - d=X.shape[1] - xc=split_classes(X,y) - nc=len(xc) - - p=min(nc-1,p) - - Cw=0 + d = X.shape[1] + xc = split_classes(X, y) + nc = len(xc) + + p = min(nc - 1, p) + + Cw = 0 for x in xc: - Cw+=np.cov(x,rowvar=False) - Cw/=nc - - mxc=np.zeros((d,nc)) - + Cw += np.cov(x, rowvar=False) + Cw /= nc + + mxc = np.zeros((d, nc)) + for i in range(nc): - mxc[:,i]=np.mean(xc[i]) - - mx0=np.mean(mxc,1) - Cb=0 + mxc[:, i] = np.mean(xc[i]) + + mx0 = np.mean(mxc, 1) + Cb = 0 for i in range(nc): - Cb+=(mxc[:,i]-mx0).reshape((-1,1))*(mxc[:,i]-mx0).reshape((1,-1)) - - w,V=la.eig(Cb,Cw+reg*np.eye(d)) - - idx=np.argsort(w.real) - - Popt=V[:,idx[-p:]] - - - + Cb += (mxc[:, i] - mx0).reshape((-1, 1)) * \ + (mxc[:, i] - mx0).reshape((1, -1)) + + w, V = linalg.eig(Cb, Cw + reg * np.eye(d)) + + idx = np.argsort(w.real) + + Popt = V[:, idx[-p:]] + def proj(X): - return (X-mx.reshape((1,-1))).dot(Popt) - + return (X - mx.reshape((1, -1))).dot(Popt) + return Popt, proj -def wda(X,y,p=2,reg=1,k=10,solver = None,maxiter=100,verbose=0,P0=None): - """ + +def wda(X, y, p=2, reg=1, k=10, solver=None, maxiter=100, verbose=0, P0=None): + """ Wasserstein Discriminant Analysis [11]_ - + The function solves the following optimization problem: .. math:: P = \\text{arg}\min_P \\frac{\\sum_i W(PX^i,PX^i)}{\\sum_{i,j\\neq i} W(PX^i,PX^j)} where : - + - :math:`P` is a linear projection operator in the Stiefel(p,d) manifold - :math:`W` is entropic regularized Wasserstein distances - - :math:`X^i` are samples in the dataset corresponding to class i - + - :math:`X^i` are samples in the dataset corresponding to class i + Parameters ---------- X : numpy.ndarray (n,d) @@ -147,54 +150,50 @@ def wda(X,y,p=2,reg=1,k=10,solver = None,maxiter=100,verbose=0,P0=None): ---------- .. [11] Flamary, R., Cuturi, M., Courty, N., & Rakotomamonjy, A. (2016). Wasserstein Discriminant Analysis. arXiv preprint arXiv:1608.08063. - - """ - - mx=np.mean(X) - X-=mx.reshape((1,-1)) - + + """ # noqa + + mx = np.mean(X) + X -= mx.reshape((1, -1)) + # data split between classes - d=X.shape[1] - xc=split_classes(X,y) + d = X.shape[1] + xc = split_classes(X, y) # compute uniform weighs - wc=[np.ones((x.shape[0]),dtype=np.float32)/x.shape[0] for x in xc] - + wc = [np.ones((x.shape[0]), dtype=np.float32) / x.shape[0] for x in xc] + def cost(P): # wda loss - loss_b=0 - loss_w=0 - - for i,xi in enumerate(xc): - xi=np.dot(xi,P) - for j,xj in enumerate(xc[i:]): - xj=np.dot(xj,P) - M=dist(xi,xj) - G=sinkhorn(wc[i],wc[j+i],M,reg,k) - if j==0: - loss_w+=np.sum(G*M) + loss_b = 0 + loss_w = 0 + + for i, xi in enumerate(xc): + xi = np.dot(xi, P) + for j, xj in enumerate(xc[i:]): + xj = np.dot(xj, P) + M = dist(xi, xj) + G = sinkhorn(wc[i], wc[j + i], M, reg, k) + if j == 0: + loss_w += np.sum(G * M) else: - loss_b+=np.sum(G*M) - - # loss inversed because minimization - return loss_w/loss_b - - + loss_b += np.sum(G * M) + + # loss inversed because minimization + return loss_w / loss_b + # declare manifold and problem - manifold = Stiefel(d, p) + manifold = Stiefel(d, p) problem = Problem(manifold=manifold, cost=cost) - + # declare solver and solve if solver is None: - solver= SteepestDescent(maxiter=maxiter,logverbosity=verbose) - elif solver in ['tr','TrustRegions']: - solver= TrustRegions(maxiter=maxiter,logverbosity=verbose) - - Popt = solver.solve(problem,x=P0) - - def proj(X): - return (X-mx.reshape((1,-1))).dot(Popt) - - return Popt, proj + solver = SteepestDescent(maxiter=maxiter, logverbosity=verbose) + elif solver in ['tr', 'TrustRegions']: + solver = TrustRegions(maxiter=maxiter, logverbosity=verbose) + Popt = solver.solve(problem, x=P0) + def proj(X): + return (X - mx.reshape((1, -1))).dot(Popt) + return Popt, proj diff --git a/ot/optim.py b/ot/optim.py index 79f4f66..adad95e 100644 --- a/ot/optim.py +++ b/ot/optim.py @@ -9,7 +9,9 @@ from .lp import emd from .bregman import sinkhorn # The corresponding scipy function does not work for matrices -def line_search_armijo(f,xk,pk,gfk,old_fval,args=(),c1=1e-4,alpha0=0.99): + + +def line_search_armijo(f, xk, pk, gfk, old_fval, args=(), c1=1e-4, alpha0=0.99): """ Armijo linesearch function that works with matrices @@ -51,20 +53,21 @@ def line_search_armijo(f,xk,pk,gfk,old_fval,args=(),c1=1e-4,alpha0=0.99): def phi(alpha1): fc[0] += 1 - return f(xk + alpha1*pk, *args) + return f(xk + alpha1 * pk, *args) if old_fval is None: phi0 = phi(0.) else: phi0 = old_fval - derphi0 = np.sum(pk*gfk) # Quickfix for matrices - alpha,phi1 = scalar_search_armijo(phi,phi0,derphi0,c1=c1,alpha0=alpha0) + derphi0 = np.sum(pk * gfk) # Quickfix for matrices + alpha, phi1 = scalar_search_armijo( + phi, phi0, derphi0, c1=c1, alpha0=alpha0) - return alpha,fc[0],phi1 + return alpha, fc[0], phi1 -def cg(a,b,M,reg,f,df,G0=None,numItermax = 200,stopThr=1e-9,verbose=False,log=False): +def cg(a, b, M, reg, f, df, G0=None, numItermax=200, stopThr=1e-9, verbose=False, log=False): """ Solve the general regularized OT problem with conditional gradient @@ -128,74 +131,74 @@ def cg(a,b,M,reg,f,df,G0=None,numItermax = 200,stopThr=1e-9,verbose=False,log=Fa """ - loop=1 + loop = 1 if log: - log={'loss':[]} + log = {'loss': []} if G0 is None: - G=np.outer(a,b) + G = np.outer(a, b) else: - G=G0 + G = G0 def cost(G): - return np.sum(M*G)+reg*f(G) + return np.sum(M * G) + reg * f(G) - f_val=cost(G) + f_val = cost(G) if log: log['loss'].append(f_val) - it=0 + it = 0 if verbose: - print('{:5s}|{:12s}|{:8s}'.format('It.','Loss','Delta loss')+'\n'+'-'*32) - print('{:5d}|{:8e}|{:8e}'.format(it,f_val,0)) + print('{:5s}|{:12s}|{:8s}'.format( + 'It.', 'Loss', 'Delta loss') + '\n' + '-' * 32) + print('{:5d}|{:8e}|{:8e}'.format(it, f_val, 0)) while loop: - it+=1 - old_fval=f_val - + it += 1 + old_fval = f_val # problem linearization - Mi=M+reg*df(G) + Mi = M + reg * df(G) # set M positive - Mi+=Mi.min() + Mi += Mi.min() # solve linear program - Gc=emd(a,b,Mi) + Gc = emd(a, b, Mi) - deltaG=Gc-G + deltaG = Gc - G # line search - alpha,fc,f_val = line_search_armijo(cost,G,deltaG,Mi,f_val) + alpha, fc, f_val = line_search_armijo(cost, G, deltaG, Mi, f_val) - G=G+alpha*deltaG + G = G + alpha * deltaG # test convergence - if it>=numItermax: - loop=0 - - delta_fval=(f_val-old_fval)/abs(f_val) - if abs(delta_fval)= numItermax: + loop = 0 + delta_fval = (f_val - old_fval) / abs(f_val) + if abs(delta_fval) < stopThr: + loop = 0 if log: log['loss'].append(f_val) if verbose: - if it%20 ==0: - print('{:5s}|{:12s}|{:8s}'.format('It.','Loss','Delta loss')+'\n'+'-'*32) - print('{:5d}|{:8e}|{:8e}'.format(it,f_val,delta_fval)) - + if it % 20 == 0: + print('{:5s}|{:12s}|{:8s}'.format( + 'It.', 'Loss', 'Delta loss') + '\n' + '-' * 32) + print('{:5d}|{:8e}|{:8e}'.format(it, f_val, delta_fval)) if log: - return G,log + return G, log else: return G -def gcg(a,b,M,reg1,reg2,f,df,G0=None,numItermax = 10,numInnerItermax = 200,stopThr=1e-9,verbose=False,log=False): + +def gcg(a, b, M, reg1, reg2, f, df, G0=None, numItermax=10, numInnerItermax=200, stopThr=1e-9, verbose=False, log=False): """ Solve the general regularized OT problem with the generalized conditional gradient @@ -264,70 +267,68 @@ def gcg(a,b,M,reg1,reg2,f,df,G0=None,numItermax = 10,numInnerItermax = 200,stopT """ - loop=1 + loop = 1 if log: - log={'loss':[]} + log = {'loss': []} if G0 is None: - G=np.outer(a,b) + G = np.outer(a, b) else: - G=G0 + G = G0 def cost(G): - return np.sum(M*G)+ reg1*np.sum(G*np.log(G)) + reg2*f(G) + return np.sum(M * G) + reg1 * np.sum(G * np.log(G)) + reg2 * f(G) - f_val=cost(G) + f_val = cost(G) if log: log['loss'].append(f_val) - it=0 + it = 0 if verbose: - print('{:5s}|{:12s}|{:8s}'.format('It.','Loss','Delta loss')+'\n'+'-'*32) - print('{:5d}|{:8e}|{:8e}'.format(it,f_val,0)) + print('{:5s}|{:12s}|{:8s}'.format( + 'It.', 'Loss', 'Delta loss') + '\n' + '-' * 32) + print('{:5d}|{:8e}|{:8e}'.format(it, f_val, 0)) while loop: - it+=1 - old_fval=f_val - + it += 1 + old_fval = f_val # problem linearization - Mi=M+reg2*df(G) + Mi = M + reg2 * df(G) # solve linear program with Sinkhorn #Gc = sinkhorn_stabilized(a,b, Mi, reg1, numItermax = numInnerItermax) - Gc = sinkhorn(a,b, Mi, reg1, numItermax = numInnerItermax) + Gc = sinkhorn(a, b, Mi, reg1, numItermax=numInnerItermax) - deltaG=Gc-G + deltaG = Gc - G # line search - dcost=Mi+reg1*(1+np.log(G)) #?? - alpha,fc,f_val = line_search_armijo(cost,G,deltaG,dcost,f_val) + dcost = Mi + reg1 * (1 + np.log(G)) # ?? + alpha, fc, f_val = line_search_armijo(cost, G, deltaG, dcost, f_val) - G=G+alpha*deltaG + G = G + alpha * deltaG # test convergence - if it>=numItermax: - loop=0 - - delta_fval=(f_val-old_fval)/abs(f_val) - if abs(delta_fval)= numItermax: + loop = 0 + delta_fval = (f_val - old_fval) / abs(f_val) + if abs(delta_fval) < stopThr: + loop = 0 if log: log['loss'].append(f_val) if verbose: - if it%20 ==0: - print('{:5s}|{:12s}|{:8s}'.format('It.','Loss','Delta loss')+'\n'+'-'*32) - print('{:5d}|{:8e}|{:8e}'.format(it,f_val,delta_fval)) - + if it % 20 == 0: + print('{:5s}|{:12s}|{:8s}'.format( + 'It.', 'Loss', 'Delta loss') + '\n' + '-' * 32) + print('{:5d}|{:8e}|{:8e}'.format(it, f_val, delta_fval)) if log: - return G,log + return G, log else: return G - -- cgit v1.2.3 From 00970175c0f8ba9a99b61a182b32e329f219d382 Mon Sep 17 00:00:00 2001 From: Rémi Flamary Date: Wed, 26 Jul 2017 12:15:40 +0200 Subject: add license and authors on all modules --- ot/__init__.py | 4 ++++ ot/bregman.py | 5 +++++ ot/da.py | 6 ++++++ ot/datasets.py | 4 ++++ ot/dr.py | 4 ++++ ot/gpu/__init__.py | 5 +++++ ot/gpu/bregman.py | 5 +++++ ot/gpu/da.py | 9 +++++++++ ot/lp/__init__.py | 4 ++++ ot/lp/emd_wrap.pyx | 9 ++++++--- ot/optim.py | 4 ++++ ot/plot.py | 3 +++ ot/utils.py | 5 +++++ 13 files changed, 64 insertions(+), 3 deletions(-) (limited to 'ot/bregman.py') diff --git a/ot/__init__.py b/ot/__init__.py index a79a5ce..c2161e4 100644 --- a/ot/__init__.py +++ b/ot/__init__.py @@ -4,6 +4,10 @@ """ +# Author: Remi Flamary +# +# License: MIT License + # All submodules and packages from . import lp diff --git a/ot/bregman.py b/ot/bregman.py index fe10880..71a5548 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -3,6 +3,11 @@ Bregman projections for regularized OT """ +# Author: Remi Flamary +# Nicolas Courty +# +# License: MIT License + import numpy as np diff --git a/ot/da.py b/ot/da.py index 5039fbd..977d532 100644 --- a/ot/da.py +++ b/ot/da.py @@ -3,6 +3,12 @@ Domain adaptation with optimal transport """ +# Author: Remi Flamary +# Nicolas Courty +# Michael Perrot +# +# License: MIT License + import numpy as np from .bregman import sinkhorn from .lp import emd diff --git a/ot/datasets.py b/ot/datasets.py index 4371a23..e4fe118 100644 --- a/ot/datasets.py +++ b/ot/datasets.py @@ -2,6 +2,10 @@ Simple example datasets for OT """ +# Author: Remi Flamary +# +# License: MIT License + import numpy as np import scipy as sp diff --git a/ot/dr.py b/ot/dr.py index 77cbae2..d30ab30 100644 --- a/ot/dr.py +++ b/ot/dr.py @@ -3,6 +3,10 @@ Dimension reduction with optimal transport """ +# Author: Remi Flamary +# +# License: MIT License + from scipy import linalg import autograd.numpy as np from pymanopt.manifolds import Stiefel diff --git a/ot/gpu/__init__.py b/ot/gpu/__init__.py index 40b11c0..c8f9433 100644 --- a/ot/gpu/__init__.py +++ b/ot/gpu/__init__.py @@ -4,4 +4,9 @@ from . import bregman from . import da from .bregman import sinkhorn +# Author: Remi Flamary +# Leo Gautheron +# +# License: MIT License + __all__ = ["bregman", "da", "sinkhorn"] diff --git a/ot/gpu/bregman.py b/ot/gpu/bregman.py index 2302f80..86bfec1 100644 --- a/ot/gpu/bregman.py +++ b/ot/gpu/bregman.py @@ -3,6 +3,11 @@ Bregman projections for regularized OT with GPU """ +# Author: Remi Flamary +# Leo Gautheron +# +# License: MIT License + import numpy as np import cudamat diff --git a/ot/gpu/da.py b/ot/gpu/da.py index c66e755..7fb488d 100644 --- a/ot/gpu/da.py +++ b/ot/gpu/da.py @@ -3,6 +3,15 @@ Domain adaptation with optimal transport with GPU implementation """ +# Author: Remi Flamary +# Nicolas Courty +# Michael Perrot +# Leo Gautheron +# +# License: MIT License + + + import numpy as np from ..utils import unif from ..da import OTDA diff --git a/ot/lp/__init__.py b/ot/lp/__init__.py index db3da78..6e0bdb8 100644 --- a/ot/lp/__init__.py +++ b/ot/lp/__init__.py @@ -3,6 +3,10 @@ Solvers for the original linear program OT problem """ +# Author: Remi Flamary +# +# License: MIT License + import numpy as np # import compiled emd from .emd_wrap import emd_c, emd2_c diff --git a/ot/lp/emd_wrap.pyx b/ot/lp/emd_wrap.pyx index 46794ab..46c96c1 100644 --- a/ot/lp/emd_wrap.pyx +++ b/ot/lp/emd_wrap.pyx @@ -1,9 +1,12 @@ # -*- coding: utf-8 -*- """ -Created on Thu Sep 11 08:42:08 2014 - -@author: rflamary +Cython linker with C solver """ + +# Author: Remi Flamary +# +# License: MIT License + import numpy as np cimport numpy as np diff --git a/ot/optim.py b/ot/optim.py index adad95e..c59089c 100644 --- a/ot/optim.py +++ b/ot/optim.py @@ -3,6 +3,10 @@ Optimization algorithms for OT """ +# Author: Remi Flamary +# +# License: MIT License + import numpy as np from scipy.optimize.linesearch import scalar_search_armijo from .lp import emd diff --git a/ot/plot.py b/ot/plot.py index 61afc9f..784a372 100644 --- a/ot/plot.py +++ b/ot/plot.py @@ -2,6 +2,9 @@ Functions for plotting OT matrices """ +# Author: Remi Flamary +# +# License: MIT License import numpy as np import matplotlib.pylab as pl diff --git a/ot/utils.py b/ot/utils.py index 1dee932..2b2f8b3 100644 --- a/ot/utils.py +++ b/ot/utils.py @@ -2,6 +2,11 @@ """ Various function that can be usefull """ + +# Author: Remi Flamary +# +# License: MIT License + import multiprocessing from functools import reduce import time -- cgit v1.2.3 From 84aa3183491260b9c3dbb9f928499cc18e5341c1 Mon Sep 17 00:00:00 2001 From: Rémi Flamary Date: Wed, 26 Jul 2017 12:22:00 +0200 Subject: pep8 --- ot/__init__.py | 2 +- ot/bregman.py | 16 +++++++++------- ot/da.py | 4 ++-- ot/optim.py | 2 +- 4 files changed, 13 insertions(+), 11 deletions(-) (limited to 'ot/bregman.py') diff --git a/ot/__init__.py b/ot/__init__.py index c2161e4..6d4c4c6 100644 --- a/ot/__init__.py +++ b/ot/__init__.py @@ -28,6 +28,6 @@ from .utils import dist, unif, tic, toc, toq __version__ = "0.3.1" -__all__ = ["emd", "emd2", "sinkhorn","sinkhorn2", "utils", 'datasets', +__all__ = ["emd", "emd2", "sinkhorn", "sinkhorn2", "utils", 'datasets', 'bregman', 'lp', 'plot', 'tic', 'toc', 'toq', 'dist', 'unif', 'barycenter', 'sinkhorn_lpl1_mm', 'da', 'optim'] diff --git a/ot/bregman.py b/ot/bregman.py index 71a5548..929388e 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -108,7 +108,8 @@ def sinkhorn(a, b, M, reg, method='sinkhorn', numItermax=1000, stopThr=1e-9, ver stopThr=stopThr, verbose=verbose, log=log, **kwargs) elif method.lower() == 'sinkhorn_epsilon_scaling': def sink(): - return sinkhorn_epsilon_scaling(a, b, M, reg, numItermax=numItermax, + return sinkhorn_epsilon_scaling( + a, b, M, reg, numItermax=numItermax, stopThr=stopThr, verbose=verbose, log=log, **kwargs) else: print('Warning : unknown method using classic Sinkhorn Knopp') @@ -216,7 +217,8 @@ def sinkhorn2(a, b, M, reg, method='sinkhorn', numItermax=1000, stopThr=1e-9, ve stopThr=stopThr, verbose=verbose, log=log, **kwargs) elif method.lower() == 'sinkhorn_epsilon_scaling': def sink(): - return sinkhorn_epsilon_scaling(a, b, M, reg, numItermax=numItermax, + return sinkhorn_epsilon_scaling( + a, b, M, reg, numItermax=numItermax, stopThr=stopThr, verbose=verbose, log=log, **kwargs) else: print('Warning : unknown method using classic Sinkhorn Knopp') @@ -593,7 +595,7 @@ def sinkhorn_stabilized(a, b, M, reg, numItermax=1000, tau=1e3, stopThr=1e-9, wa cpt = cpt + 1 - #print('err=',err,' cpt=',cpt) + # print('err=',err,' cpt=',cpt) if log: log['logu'] = alpha / reg + np.log(u) log['logv'] = beta / reg + np.log(v) @@ -778,7 +780,7 @@ def sinkhorn_epsilon_scaling(a, b, M, reg, numItermax=100, epsilon0=1e4, numInne loop = False cpt = cpt + 1 - #print('err=',err,' cpt=',cpt) + # print('err=',err,' cpt=',cpt) if log: log['alpha'] = alpha log['beta'] = beta @@ -965,16 +967,16 @@ def unmix(a, D, M, M0, h0, reg, reg0, alpha, numItermax=1000, stopThr=1e-3, verb """ - #M = M/np.median(M) + # M = M/np.median(M) K = np.exp(-M / reg) - #M0 = M0/np.median(M0) + # M0 = M0/np.median(M0) K0 = np.exp(-M0 / reg0) old = h0 err = 1 cpt = 0 - #log = {'niter':0, 'all_err':[]} + # log = {'niter':0, 'all_err':[]} if log: log = {'err': []} diff --git a/ot/da.py b/ot/da.py index 977d532..4f9bce5 100644 --- a/ot/da.py +++ b/ot/da.py @@ -478,7 +478,7 @@ def joint_OT_mapping_kernel(xs, xt, mu=1, eta=0.001, kerneltype='gaussian', sigm Kp[:ns, :ns] = K # ls regu - #K0 = K1.T.dot(K1)+eta*I + # K0 = K1.T.dot(K1)+eta*I # Kreg=I # RKHS regul @@ -490,7 +490,7 @@ def joint_OT_mapping_kernel(xs, xt, mu=1, eta=0.001, kerneltype='gaussian', sigm I = np.eye(ns) # ls regul - #K0 = K1.T.dot(K1)+eta*I + # K0 = K1.T.dot(K1)+eta*I # Kreg=I # proper kernel ridge diff --git a/ot/optim.py b/ot/optim.py index c59089c..1d09adc 100644 --- a/ot/optim.py +++ b/ot/optim.py @@ -304,7 +304,7 @@ def gcg(a, b, M, reg1, reg2, f, df, G0=None, numItermax=10, numInnerItermax=200, Mi = M + reg2 * df(G) # solve linear program with Sinkhorn - #Gc = sinkhorn_stabilized(a,b, Mi, reg1, numItermax = numInnerItermax) + # Gc = sinkhorn_stabilized(a,b, Mi, reg1, numItermax = numInnerItermax) Gc = sinkhorn(a, b, Mi, reg1, numItermax=numInnerItermax) deltaG = Gc - G -- cgit v1.2.3 From 96f8b96cdd50be633d4f3c6f3255cb456e492e08 Mon Sep 17 00:00:00 2001 From: Rémi Flamary Date: Wed, 26 Jul 2017 12:33:14 +0200 Subject: valid flake8 --- ot/bregman.py | 4 ++-- ot/gpu/bregman.py | 2 +- ot/gpu/da.py | 3 +-- 3 files changed, 4 insertions(+), 5 deletions(-) (limited to 'ot/bregman.py') diff --git a/ot/bregman.py b/ot/bregman.py index 929388e..d63c51d 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -110,7 +110,7 @@ def sinkhorn(a, b, M, reg, method='sinkhorn', numItermax=1000, stopThr=1e-9, ver def sink(): return sinkhorn_epsilon_scaling( a, b, M, reg, numItermax=numItermax, - stopThr=stopThr, verbose=verbose, log=log, **kwargs) + stopThr=stopThr, verbose=verbose, log=log, **kwargs) else: print('Warning : unknown method using classic Sinkhorn Knopp') @@ -219,7 +219,7 @@ def sinkhorn2(a, b, M, reg, method='sinkhorn', numItermax=1000, stopThr=1e-9, ve def sink(): return sinkhorn_epsilon_scaling( a, b, M, reg, numItermax=numItermax, - stopThr=stopThr, verbose=verbose, log=log, **kwargs) + stopThr=stopThr, verbose=verbose, log=log, **kwargs) else: print('Warning : unknown method using classic Sinkhorn Knopp') diff --git a/ot/gpu/bregman.py b/ot/gpu/bregman.py index 86bfec1..47939c4 100644 --- a/ot/gpu/bregman.py +++ b/ot/gpu/bregman.py @@ -4,7 +4,7 @@ Bregman projections for regularized OT with GPU """ # Author: Remi Flamary -# Leo Gautheron +# Leo Gautheron # # License: MIT License diff --git a/ot/gpu/da.py b/ot/gpu/da.py index 7fb488d..05c580f 100644 --- a/ot/gpu/da.py +++ b/ot/gpu/da.py @@ -6,12 +6,11 @@ Domain adaptation with optimal transport with GPU implementation # Author: Remi Flamary # Nicolas Courty # Michael Perrot -# Leo Gautheron +# Leo Gautheron # # License: MIT License - import numpy as np from ..utils import unif from ..da import OTDA -- cgit v1.2.3 From 6fdf5de8fa27fa16d6b8910fe96eb67b7761aa0e Mon Sep 17 00:00:00 2001 From: Rémi Flamary Date: Wed, 21 Mar 2018 08:29:50 +0100 Subject: add linear mapping test + autopep8 --- examples/plot_otda_linear_mapping.py | 5 ++--- ot/bregman.py | 30 ++++++++++++++++++++---------- ot/lp/__init__.py | 3 ++- ot/optim.py | 9 ++++++--- ot/utils.py | 2 +- test/test_da.py | 18 ++++++++++++++++++ 6 files changed, 49 insertions(+), 18 deletions(-) (limited to 'ot/bregman.py') diff --git a/examples/plot_otda_linear_mapping.py b/examples/plot_otda_linear_mapping.py index 165fe72..7a3b761 100644 --- a/examples/plot_otda_linear_mapping.py +++ b/examples/plot_otda_linear_mapping.py @@ -9,7 +9,6 @@ Created on Tue Mar 20 14:31:15 2018 import numpy as np import pylab as pl import ot -from scipy import ndimage ############################################################################## # Generate data @@ -87,8 +86,8 @@ def minmax(I): # Loading images -I1 = ndimage.imread('../data/ocean_day.jpg').astype(np.float64) / 256 -I2 = ndimage.imread('../data/ocean_sunset.jpg').astype(np.float64) / 256 +I1 = pl.imread('../data/ocean_day.jpg').astype(np.float64) / 256 +I2 = pl.imread('../data/ocean_sunset.jpg').astype(np.float64) / 256 X1 = im2mat(I1) diff --git a/ot/bregman.py b/ot/bregman.py index d63c51d..07b8660 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -11,7 +11,8 @@ Bregman projections for regularized OT import numpy as np -def sinkhorn(a, b, M, reg, method='sinkhorn', numItermax=1000, stopThr=1e-9, verbose=False, log=False, **kwargs): +def sinkhorn(a, b, M, reg, method='sinkhorn', numItermax=1000, + stopThr=1e-9, verbose=False, log=False, **kwargs): u""" Solve the entropic regularization optimal transport problem and return the OT matrix @@ -120,7 +121,8 @@ def sinkhorn(a, b, M, reg, method='sinkhorn', numItermax=1000, stopThr=1e-9, ver return sink() -def sinkhorn2(a, b, M, reg, method='sinkhorn', numItermax=1000, stopThr=1e-9, verbose=False, log=False, **kwargs): +def sinkhorn2(a, b, M, reg, method='sinkhorn', numItermax=1000, + stopThr=1e-9, verbose=False, log=False, **kwargs): u""" Solve the entropic regularization optimal transport problem and return the loss @@ -233,7 +235,8 @@ def sinkhorn2(a, b, M, reg, method='sinkhorn', numItermax=1000, stopThr=1e-9, ve return sink() -def sinkhorn_knopp(a, b, M, reg, numItermax=1000, stopThr=1e-9, verbose=False, log=False, **kwargs): +def sinkhorn_knopp(a, b, M, reg, numItermax=1000, + stopThr=1e-9, verbose=False, log=False, **kwargs): """ Solve the entropic regularization optimal transport problem and return the OT matrix @@ -403,7 +406,8 @@ def sinkhorn_knopp(a, b, M, reg, numItermax=1000, stopThr=1e-9, verbose=False, l return u.reshape((-1, 1)) * K * v.reshape((1, -1)) -def sinkhorn_stabilized(a, b, M, reg, numItermax=1000, tau=1e3, stopThr=1e-9, warmstart=None, verbose=False, print_period=20, log=False, **kwargs): +def sinkhorn_stabilized(a, b, M, reg, numItermax=1000, tau=1e3, stopThr=1e-9, + warmstart=None, verbose=False, print_period=20, log=False, **kwargs): """ Solve the entropic regularization OT problem with log stabilization @@ -526,11 +530,13 @@ def sinkhorn_stabilized(a, b, M, reg, numItermax=1000, tau=1e3, stopThr=1e-9, wa def get_K(alpha, beta): """log space computation""" - return np.exp(-(M - alpha.reshape((na, 1)) - beta.reshape((1, nb))) / reg) + return np.exp(-(M - alpha.reshape((na, 1)) - + beta.reshape((1, nb))) / reg) def get_Gamma(alpha, beta, u, v): """log space gamma computation""" - return np.exp(-(M - alpha.reshape((na, 1)) - beta.reshape((1, nb))) / reg + np.log(u.reshape((na, 1))) + np.log(v.reshape((1, nb)))) + return np.exp(-(M - alpha.reshape((na, 1)) - beta.reshape((1, nb))) / + reg + np.log(u.reshape((na, 1))) + np.log(v.reshape((1, nb)))) # print(np.min(K)) @@ -620,7 +626,8 @@ def sinkhorn_stabilized(a, b, M, reg, numItermax=1000, tau=1e3, stopThr=1e-9, wa return get_Gamma(alpha, beta, u, v) -def sinkhorn_epsilon_scaling(a, b, M, reg, numItermax=100, epsilon0=1e4, numInnerItermax=100, tau=1e3, stopThr=1e-9, warmstart=None, verbose=False, print_period=10, log=False, **kwargs): +def sinkhorn_epsilon_scaling(a, b, M, reg, numItermax=100, epsilon0=1e4, numInnerItermax=100, + tau=1e3, stopThr=1e-9, warmstart=None, verbose=False, print_period=10, log=False, **kwargs): """ Solve the entropic regularization optimal transport problem with log stabilization and epsilon scaling. @@ -739,7 +746,8 @@ def sinkhorn_epsilon_scaling(a, b, M, reg, numItermax=100, epsilon0=1e4, numInne def get_K(alpha, beta): """log space computation""" - return np.exp(-(M - alpha.reshape((na, 1)) - beta.reshape((1, nb))) / reg) + return np.exp(-(M - alpha.reshape((na, 1)) - + beta.reshape((1, nb))) / reg) # print(np.min(K)) def get_reg(n): # exponential decreasing @@ -811,7 +819,8 @@ def projC(gamma, q): return np.multiply(gamma, q / np.maximum(np.sum(gamma, axis=0), 1e-10)) -def barycenter(A, M, reg, weights=None, numItermax=1000, stopThr=1e-4, verbose=False, log=False): +def barycenter(A, M, reg, weights=None, numItermax=1000, + stopThr=1e-4, verbose=False, log=False): """Compute the entropic regularized wasserstein barycenter of distributions A The function solves the following optimization problem: @@ -904,7 +913,8 @@ def barycenter(A, M, reg, weights=None, numItermax=1000, stopThr=1e-4, verbose=F return geometricBar(weights, UKv) -def unmix(a, D, M, M0, h0, reg, reg0, alpha, numItermax=1000, stopThr=1e-3, verbose=False, log=False): +def unmix(a, D, M, M0, h0, reg, reg0, alpha, numItermax=1000, + stopThr=1e-3, verbose=False, log=False): """ Compute the unmixing of an observation with a given dictionary using Wasserstein distance diff --git a/ot/lp/__init__.py b/ot/lp/__init__.py index 5c09da2..6371feb 100644 --- a/ot/lp/__init__.py +++ b/ot/lp/__init__.py @@ -107,7 +107,8 @@ def emd(a, b, M, numItermax=100000, log=False): return G -def emd2(a, b, M, processes=multiprocessing.cpu_count(), numItermax=100000, log=False, return_matrix=False): +def emd2(a, b, M, processes=multiprocessing.cpu_count(), + numItermax=100000, log=False, return_matrix=False): """Solves the Earth Movers distance problem and returns the loss .. math:: diff --git a/ot/optim.py b/ot/optim.py index 1d09adc..f31fae2 100644 --- a/ot/optim.py +++ b/ot/optim.py @@ -15,7 +15,8 @@ from .bregman import sinkhorn # The corresponding scipy function does not work for matrices -def line_search_armijo(f, xk, pk, gfk, old_fval, args=(), c1=1e-4, alpha0=0.99): +def line_search_armijo(f, xk, pk, gfk, old_fval, + args=(), c1=1e-4, alpha0=0.99): """ Armijo linesearch function that works with matrices @@ -71,7 +72,8 @@ def line_search_armijo(f, xk, pk, gfk, old_fval, args=(), c1=1e-4, alpha0=0.99): return alpha, fc[0], phi1 -def cg(a, b, M, reg, f, df, G0=None, numItermax=200, stopThr=1e-9, verbose=False, log=False): +def cg(a, b, M, reg, f, df, G0=None, numItermax=200, + stopThr=1e-9, verbose=False, log=False): """ Solve the general regularized OT problem with conditional gradient @@ -202,7 +204,8 @@ def cg(a, b, M, reg, f, df, G0=None, numItermax=200, stopThr=1e-9, verbose=False return G -def gcg(a, b, M, reg1, reg2, f, df, G0=None, numItermax=10, numInnerItermax=200, stopThr=1e-9, verbose=False, log=False): +def gcg(a, b, M, reg1, reg2, f, df, G0=None, numItermax=10, + numInnerItermax=200, stopThr=1e-9, verbose=False, log=False): """ Solve the general regularized OT problem with the generalized conditional gradient diff --git a/ot/utils.py b/ot/utils.py index 9eab3fc..16862ea 100644 --- a/ot/utils.py +++ b/ot/utils.py @@ -316,7 +316,7 @@ def _is_deprecated(func): closures = [] is_deprecated = ('deprecated' in ''.join([c.cell_contents for c in closures - if isinstance(c.cell_contents, str)])) + if isinstance(c.cell_contents, str)])) return is_deprecated diff --git a/test/test_da.py b/test/test_da.py index 593dc53..7b63daf 100644 --- a/test/test_da.py +++ b/test/test_da.py @@ -444,6 +444,24 @@ def test_mapping_transport_class(): assert len(otda.log_.keys()) != 0 +def test_linear_mapping(): + + ns = 150 + nt = 200 + + Xs, ys = get_data_classif('3gauss', ns) + Xt, yt = get_data_classif('3gauss2', nt) + + A, b = ot.da.OT_mapping_linear(Xs, Xt) + + Xst = Xs.dot(A) + b + + Ct = np.cov(Xt.T) + Cst = np.cov(Xst.T) + + np.testing.assert_allclose(Ct, Cst, rtol=1e-2, atol=1e-2) + + def test_otda(): n_samples = 150 # nb samples -- cgit v1.2.3 From 94eb4a24fe09c8a193933d8c7f48a657da4e6c52 Mon Sep 17 00:00:00 2001 From: Rémi Flamary Date: Fri, 11 May 2018 16:07:03 +0200 Subject: update documentation in bregman --- ot/bregman.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'ot/bregman.py') diff --git a/ot/bregman.py b/ot/bregman.py index 07b8660..9c84aed 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -844,6 +844,8 @@ def barycenter(A, M, reg, weights=None, numItermax=1000, loss matrix for OT reg : float Regularization term >0 + weights : np.ndarray (n,) + Weights of each histogram i_i on the simplex numItermax : int, optional Max number of iterations stopThr : float, optional -- cgit v1.2.3 From 3aee908ad42d65897f1916de6eab84921ac94a10 Mon Sep 17 00:00:00 2001 From: Rémi Flamary Date: Fri, 11 May 2018 16:58:06 +0200 Subject: pep8 --- ot/bregman.py | 2 +- ot/lp/cvx.py | 104 ++++++++++++++++++++++++++++++---------------------------- 2 files changed, 54 insertions(+), 52 deletions(-) (limited to 'ot/bregman.py') diff --git a/ot/bregman.py b/ot/bregman.py index 9c84aed..e788ef5 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -845,7 +845,7 @@ def barycenter(A, M, reg, weights=None, numItermax=1000, reg : float Regularization term >0 weights : np.ndarray (n,) - Weights of each histogram i_i on the simplex + Weights of each histogram i_i on the simplex numItermax : int, optional Max number of iterations stopThr : float, optional diff --git a/ot/lp/cvx.py b/ot/lp/cvx.py index 4d08916..93097d1 100644 --- a/ot/lp/cvx.py +++ b/ot/lp/cvx.py @@ -15,7 +15,8 @@ try: import cvxopt from cvxopt import solvers, matrix, sparse, spmatrix except ImportError: - cvxopt=False + cvxopt = False + def scipy_sparse_to_spmatrix(A): """Efficient conversion from scipy sparse matrix to cvxopt sparse matrix""" @@ -23,7 +24,8 @@ def scipy_sparse_to_spmatrix(A): SP = spmatrix(coo.data.tolist(), coo.row.tolist(), coo.col.tolist(), size=A.shape) return SP -def barycenter(A, M, weights=None, verbose=False, log=False,solver='interior-point'): + +def barycenter(A, M, weights=None, verbose=False, log=False, solver='interior-point'): """Compute the entropic regularized wasserstein barycenter of distributions A The function solves the following optimization problem [16]: @@ -36,7 +38,7 @@ def barycenter(A, M, weights=None, verbose=False, log=False,solver='interior-poi - :math:`W_1(\cdot,\cdot)` is the Wasserstein distance (see ot.emd.sinkhorn) - :math:`\mathbf{a}_i` are training distributions in the columns of matrix :math:`\mathbf{A}` - The linear program is solved using the default cvxopt solver if installed. + The linear program is solved using the default cvxopt solver if installed. If cvxopt is not installed it uses the lp solver from scipy.optimize. Parameters @@ -48,13 +50,13 @@ def barycenter(A, M, weights=None, verbose=False, log=False,solver='interior-poi reg : float Regularization term >0 weights : np.ndarray (n,) - Weights of each histogram i_i on the simplex + Weights of each histogram i_i on the simplex verbose : bool, optional Print information along iterations log : bool, optional record log if True solver : string, optional - the solver used, default 'interior-point' use the lp solver from + the solver used, default 'interior-point' use the lp solver from scipy.optimize. None, or 'glpk' or 'mosek' use the solver from cvxopt. Returns @@ -78,61 +80,61 @@ def barycenter(A, M, weights=None, verbose=False, log=False,solver='interior-poi weights = np.ones(A.shape[1]) / A.shape[1] else: assert(len(weights) == A.shape[1]) - - n_distributions=A.shape[1] - n=A.shape[0] - - n2=n*n - c=np.zeros((0)) - b_eq1=np.zeros((0)) + + n_distributions = A.shape[1] + n = A.shape[0] + + n2 = n * n + c = np.zeros((0)) + b_eq1 = np.zeros((0)) for i in range(n_distributions): - c=np.concatenate((c,M.ravel()*weights[i])) - b_eq1=np.concatenate((b_eq1,A[:,i])) - c=np.concatenate((c,np.zeros(n))) - - lst_idiag1=[sps.kron(sps.eye(n),np.ones((1,n))) for i in range(n_distributions)] + c = np.concatenate((c, M.ravel() * weights[i])) + b_eq1 = np.concatenate((b_eq1, A[:, i])) + c = np.concatenate((c, np.zeros(n))) + + lst_idiag1 = [sps.kron(sps.eye(n), np.ones((1, n))) for i in range(n_distributions)] # row constraints - A_eq1=sps.hstack((sps.block_diag(lst_idiag1),sps.coo_matrix((n_distributions*n,n)))) - + A_eq1 = sps.hstack((sps.block_diag(lst_idiag1), sps.coo_matrix((n_distributions * n, n)))) + # columns constraints - lst_idiag2=[] - lst_eye=[] + lst_idiag2 = [] + lst_eye = [] for i in range(n_distributions): - if i==0: - lst_idiag2.append(sps.kron(np.ones((1,n)),sps.eye(n))) + if i == 0: + lst_idiag2.append(sps.kron(np.ones((1, n)), sps.eye(n))) lst_eye.append(-sps.eye(n)) else: - lst_idiag2.append(sps.kron(np.ones((1,n)),sps.eye(n-1,n))) - lst_eye.append(-sps.eye(n-1,n)) - - A_eq2=sps.hstack((sps.block_diag(lst_idiag2),sps.vstack(lst_eye))) - b_eq2=np.zeros((A_eq2.shape[0])) - + lst_idiag2.append(sps.kron(np.ones((1, n)), sps.eye(n - 1, n))) + lst_eye.append(-sps.eye(n - 1, n)) + + A_eq2 = sps.hstack((sps.block_diag(lst_idiag2), sps.vstack(lst_eye))) + b_eq2 = np.zeros((A_eq2.shape[0])) + # full problem - A_eq=sps.vstack((A_eq1,A_eq2)) - b_eq=np.concatenate((b_eq1,b_eq2)) - - if not cvxopt or solver in ['interior-point']: # cvxopt not installed or simplex/interior point - + A_eq = sps.vstack((A_eq1, A_eq2)) + b_eq = np.concatenate((b_eq1, b_eq2)) + + if not cvxopt or solver in ['interior-point']: # cvxopt not installed or simplex/interior point + if solver is None: - solver='interior-point' - - options={'sparse':True,'disp': verbose} - sol=sp.optimize.linprog(c,A_eq=A_eq,b_eq=b_eq,method=solver,options=options) - x=sol.x - b=x[-n:] - + solver = 'interior-point' + + options = {'sparse': True, 'disp': verbose} + sol = sp.optimize.linprog(c, A_eq=A_eq, b_eq=b_eq, method=solver, options=options) + x = sol.x + b = x[-n:] + else: - - h=np.zeros((n_distributions*n2+n)) - G=-sps.eye(n_distributions*n2+n) - - sol=solvers.lp(matrix(c),scipy_sparse_to_spmatrix(G),matrix(h),A=scipy_sparse_to_spmatrix(A_eq),b=matrix(b_eq),solver=solver) - - x=np.array(sol['x']) - b=x[-n:].ravel() - + + h = np.zeros((n_distributions * n2 + n)) + G = -sps.eye(n_distributions * n2 + n) + + sol = solvers.lp(matrix(c), scipy_sparse_to_spmatrix(G), matrix(h), A=scipy_sparse_to_spmatrix(A_eq), b=matrix(b_eq), solver=solver) + + x = np.array(sol['x']) + b = x[-n:].ravel() + if log: return b, sol else: - return b \ No newline at end of file + return b -- cgit v1.2.3 From 54f0b47e55c966d5492e4ce19ec4e704ef3278d6 Mon Sep 17 00:00:00 2001 From: Rémi Flamary Date: Tue, 29 May 2018 16:08:33 +0200 Subject: update documentation for barycenter function --- ot/bregman.py | 4 ++-- ot/lp/cvx.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) (limited to 'ot/bregman.py') diff --git a/ot/bregman.py b/ot/bregman.py index e788ef5..b017c1a 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -839,13 +839,13 @@ def barycenter(A, M, reg, weights=None, numItermax=1000, Parameters ---------- A : np.ndarray (d,n) - n training distributions of size d + n training distributions a_i of size d M : np.ndarray (d,d) loss matrix for OT reg : float Regularization term >0 weights : np.ndarray (n,) - Weights of each histogram i_i on the simplex + Weights of each histogram a_i on the simplex (barycentric coodinates) numItermax : int, optional Max number of iterations stopThr : float, optional diff --git a/ot/lp/cvx.py b/ot/lp/cvx.py index fe9ac76..c8c75bc 100644 --- a/ot/lp/cvx.py +++ b/ot/lp/cvx.py @@ -46,13 +46,13 @@ def barycenter(A, M, weights=None, verbose=False, log=False, solver='interior-po Parameters ---------- A : np.ndarray (d,n) - n training distributions of size d + n training distributions a_i of size d M : np.ndarray (d,d) loss matrix for OT reg : float Regularization term >0 weights : np.ndarray (n,) - Weights of each histogram i_i on the simplex + Weights of each histogram a_i on the simplex (barycentric coodinates) verbose : bool, optional Print information along iterations log : bool, optional -- cgit v1.2.3 From cb6bdc516697e3bad6776b897f22c8b6a22f13cd Mon Sep 17 00:00:00 2001 From: LeoGautheron Date: Wed, 11 Jul 2018 22:28:38 +0200 Subject: Speed-up Sinkhorn Speed-up in 3 places: - the computation of pairwise distance is faster with sklearn.metrics.pairwise.euclidean_distances - faster computation of K = np.exp(-M / reg) - faster computation of the error every 10 iterations Example with this little script: import time import numpy as np import ot rng = np.random.RandomState(0) transport = ot.da.SinkhornTransport() time1 = time.time() Xs, ys, Xt = rng.randn(10000, 100), rng.randint(0, 2, size=10000), rng.randn(10000, 100) transport.fit(Xs=Xs, Xt=Xt) time2 = time.time() print("OT Computation Time {:6.2f} sec".format(time2-time1)) transport = ot.da.SinkhornLpl1Transport() transport.fit(Xs=Xs, ys=ys, Xt=Xt) time3 = time.time() print("OT LpL1 Computation Time {:6.2f} sec".format(time3-time2)) Before OT Computation Time 19.93 sec OT LpL1 Computation Time 133.43 sec After OT Computation Time 7.55 sec OT LpL1 Computation Time 82.25 sec --- ot/bregman.py | 14 +++++++++++--- ot/utils.py | 4 +++- 2 files changed, 14 insertions(+), 4 deletions(-) (limited to 'ot/bregman.py') diff --git a/ot/bregman.py b/ot/bregman.py index b017c1a..55c44f6 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -344,8 +344,13 @@ def sinkhorn_knopp(a, b, M, reg, numItermax=1000, # print(reg) - K = np.exp(-M / reg) + K = np.empty(M.shape, dtype=M.dtype) + np.divide(M, -reg, out=K) + np.exp(K, out=K) + # print(np.min(K)) + tmp = np.empty(K.shape, dtype=M.dtype) + tmp2 = np.empty(b.shape, dtype=M.dtype) Kp = (1 / a).reshape(-1, 1) * K cpt = 0 @@ -373,8 +378,11 @@ def sinkhorn_knopp(a, b, M, reg, numItermax=1000, err = np.sum((u - uprev)**2) / np.sum((u)**2) + \ np.sum((v - vprev)**2) / np.sum((v)**2) else: - transp = u.reshape(-1, 1) * (K * v) - err = np.linalg.norm((np.sum(transp, axis=0) - b))**2 + np.multiply(u.reshape(-1, 1), K, out=tmp) + np.multiply(tmp, v.reshape(1, -1), out=tmp) + np.sum(tmp, axis=0, out=tmp2) + tmp2 -= b + err = np.linalg.norm(tmp2)**2 if log: log['err'].append(err) diff --git a/ot/utils.py b/ot/utils.py index 7dac283..5b052ac 100644 --- a/ot/utils.py +++ b/ot/utils.py @@ -13,6 +13,7 @@ import time import numpy as np from scipy.spatial.distance import cdist +from sklearn.metrics.pairwise import euclidean_distances import sys import warnings try: @@ -104,7 +105,8 @@ def dist(x1, x2=None, metric='sqeuclidean'): """ if x2 is None: x2 = x1 - + if metric == "sqeuclidean": + return euclidean_distances(x1, x2, squared=True) return cdist(x1, x2, metric=metric) -- cgit v1.2.3 From 0764e356325df7e18f72c0ff468bfa8f8ee35059 Mon Sep 17 00:00:00 2001 From: LeoGautheron Date: Mon, 16 Jul 2018 06:56:34 +0200 Subject: Add comment & fix flake8 error --- ot/bregman.py | 1 + ot/utils.py | 1 + 2 files changed, 2 insertions(+) (limited to 'ot/bregman.py') diff --git a/ot/bregman.py b/ot/bregman.py index 55c44f6..c8e69ce 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -344,6 +344,7 @@ def sinkhorn_knopp(a, b, M, reg, numItermax=1000, # print(reg) + # Next 3 lines equivalent to K= np.exp(-M/reg), but faster to compute K = np.empty(M.shape, dtype=M.dtype) np.divide(M, -reg, out=K) np.exp(K, out=K) diff --git a/ot/utils.py b/ot/utils.py index 14cc805..bb21b38 100644 --- a/ot/utils.py +++ b/ot/utils.py @@ -76,6 +76,7 @@ def clean_zeros(a, b, M): b2 = b[b > 0] return a2, b2, M2 + def euclidean_distances(X, Y, squared=False): """ Considering the rows of X (and Y=X) as vectors, compute the -- cgit v1.2.3 From c0c959da8e62d57587ed36e8ba359ca095c5b423 Mon Sep 17 00:00:00 2001 From: Rémi Flamary Date: Tue, 24 Jul 2018 13:55:55 +0200 Subject: speedup einsum constraint violation --- ot/bregman.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) (limited to 'ot/bregman.py') diff --git a/ot/bregman.py b/ot/bregman.py index c8e69ce..26b7b53 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -350,7 +350,6 @@ def sinkhorn_knopp(a, b, M, reg, numItermax=1000, np.exp(K, out=K) # print(np.min(K)) - tmp = np.empty(K.shape, dtype=M.dtype) tmp2 = np.empty(b.shape, dtype=M.dtype) Kp = (1 / a).reshape(-1, 1) * K @@ -379,11 +378,9 @@ def sinkhorn_knopp(a, b, M, reg, numItermax=1000, err = np.sum((u - uprev)**2) / np.sum((u)**2) + \ np.sum((v - vprev)**2) / np.sum((v)**2) else: - np.multiply(u.reshape(-1, 1), K, out=tmp) - np.multiply(tmp, v.reshape(1, -1), out=tmp) - np.sum(tmp, axis=0, out=tmp2) - tmp2 -= b - err = np.linalg.norm(tmp2)**2 + # compute right marginal tmp2= (diag(u)Kdiag(v))^T1 + np.einsum('i,ij,j->j',u,K,v,out=tmp2) + err = np.linalg.norm(tmp2-b)**2 # violation of marginal if log: log['err'].append(err) -- cgit v1.2.3 From 66816cba7cd666706d054bddded1da6035e78c2a Mon Sep 17 00:00:00 2001 From: Rémi Flamary Date: Tue, 24 Jul 2018 14:13:23 +0200 Subject: pep8 all the way --- ot/bregman.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'ot/bregman.py') diff --git a/ot/bregman.py b/ot/bregman.py index 26b7b53..ab84bcf 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -378,9 +378,9 @@ def sinkhorn_knopp(a, b, M, reg, numItermax=1000, err = np.sum((u - uprev)**2) / np.sum((u)**2) + \ np.sum((v - vprev)**2) / np.sum((v)**2) else: - # compute right marginal tmp2= (diag(u)Kdiag(v))^T1 - np.einsum('i,ij,j->j',u,K,v,out=tmp2) - err = np.linalg.norm(tmp2-b)**2 # violation of marginal + # compute right marginal tmp2= (diag(u)Kdiag(v))^T1 + np.einsum('i,ij,j->j', u, K, v, out=tmp2) + err = np.linalg.norm(tmp2 - b)**2 # violation of marginal if log: log['err'].append(err) -- cgit v1.2.3 From bbe411775b3d5abb5d6fb525262cccce3f73d345 Mon Sep 17 00:00:00 2001 From: Rémi Flamary Date: Tue, 24 Jul 2018 14:31:45 +0200 Subject: test eisum instead of dot --- ot/bregman.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'ot/bregman.py') diff --git a/ot/bregman.py b/ot/bregman.py index ab84bcf..d2ade46 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -358,9 +358,9 @@ def sinkhorn_knopp(a, b, M, reg, numItermax=1000, while (err > stopThr and cpt < numItermax): uprev = u vprev = v - KtransposeU = np.dot(K.T, u) + KtransposeU = np.einsum('ij,i->j',K,u)#np.dot(K.T, u) v = np.divide(b, KtransposeU) - u = 1. / np.dot(Kp, v) + u = 1. / np.einsum('ij,j->i',Kp,v)#np.dot(Kp, v) if (np.any(KtransposeU == 0) or np.any(np.isnan(u)) or np.any(np.isnan(v)) or -- cgit v1.2.3 From a04112c69a62182c061d4b65e71ebb43c866d3e1 Mon Sep 17 00:00:00 2001 From: Rémi Flamary Date: Tue, 24 Jul 2018 14:33:32 +0200 Subject: correction size --- ot/bregman.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) (limited to 'ot/bregman.py') diff --git a/ot/bregman.py b/ot/bregman.py index d2ade46..29ca9fd 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -358,9 +358,14 @@ def sinkhorn_knopp(a, b, M, reg, numItermax=1000, while (err > stopThr and cpt < numItermax): uprev = u vprev = v - KtransposeU = np.einsum('ij,i->j',K,u)#np.dot(K.T, u) - v = np.divide(b, KtransposeU) - u = 1. / np.einsum('ij,j->i',Kp,v)#np.dot(Kp, v) + if nbb: + KtransposeU = np.einsum('ij,i,k->jk',K,u)#np.dot(K.T, u) + v = np.divide(b, KtransposeU) + u = 1. / np.einsum('ij,jk->ik',Kp,v)#np.dot(Kp, v) + else: + KtransposeU = np.einsum('ij,i->j',K,u)#np.dot(K.T, u) + v = np.divide(b, KtransposeU) + u = 1. / np.einsum('ij,j->i',Kp,v)#np.dot(Kp, v) if (np.any(KtransposeU == 0) or np.any(np.isnan(u)) or np.any(np.isnan(v)) or -- cgit v1.2.3 From 603c0eee29db890b0092ea8c848473bf413e186f Mon Sep 17 00:00:00 2001 From: Rémi Flamary Date: Tue, 24 Jul 2018 14:34:17 +0200 Subject: pb index --- ot/bregman.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'ot/bregman.py') diff --git a/ot/bregman.py b/ot/bregman.py index 29ca9fd..57cedb2 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -359,7 +359,7 @@ def sinkhorn_knopp(a, b, M, reg, numItermax=1000, uprev = u vprev = v if nbb: - KtransposeU = np.einsum('ij,i,k->jk',K,u)#np.dot(K.T, u) + KtransposeU = np.einsum('ij,ik->jk',K,u)#np.dot(K.T, u) v = np.divide(b, KtransposeU) u = 1. / np.einsum('ij,jk->ik',Kp,v)#np.dot(Kp, v) else: -- cgit v1.2.3 From 5e3392a029e675c7e19f8b1723fcfdb9aa9142aa Mon Sep 17 00:00:00 2001 From: Rémi Flamary Date: Tue, 24 Jul 2018 14:35:58 +0200 Subject: cancel einsum --- ot/bregman.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) (limited to 'ot/bregman.py') diff --git a/ot/bregman.py b/ot/bregman.py index 57cedb2..1873c46 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -358,14 +358,11 @@ def sinkhorn_knopp(a, b, M, reg, numItermax=1000, while (err > stopThr and cpt < numItermax): uprev = u vprev = v - if nbb: - KtransposeU = np.einsum('ij,ik->jk',K,u)#np.dot(K.T, u) - v = np.divide(b, KtransposeU) - u = 1. / np.einsum('ij,jk->ik',Kp,v)#np.dot(Kp, v) - else: - KtransposeU = np.einsum('ij,i->j',K,u)#np.dot(K.T, u) - v = np.divide(b, KtransposeU) - u = 1. / np.einsum('ij,j->i',Kp,v)#np.dot(Kp, v) + + KtransposeU = np.dot(K.T, u) + v = np.divide(b, KtransposeU) + u = 1. / np.dot(Kp, v) + if (np.any(KtransposeU == 0) or np.any(np.isnan(u)) or np.any(np.isnan(v)) or -- cgit v1.2.3 From ace77962d2ae6407916ee7e4377f5c7ed0a8d8f2 Mon Sep 17 00:00:00 2001 From: Rémi Flamary Date: Tue, 24 Jul 2018 14:40:42 +0200 Subject: final makefile bench --- ot/bregman.py | 1 - 1 file changed, 1 deletion(-) (limited to 'ot/bregman.py') diff --git a/ot/bregman.py b/ot/bregman.py index 1873c46..58e74de 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -362,7 +362,6 @@ def sinkhorn_knopp(a, b, M, reg, numItermax=1000, KtransposeU = np.dot(K.T, u) v = np.divide(b, KtransposeU) u = 1. / np.dot(Kp, v) - if (np.any(KtransposeU == 0) or np.any(np.isnan(u)) or np.any(np.isnan(v)) or -- cgit v1.2.3 From f4bfeb73da098384aa67599e7f729fb683a1bcc9 Mon Sep 17 00:00:00 2001 From: Rémi Flamary Date: Tue, 24 Jul 2018 15:54:56 +0200 Subject: ensum tets marginals sinkhorn --- ot/bregman.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) (limited to 'ot/bregman.py') diff --git a/ot/bregman.py b/ot/bregman.py index 58e74de..c755f51 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -396,10 +396,7 @@ def sinkhorn_knopp(a, b, M, reg, numItermax=1000, log['v'] = v if nbb: # return only loss - res = np.zeros((nbb)) - for i in range(nbb): - res[i] = np.sum( - u[:, i].reshape((-1, 1)) * K * v[:, i].reshape((1, -1)) * M) + res = np.einsum('ik,ij,jk,ij->k', u, K, v, M) if log: return res, log else: -- cgit v1.2.3 From d99abf078537acf6cf49480b9790a9c450889031 Mon Sep 17 00:00:00 2001 From: Nicolas Courty Date: Fri, 7 Sep 2018 11:58:42 +0200 Subject: Wasserstein convolutional barycenter --- README.md | 4 +- data/duck.png | Bin 0 -> 5112 bytes data/heart.png | Bin 0 -> 5225 bytes data/redcross.png | Bin 0 -> 1683 bytes data/tooth.png | Bin 0 -> 4931 bytes examples/plot_convolutional_barycenter.py | 92 ++++++++++++++++++++++++++ ot/bregman.py | 106 ++++++++++++++++++++++++++++++ 7 files changed, 201 insertions(+), 1 deletion(-) create mode 100644 data/duck.png create mode 100644 data/heart.png create mode 100644 data/redcross.png create mode 100644 data/tooth.png create mode 100644 examples/plot_convolutional_barycenter.py (limited to 'ot/bregman.py') diff --git a/README.md b/README.md index dded582..1105362 100644 --- a/README.md +++ b/README.md @@ -227,4 +227,6 @@ You can also post bug reports and feature requests in Github issues. Make sure t [19] Seguy, V., Bhushan Damodaran, B., Flamary, R., Courty, N., Rolet, A.& Blondel, M. [Large-scale Optimal Transport and Mapping Estimation](https://arxiv.org/pdf/1711.02283.pdf). International Conference on Learning Representation (2018) -[20] Cuturi, M. and Doucet, A. (2014) [Fast Computation of Wasserstein Barycenters](http://proceedings.mlr.press/v32/cuturi14.html). International Conference in Machine Learning \ No newline at end of file +[20] Cuturi, M. and Doucet, A. (2014) [Fast Computation of Wasserstein Barycenters](http://proceedings.mlr.press/v32/cuturi14.html). International Conference in Machine Learning + +[21] Solomon, J., De Goes, F., Peyré, G., Cuturi, M., Butscher, A., Nguyen, A. & Guibas, L. (2015). [Convolutional wasserstein distances: Efficient optimal transportation on geometric domains](https://dl.acm.org/citation.cfm?id=2766963). ACM Transactions on Graphics (TOG), 34(4), 66. \ No newline at end of file diff --git a/data/duck.png b/data/duck.png new file mode 100644 index 0000000..9181697 Binary files /dev/null and b/data/duck.png differ diff --git a/data/heart.png b/data/heart.png new file mode 100644 index 0000000..44a6385 Binary files /dev/null and b/data/heart.png differ diff --git a/data/redcross.png b/data/redcross.png new file mode 100644 index 0000000..8d0a6fa Binary files /dev/null and b/data/redcross.png differ diff --git a/data/tooth.png b/data/tooth.png new file mode 100644 index 0000000..cd92c9d Binary files /dev/null and b/data/tooth.png differ diff --git a/examples/plot_convolutional_barycenter.py b/examples/plot_convolutional_barycenter.py new file mode 100644 index 0000000..d231da9 --- /dev/null +++ b/examples/plot_convolutional_barycenter.py @@ -0,0 +1,92 @@ + +#%% +# -*- coding: utf-8 -*- +""" +============================================ +Convolutional Wasserstein Barycenter example +============================================ + +This example is designed to illustrate how the Convolutional Wasserstein Barycenter +function of POT works. +""" + +# Author: Nicolas Courty +# +# License: MIT License + + +import numpy as np +import pylab as pl +import ot + +############################################################################## +# Data preparation +# ---------------- +# +# The four distributions are constructed from 4 simple images + + +f1 = 1 - pl.imread('../data/redcross.png')[:, :, 2] +f2 = 1 - pl.imread('../data/duck.png')[:, :, 2] +f3 = 1 - pl.imread('../data/heart.png')[:, :, 2] +f4 = 1 - pl.imread('../data/tooth.png')[:, :, 2] + +A = [] +f1=f1/np.sum(f1) +f2=f2/np.sum(f2) +f3=f3/np.sum(f3) +f4=f4/np.sum(f4) +A.append(f1) +A.append(f2) +A.append(f3) +A.append(f4) +A=np.array(A) + +nb_images = 5 + +# those are the four corners coordinates that will be interpolated by bilinear +# interpolation +v1=np.array((1,0,0,0)) +v2=np.array((0,1,0,0)) +v3=np.array((0,0,1,0)) +v4=np.array((0,0,0,1)) + + +############################################################################## +# Barycenter computation and visualization +# ---------------------------------------- +# + +pl.figure(figsize=(10,10)) +pl.title('Convolutional Wasserstein Barycenters in POT') +cm='Blues' +# regularization parameter +reg=0.004 +for i in range(nb_images): + for j in range(nb_images): + pl.subplot(nb_images,nb_images,i*nb_images+j+1) + tx=float(i)/(nb_images-1) + ty=float(j)/(nb_images-1) + + # weights are constructed by bilinear interpolation + tmp1=(1-tx)*v1+tx*v2 + tmp2=(1-tx)*v3+tx*v4 + weights=(1-ty)*tmp1+ty*tmp2 + + if i==0 and j==0: + pl.imshow(f1,cmap=cm) + pl.axis('off') + elif i==0 and j==(nb_images-1): + pl.imshow(f3,cmap=cm) + pl.axis('off') + elif i==(nb_images-1) and j==0: + pl.imshow(f2,cmap=cm) + pl.axis('off') + elif i==(nb_images-1) and j==(nb_images-1): + pl.imshow(f4,cmap=cm) + pl.axis('off') + else: + # call to barycenter computation + pl.imshow(ot.convolutional_barycenter2d(A,reg,weights),cmap=cm) + pl.axis('off') +pl.show() \ No newline at end of file diff --git a/ot/bregman.py b/ot/bregman.py index c755f51..05f4d9d 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -918,6 +918,112 @@ def barycenter(A, M, reg, weights=None, numItermax=1000, else: return geometricBar(weights, UKv) +def convolutional_barycenter2d(A,reg,weights=None,numItermax = 10000, stopThr=1e-9, verbose=False, log=False): + """Compute the entropic regularized wasserstein barycenter of distributions A + where A is a collection of 2D images. + + The function solves the following optimization problem: + + .. math:: + \mathbf{a} = arg\min_\mathbf{a} \sum_i W_{reg}(\mathbf{a},\mathbf{a}_i) + + where : + + - :math:`W_{reg}(\cdot,\cdot)` is the entropic regularized Wasserstein distance (see ot.bregman.sinkhorn) + - :math:`\mathbf{a}_i` are training distributions (2D images) in the mast two dimensions of matrix :math:`\mathbf{A}` + - reg is the regularization strength scalar value + + The algorithm used for solving the problem is the Sinkhorn-Knopp matrix scaling algorithm as proposed in [21]_ + + Parameters + ---------- + A : np.ndarray (n,w,h) + n distributions (2D images) of size w x h + reg : float + Regularization term >0 + weights : np.ndarray (n,) + Weights of each image on the simplex (barycentric coodinates) + numItermax : int, optional + Max number of iterations + stopThr : float, optional + Stop threshol on error (>0) + verbose : bool, optional + Print information along iterations + log : bool, optional + record log if True + + + Returns + ------- + a : (w,h) ndarray + 2D Wasserstein barycenter + log : dict + log dictionary return only if log==True in parameters + + + References + ---------- + + .. [21] Solomon, J., De Goes, F., Peyré, G., Cuturi, M., Butscher, A., Nguyen, A. & Guibas, L. (2015). + Convolutional wasserstein distances: Efficient optimal transportation on geometric domains + ACM Transactions on Graphics (TOG), 34(4), 66 + + + """ + + if weights is None: + weights = np.ones(A.shape[0]) / A.shape[0] + else: + assert(len(weights) == A.shape[0]) + + if log: + log = {'err': []} + + b=np.zeros_like(A[0,:,:]) + U=np.ones_like(A) + KV=np.ones_like(A) + threshold = 1e-30 # in order to avoids numerical precision issues + + cpt = 0 + err=1 + + # build the convolution operator + t = np.linspace(0,1,A.shape[1]) + [Y,X] = np.meshgrid(t,t) + xi1 = np.exp(-(X-Y)**2/reg) + K = lambda x: np.dot(np.dot(xi1,x),xi1) + + while (err>stopThr and cpt Date: Fri, 7 Sep 2018 12:04:44 +0200 Subject: pep8 normalization --- examples/plot_convolutional_barycenter.py | 70 +++++++++++++++---------------- ot/bregman.py | 68 +++++++++++++++--------------- 2 files changed, 70 insertions(+), 68 deletions(-) (limited to 'ot/bregman.py') diff --git a/examples/plot_convolutional_barycenter.py b/examples/plot_convolutional_barycenter.py index d231da9..7ccdbe3 100644 --- a/examples/plot_convolutional_barycenter.py +++ b/examples/plot_convolutional_barycenter.py @@ -1,4 +1,4 @@ - + #%% # -*- coding: utf-8 -*- """ @@ -32,24 +32,24 @@ f3 = 1 - pl.imread('../data/heart.png')[:, :, 2] f4 = 1 - pl.imread('../data/tooth.png')[:, :, 2] A = [] -f1=f1/np.sum(f1) -f2=f2/np.sum(f2) -f3=f3/np.sum(f3) -f4=f4/np.sum(f4) +f1 = f1 / np.sum(f1) +f2 = f2 / np.sum(f2) +f3 = f3 / np.sum(f3) +f4 = f4 / np.sum(f4) A.append(f1) A.append(f2) A.append(f3) A.append(f4) -A=np.array(A) +A = np.array(A) nb_images = 5 # those are the four corners coordinates that will be interpolated by bilinear # interpolation -v1=np.array((1,0,0,0)) -v2=np.array((0,1,0,0)) -v3=np.array((0,0,1,0)) -v4=np.array((0,0,0,1)) +v1 = np.array((1, 0, 0, 0)) +v2 = np.array((0, 1, 0, 0)) +v3 = np.array((0, 0, 1, 0)) +v4 = np.array((0, 0, 0, 1)) ############################################################################## @@ -57,36 +57,36 @@ v4=np.array((0,0,0,1)) # ---------------------------------------- # -pl.figure(figsize=(10,10)) +pl.figure(figsize=(10, 10)) pl.title('Convolutional Wasserstein Barycenters in POT') -cm='Blues' +cm = 'Blues' # regularization parameter -reg=0.004 +reg = 0.004 for i in range(nb_images): for j in range(nb_images): - pl.subplot(nb_images,nb_images,i*nb_images+j+1) - tx=float(i)/(nb_images-1) - ty=float(j)/(nb_images-1) - + pl.subplot(nb_images, nb_images, i * nb_images + j + 1) + tx = float(i) / (nb_images - 1) + ty = float(j) / (nb_images - 1) + # weights are constructed by bilinear interpolation - tmp1=(1-tx)*v1+tx*v2 - tmp2=(1-tx)*v3+tx*v4 - weights=(1-ty)*tmp1+ty*tmp2 - - if i==0 and j==0: - pl.imshow(f1,cmap=cm) - pl.axis('off') - elif i==0 and j==(nb_images-1): - pl.imshow(f3,cmap=cm) - pl.axis('off') - elif i==(nb_images-1) and j==0: - pl.imshow(f2,cmap=cm) - pl.axis('off') - elif i==(nb_images-1) and j==(nb_images-1): - pl.imshow(f4,cmap=cm) - pl.axis('off') + tmp1 = (1 - tx) * v1 + tx * v2 + tmp2 = (1 - tx) * v3 + tx * v4 + weights = (1 - ty) * tmp1 + ty * tmp2 + + if i == 0 and j == 0: + pl.imshow(f1, cmap=cm) + pl.axis('off') + elif i == 0 and j == (nb_images - 1): + pl.imshow(f3, cmap=cm) + pl.axis('off') + elif i == (nb_images - 1) and j == 0: + pl.imshow(f2, cmap=cm) + pl.axis('off') + elif i == (nb_images - 1) and j == (nb_images - 1): + pl.imshow(f4, cmap=cm) + pl.axis('off') else: # call to barycenter computation - pl.imshow(ot.convolutional_barycenter2d(A,reg,weights),cmap=cm) + pl.imshow(ot.convolutional_barycenter2d(A, reg, weights), cmap=cm) pl.axis('off') -pl.show() \ No newline at end of file +pl.show() diff --git a/ot/bregman.py b/ot/bregman.py index 05f4d9d..f844f03 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -918,7 +918,8 @@ def barycenter(A, M, reg, weights=None, numItermax=1000, else: return geometricBar(weights, UKv) -def convolutional_barycenter2d(A,reg,weights=None,numItermax = 10000, stopThr=1e-9, verbose=False, log=False): + +def convolutional_barycenter2d(A, reg, weights=None, numItermax=10000, stopThr=1e-9, verbose=False, log=False): """Compute the entropic regularized wasserstein barycenter of distributions A where A is a collection of 2D images. @@ -979,51 +980,52 @@ def convolutional_barycenter2d(A,reg,weights=None,numItermax = 10000, stopThr=1e if log: log = {'err': []} - b=np.zeros_like(A[0,:,:]) - U=np.ones_like(A) - KV=np.ones_like(A) - threshold = 1e-30 # in order to avoids numerical precision issues + b = np.zeros_like(A[0, :, :]) + U = np.ones_like(A) + KV = np.ones_like(A) + threshold = 1e-30 # in order to avoids numerical precision issues cpt = 0 - err=1 - - # build the convolution operator - t = np.linspace(0,1,A.shape[1]) - [Y,X] = np.meshgrid(t,t) - xi1 = np.exp(-(X-Y)**2/reg) - K = lambda x: np.dot(np.dot(xi1,x),xi1) - - while (err>stopThr and cpt stopThr and cpt < numItermax): + + bold = b + cpt = cpt + 1 + + b = np.zeros_like(A[0, :, :]) for r in range(A.shape[0]): - KV[r,:,:]=K(A[r,:,:]/np.maximum(threshold,K(U[r,:,:]))) - b += weights[r] * np.log(np.maximum(threshold, U[r,:,:]*KV[r,:,:])) + KV[r, :, :] = K(A[r, :, :] / np.maximum(threshold, K(U[r, :, :]))) + b += weights[r] * np.log(np.maximum(threshold, U[r, :, :] * KV[r, :, :])) b = np.exp(b) for r in range(A.shape[0]): - U[r,:,:]=b/np.maximum(threshold,KV[r,:,:]) - - if cpt%10==1: - err=np.sum(np.abs(bold-b)) + U[r, :, :] = b / np.maximum(threshold, KV[r, :, :]) + + if cpt % 10 == 1: + err = np.sum(np.abs(bold - b)) # log and verbose print if log: log['err'].append(err) if verbose: - if cpt%200 ==0: - print('{:5s}|{:12s}'.format('It.','Err')+'\n'+'-'*19) - print('{:5d}|{:8e}|'.format(cpt,err)) + if cpt % 200 == 0: + print('{:5s}|{:12s}'.format('It.', 'Err') + '\n' + '-' * 19) + print('{:5d}|{:8e}|'.format(cpt, err)) if log: - log['niter']=cpt - log['U']=U - return b,log + log['niter'] = cpt + log['U'] = U + return b, log else: - return b - + return b + def unmix(a, D, M, M0, h0, reg, reg0, alpha, numItermax=1000, stopThr=1e-3, verbose=False, log=False): -- cgit v1.2.3 From e8c6d2fc9c6b08bbed11628326711ab29c155bac Mon Sep 17 00:00:00 2001 From: Nicolas Courty Date: Fri, 7 Sep 2018 12:34:14 +0200 Subject: pep8 fixed (contd) --- ot/bregman.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) (limited to 'ot/bregman.py') diff --git a/ot/bregman.py b/ot/bregman.py index f844f03..5327dbc 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -920,8 +920,8 @@ def barycenter(A, M, reg, weights=None, numItermax=1000, def convolutional_barycenter2d(A, reg, weights=None, numItermax=10000, stopThr=1e-9, verbose=False, log=False): - """Compute the entropic regularized wasserstein barycenter of distributions A - where A is a collection of 2D images. + """Compute the entropic regularized wasserstein barycenter of distributions A + where A is a collection of 2D images. The function solves the following optimization problem: @@ -966,8 +966,8 @@ def convolutional_barycenter2d(A, reg, weights=None, numItermax=10000, stopThr=1 ---------- .. [21] Solomon, J., De Goes, F., Peyré, G., Cuturi, M., Butscher, A., Nguyen, A. & Guibas, L. (2015). - Convolutional wasserstein distances: Efficient optimal transportation on geometric domains - ACM Transactions on Graphics (TOG), 34(4), 66 + Convolutional wasserstein distances: Efficient optimal transportation on geometric domains + ACM Transactions on Graphics (TOG), 34(4), 66 """ @@ -993,7 +993,8 @@ def convolutional_barycenter2d(A, reg, weights=None, numItermax=10000, stopThr=1 [Y, X] = np.meshgrid(t, t) xi1 = np.exp(-(X - Y)**2 / reg) - def K(x): return np.dot(np.dot(xi1, x), xi1) + def K(x): + return np.dot(np.dot(xi1, x), xi1) while (err > stopThr and cpt < numItermax): -- cgit v1.2.3 From d19295b9cb29d21e09eeb28ac4b0e61990727023 Mon Sep 17 00:00:00 2001 From: Nicolas Courty Date: Fri, 7 Sep 2018 14:41:00 +0200 Subject: stabThr and pep8 --- ot/bregman.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) (limited to 'ot/bregman.py') diff --git a/ot/bregman.py b/ot/bregman.py index 5327dbc..748ac30 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -919,7 +919,7 @@ def barycenter(A, M, reg, weights=None, numItermax=1000, return geometricBar(weights, UKv) -def convolutional_barycenter2d(A, reg, weights=None, numItermax=10000, stopThr=1e-9, verbose=False, log=False): +def convolutional_barycenter2d(A, reg, weights=None, numItermax=10000, stopThr=1e-9, stabThr=1e-30, verbose=False, log=False): """Compute the entropic regularized wasserstein barycenter of distributions A where A is a collection of 2D images. @@ -948,6 +948,8 @@ def convolutional_barycenter2d(A, reg, weights=None, numItermax=10000, stopThr=1 Max number of iterations stopThr : float, optional Stop threshol on error (>0) + stabThr : float, optional + Stabilization threshold to avoid numerical precision issue verbose : bool, optional Print information along iterations log : bool, optional @@ -983,7 +985,6 @@ def convolutional_barycenter2d(A, reg, weights=None, numItermax=10000, stopThr=1 b = np.zeros_like(A[0, :, :]) U = np.ones_like(A) KV = np.ones_like(A) - threshold = 1e-30 # in order to avoids numerical precision issues cpt = 0 err = 1 @@ -993,7 +994,7 @@ def convolutional_barycenter2d(A, reg, weights=None, numItermax=10000, stopThr=1 [Y, X] = np.meshgrid(t, t) xi1 = np.exp(-(X - Y)**2 / reg) - def K(x): + def K(x): return np.dot(np.dot(xi1, x), xi1) while (err > stopThr and cpt < numItermax): @@ -1003,11 +1004,11 @@ def convolutional_barycenter2d(A, reg, weights=None, numItermax=10000, stopThr=1 b = np.zeros_like(A[0, :, :]) for r in range(A.shape[0]): - KV[r, :, :] = K(A[r, :, :] / np.maximum(threshold, K(U[r, :, :]))) - b += weights[r] * np.log(np.maximum(threshold, U[r, :, :] * KV[r, :, :])) + KV[r, :, :] = K(A[r, :, :] / np.maximum(stabThr, K(U[r, :, :]))) + b += weights[r] * np.log(np.maximum(stabThr, U[r, :, :] * KV[r, :, :])) b = np.exp(b) for r in range(A.shape[0]): - U[r, :, :] = b / np.maximum(threshold, KV[r, :, :]) + U[r, :, :] = b / np.maximum(stabThr, KV[r, :, :]) if cpt % 10 == 1: err = np.sum(np.abs(bold - b)) -- cgit v1.2.3 From dab572396be97fcf5439e4e20f887165b1ade62c Mon Sep 17 00:00:00 2001 From: Nicolas Courty Date: Fri, 7 Sep 2018 14:49:20 +0200 Subject: whitetrail pep8 --- ot/bregman.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'ot/bregman.py') diff --git a/ot/bregman.py b/ot/bregman.py index 748ac30..35e51f8 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -920,8 +920,8 @@ def barycenter(A, M, reg, weights=None, numItermax=1000, def convolutional_barycenter2d(A, reg, weights=None, numItermax=10000, stopThr=1e-9, stabThr=1e-30, verbose=False, log=False): - """Compute the entropic regularized wasserstein barycenter of distributions A - where A is a collection of 2D images. + """Compute the entropic regularized wasserstein barycenter of distributions A + where A is a collection of 2D images. The function solves the following optimization problem: @@ -949,7 +949,7 @@ def convolutional_barycenter2d(A, reg, weights=None, numItermax=10000, stopThr=1 stopThr : float, optional Stop threshol on error (>0) stabThr : float, optional - Stabilization threshold to avoid numerical precision issue + Stabilization threshold to avoid numerical precision issue verbose : bool, optional Print information along iterations log : bool, optional @@ -967,9 +967,9 @@ def convolutional_barycenter2d(A, reg, weights=None, numItermax=10000, stopThr=1 References ---------- - .. [21] Solomon, J., De Goes, F., Peyré, G., Cuturi, M., Butscher, A., Nguyen, A. & Guibas, L. (2015). - Convolutional wasserstein distances: Efficient optimal transportation on geometric domains - ACM Transactions on Graphics (TOG), 34(4), 66 + .. [21] Solomon, J., De Goes, F., Peyré, G., Cuturi, M., Butscher, A., Nguyen, A. & Guibas, L. (2015). + Convolutional wasserstein distances: Efficient optimal transportation on geometric domains + ACM Transactions on Graphics (TOG), 34(4), 66 """ -- cgit v1.2.3 From 653fd0084c529bc74dabf93c68a9bdd5ac8f377a Mon Sep 17 00:00:00 2001 From: alain Date: Mon, 24 Sep 2018 09:05:47 +0200 Subject: adding greenkhorn --- ot/bregman.py | 151 ++++++++++++++++++++++++++++++++++++++++++++++++++- test/test_bregman.py | 4 +- 2 files changed, 153 insertions(+), 2 deletions(-) (limited to 'ot/bregman.py') diff --git a/ot/bregman.py b/ot/bregman.py index c755f51..1f9874e 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -47,7 +47,7 @@ def sinkhorn(a, b, M, reg, method='sinkhorn', numItermax=1000, reg : float Regularization term >0 method : str - method used for the solver either 'sinkhorn', 'sinkhorn_stabilized' or + method used for the solver either 'sinkhorn', 'greenkhorn', 'sinkhorn_stabilized' or 'sinkhorn_epsilon_scaling', see those function for specific parameters numItermax : int, optional Max number of iterations @@ -103,6 +103,10 @@ def sinkhorn(a, b, M, reg, method='sinkhorn', numItermax=1000, def sink(): return sinkhorn_knopp(a, b, M, reg, numItermax=numItermax, stopThr=stopThr, verbose=verbose, log=log, **kwargs) + if method.lower() == 'greenkhorn': + def sink(): + return greenkhorn(a, b, M, reg, numItermax=numItermax, + stopThr=stopThr, verbose=verbose, log=log) elif method.lower() == 'sinkhorn_stabilized': def sink(): return sinkhorn_stabilized(a, b, M, reg, numItermax=numItermax, @@ -197,6 +201,8 @@ def sinkhorn2(a, b, M, reg, method='sinkhorn', numItermax=1000, .. [10] Chizat, L., Peyré, G., Schmitzer, B., & Vialard, F. X. (2016). Scaling algorithms for unbalanced transport problems. arXiv preprint arXiv:1607.05816. + [21] Altschuler J., Weed J., Rigollet P. : Near-linear time approximation algorithms for optimal transport via Sinkhorn iteration, Advances in Neural Information Processing Systems (NIPS) 31, 2017 + See Also @@ -204,6 +210,7 @@ def sinkhorn2(a, b, M, reg, method='sinkhorn', numItermax=1000, ot.lp.emd : Unregularized OT ot.optim.cg : General regularized OT ot.bregman.sinkhorn_knopp : Classic Sinkhorn [2] + ot.bregman.greenkhorn : Greenkhorn [21] ot.bregman.sinkhorn_stabilized: Stabilized sinkhorn [9][10] ot.bregman.sinkhorn_epsilon_scaling: Sinkhorn with epslilon scaling [9][10] @@ -410,6 +417,148 @@ def sinkhorn_knopp(a, b, M, reg, numItermax=1000, return u.reshape((-1, 1)) * K * v.reshape((1, -1)) + +def greenkhorn(a, b, M, reg, numItermax=10000, stopThr=1e-9, verbose=False, log = False): + """ + Solve the entropic regularization optimal transport problem and return the OT matrix + + The algorithm used is based on the paper + + Near-linear time approximation algorithms for optimal transport via Sinkhorn iteration + by Jason Altschuler, Jonathan Weed, Philippe Rigollet + appeared at NIPS 2017 + + which is a stochastic version of the Sinkhorn-Knopp algorithm [2]. + + The function solves the following optimization problem: + + .. math:: + \gamma = arg\min_\gamma <\gamma,M>_F + reg\cdot\Omega(\gamma) + + s.t. \gamma 1 = a + + \gamma^T 1= b + + \gamma\geq 0 + where : + + - M is the (ns,nt) metric cost matrix + - :math:`\Omega` is the entropic regularization term :math:`\Omega(\gamma)=\sum_{i,j} \gamma_{i,j}\log(\gamma_{i,j})` + - a and b are source and target weights (sum to 1) + + + + Parameters + ---------- + a : np.ndarray (ns,) + samples weights in the source domain + b : np.ndarray (nt,) or np.ndarray (nt,nbb) + samples in the target domain, compute sinkhorn with multiple targets + and fixed M if b is a matrix (return OT loss + dual variables in log) + M : np.ndarray (ns,nt) + loss matrix + reg : float + Regularization term >0 + numItermax : int, optional + Max number of iterations + stopThr : float, optional + Stop threshol on error (>0) + log : bool, optional + record log if True + + + Returns + ------- + gamma : (ns x nt) ndarray + Optimal transportation matrix for the given parameters + log : dict + log dictionary return only if log==True in parameters + + Examples + -------- + + >>> import ot + >>> a=[.5,.5] + >>> b=[.5,.5] + >>> M=[[0.,1.],[1.,0.]] + >>> ot.sinkhorn(a,b,M,1) + array([[ 0.36552929, 0.13447071], + [ 0.13447071, 0.36552929]]) + + + References + ---------- + + .. [2] M. Cuturi, Sinkhorn Distances : Lightspeed Computation of Optimal Transport, Advances in Neural Information Processing Systems (NIPS) 26, 2013 + [21] J. Altschuler, J.Weed, P. Rigollet : Near-linear time approximation algorithms for optimal transport via Sinkhorn iteration, Advances in Neural Information Processing Systems (NIPS) 31, 2017 + + + See Also + -------- + ot.lp.emd : Unregularized OT + ot.optim.cg : General regularized OT + + """ + + i = 0 + + n = a.shape[0] + m = b.shape[0] + + # Next 3 lines equivalent to K= np.exp(-M/reg), but faster to compute + K = np.empty(M.shape, dtype=M.dtype) + np.divide(M, -reg, out=K) + np.exp(K, out=K) + + u = np.ones(n)/n + v = np.ones(m)/m + G = np.diag(u)@K@np.diag(v) + + one_n = np.ones(n) + one_m = np.ones(m) + viol = G@one_m - a + viol_2 = G.T@one_n - b + stopThr_val = 1 + if log: + log['u'] = u + log['v'] = v + + while i < numItermax and stopThr_val > stopThr: + i +=1 + i_1 = np.argmax(np.abs(viol)) + i_2 = np.argmax(np.abs(viol_2)) + m_viol_1 = np.abs(viol[i_1]) + m_viol_2 = np.abs(viol_2[i_2]) + stopThr_val = np.maximum(m_viol_1,m_viol_2) + + if m_viol_1 > m_viol_2: + old_u = u[i_1] + u[i_1] = a[i_1]/(K[i_1,:]@v) + G[i_1,:] = u[i_1]*K[i_1,:]*v + + viol[i_1] = u[i_1]*K[i_1,:]@v - a[i_1] + viol_2 = viol_2 + ( K[i_1,:].T*(u[i_1] - old_u)*v) + + else: + old_v = v[i_2] + v[i_2] = b[i_2]/(K[:,i_2].T@u) + G[:,i_2] = u*K[:,i_2]*v[i_2] + #aviol = (G@one_m - a) + #aviol_2 = (G.T@one_n - b) + viol = viol + ( -old_v + v[i_2])*K[:,i_2]*u + viol_2[i_2] = v[i_2]*K[:,i_2]@u - b[i_2] + + #print('b',np.max(abs(aviol -viol)),np.max(abs(aviol_2 - viol_2))) + + if log: + log['u'] = u + log['v'] = v + + if log: + return G,log + else: + return G + def sinkhorn_stabilized(a, b, M, reg, numItermax=1000, tau=1e3, stopThr=1e-9, warmstart=None, verbose=False, print_period=20, log=False, **kwargs): """ diff --git a/test/test_bregman.py b/test/test_bregman.py index c8e9179..03e38bf 100644 --- a/test/test_bregman.py +++ b/test/test_bregman.py @@ -71,12 +71,14 @@ def test_sinkhorn_variants(): Ges = ot.sinkhorn( u, u, M, 1, method='sinkhorn_epsilon_scaling', stopThr=1e-10) Gerr = ot.sinkhorn(u, u, M, 1, method='do_not_exists', stopThr=1e-10) + G_green = ot.sinkhorn(u, u, M, 1, method='greenkhorn', stopThr=1e-10) # check values np.testing.assert_allclose(G0, Gs, atol=1e-05) np.testing.assert_allclose(G0, Ges, atol=1e-05) np.testing.assert_allclose(G0, Gerr) - + np.testing.assert_allclose(G0, G_green, atol = 1e-32) + print(G0,G_green) def test_bary(): -- cgit v1.2.3 From eb17e022fee209f3d363a6f8dcbb0064fccde1ad Mon Sep 17 00:00:00 2001 From: Rémi Flamary Date: Mon, 24 Sep 2018 10:07:00 +0200 Subject: correct if error bug --- ot/bregman.py | 58 +++++++++++++++++++++++++++++----------------------------- 1 file changed, 29 insertions(+), 29 deletions(-) (limited to 'ot/bregman.py') diff --git a/ot/bregman.py b/ot/bregman.py index 1f9874e..8538c92 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -103,10 +103,10 @@ def sinkhorn(a, b, M, reg, method='sinkhorn', numItermax=1000, def sink(): return sinkhorn_knopp(a, b, M, reg, numItermax=numItermax, stopThr=stopThr, verbose=verbose, log=log, **kwargs) - if method.lower() == 'greenkhorn': + elif method.lower() == 'greenkhorn': def sink(): return greenkhorn(a, b, M, reg, numItermax=numItermax, - stopThr=stopThr, verbose=verbose, log=log) + stopThr=stopThr, verbose=verbose, log=log) elif method.lower() == 'sinkhorn_stabilized': def sink(): return sinkhorn_stabilized(a, b, M, reg, numItermax=numItermax, @@ -417,17 +417,16 @@ def sinkhorn_knopp(a, b, M, reg, numItermax=1000, return u.reshape((-1, 1)) * K * v.reshape((1, -1)) - -def greenkhorn(a, b, M, reg, numItermax=10000, stopThr=1e-9, verbose=False, log = False): +def greenkhorn(a, b, M, reg, numItermax=10000, stopThr=1e-9, verbose=False, log=False): """ Solve the entropic regularization optimal transport problem and return the OT matrix - + The algorithm used is based on the paper - + Near-linear time approximation algorithms for optimal transport via Sinkhorn iteration by Jason Altschuler, Jonathan Weed, Philippe Rigollet appeared at NIPS 2017 - + which is a stochastic version of the Sinkhorn-Knopp algorithm [2]. The function solves the following optimization problem: @@ -499,21 +498,21 @@ def greenkhorn(a, b, M, reg, numItermax=10000, stopThr=1e-9, verbose=False, log ot.optim.cg : General regularized OT """ - + i = 0 - + n = a.shape[0] m = b.shape[0] - + # Next 3 lines equivalent to K= np.exp(-M/reg), but faster to compute K = np.empty(M.shape, dtype=M.dtype) np.divide(M, -reg, out=K) np.exp(K, out=K) - - u = np.ones(n)/n - v = np.ones(m)/m + + u = np.ones(n) / n + v = np.ones(m) / m G = np.diag(u)@K@np.diag(v) - + one_n = np.ones(n) one_m = np.ones(m) viol = G@one_m - a @@ -524,41 +523,42 @@ def greenkhorn(a, b, M, reg, numItermax=10000, stopThr=1e-9, verbose=False, log log['v'] = v while i < numItermax and stopThr_val > stopThr: - i +=1 + i += 1 i_1 = np.argmax(np.abs(viol)) i_2 = np.argmax(np.abs(viol_2)) m_viol_1 = np.abs(viol[i_1]) m_viol_2 = np.abs(viol_2[i_2]) - stopThr_val = np.maximum(m_viol_1,m_viol_2) - + stopThr_val = np.maximum(m_viol_1, m_viol_2) + if m_viol_1 > m_viol_2: old_u = u[i_1] - u[i_1] = a[i_1]/(K[i_1,:]@v) - G[i_1,:] = u[i_1]*K[i_1,:]*v + u[i_1] = a[i_1] / (K[i_1, :]@v) + G[i_1, :] = u[i_1] * K[i_1, :] * v - viol[i_1] = u[i_1]*K[i_1,:]@v - a[i_1] - viol_2 = viol_2 + ( K[i_1,:].T*(u[i_1] - old_u)*v) + viol[i_1] = u[i_1] * K[i_1, :]@v - a[i_1] + viol_2 = viol_2 + (K[i_1, :].T * (u[i_1] - old_u) * v) else: old_v = v[i_2] - v[i_2] = b[i_2]/(K[:,i_2].T@u) - G[:,i_2] = u*K[:,i_2]*v[i_2] + v[i_2] = b[i_2] / (K[:, i_2].T@u) + G[:, i_2] = u * K[:, i_2] * v[i_2] #aviol = (G@one_m - a) #aviol_2 = (G.T@one_n - b) - viol = viol + ( -old_v + v[i_2])*K[:,i_2]*u - viol_2[i_2] = v[i_2]*K[:,i_2]@u - b[i_2] - + viol = viol + (-old_v + v[i_2]) * K[:, i_2] * u + viol_2[i_2] = v[i_2] * K[:, i_2]@u - b[i_2] + #print('b',np.max(abs(aviol -viol)),np.max(abs(aviol_2 - viol_2))) - + if log: log['u'] = u log['v'] = v - + if log: - return G,log + return G, log else: return G + def sinkhorn_stabilized(a, b, M, reg, numItermax=1000, tau=1e3, stopThr=1e-9, warmstart=None, verbose=False, print_period=20, log=False, **kwargs): """ -- cgit v1.2.3 From 7ffd4fef3260e086b0b1ed050f5cb4b83195b122 Mon Sep 17 00:00:00 2001 From: Rémi Flamary Date: Mon, 24 Sep 2018 10:14:44 +0200 Subject: remove @ for python compatibility+ comments alexandre --- ot/bregman.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) (limited to 'ot/bregman.py') diff --git a/ot/bregman.py b/ot/bregman.py index 8538c92..faa6365 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -480,7 +480,7 @@ def greenkhorn(a, b, M, reg, numItermax=10000, stopThr=1e-9, verbose=False, log= >>> a=[.5,.5] >>> b=[.5,.5] >>> M=[[0.,1.],[1.,0.]] - >>> ot.sinkhorn(a,b,M,1) + >>> ot.bregman.greenkhorn(a,b,M,1) array([[ 0.36552929, 0.13447071], [ 0.13447071, 0.36552929]]) @@ -505,18 +505,18 @@ def greenkhorn(a, b, M, reg, numItermax=10000, stopThr=1e-9, verbose=False, log= m = b.shape[0] # Next 3 lines equivalent to K= np.exp(-M/reg), but faster to compute - K = np.empty(M.shape, dtype=M.dtype) + K = np.empty_like(M) np.divide(M, -reg, out=K) np.exp(K, out=K) - u = np.ones(n) / n - v = np.ones(m) / m - G = np.diag(u)@K@np.diag(v) + u = np.full(n, 1. / n) + v = np.full(m, 1. / m) + G = u[:, np.newaxis] * K * v[np.newaxis, :] one_n = np.ones(n) one_m = np.ones(m) - viol = G@one_m - a - viol_2 = G.T@one_n - b + viol = G.sum(1) - a + viol_2 = G.sum(0) - b stopThr_val = 1 if log: log['u'] = u @@ -532,26 +532,26 @@ def greenkhorn(a, b, M, reg, numItermax=10000, stopThr=1e-9, verbose=False, log= if m_viol_1 > m_viol_2: old_u = u[i_1] - u[i_1] = a[i_1] / (K[i_1, :]@v) + u[i_1] = a[i_1] / (K[i_1, :].dot(v)) G[i_1, :] = u[i_1] * K[i_1, :] * v - viol[i_1] = u[i_1] * K[i_1, :]@v - a[i_1] + viol[i_1] = u[i_1] * K[i_1, :].dot(v) - a[i_1] viol_2 = viol_2 + (K[i_1, :].T * (u[i_1] - old_u) * v) else: old_v = v[i_2] - v[i_2] = b[i_2] / (K[:, i_2].T@u) + v[i_2] = b[i_2] / (K[:, i_2].T.dot(u)) G[:, i_2] = u * K[:, i_2] * v[i_2] #aviol = (G@one_m - a) #aviol_2 = (G.T@one_n - b) viol = viol + (-old_v + v[i_2]) * K[:, i_2] * u - viol_2[i_2] = v[i_2] * K[:, i_2]@u - b[i_2] + viol_2[i_2] = v[i_2] * K[:, i_2].dot(u) - b[i_2] #print('b',np.max(abs(aviol -viol)),np.max(abs(aviol_2 - viol_2))) - if log: - log['u'] = u - log['v'] = v + if log: + log['u'] = u + log['v'] = v if log: return G, log -- cgit v1.2.3 From 24a53ef2dba0a43c282f6b31937c3e7901df7930 Mon Sep 17 00:00:00 2001 From: Rémi Flamary Date: Mon, 24 Sep 2018 10:17:21 +0200 Subject: add contributor --- README.md | 1 + ot/bregman.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) (limited to 'ot/bregman.py') diff --git a/README.md b/README.md index 6a6686c..4d824ce 100644 --- a/README.md +++ b/README.md @@ -165,6 +165,7 @@ The contributors to this library are: * [Antoine Rolet](https://arolet.github.io/) * Erwan Vautier (Gromov-Wasserstein) * [Kilian Fatras](https://kilianfatras.github.io/) +* [Alain Rakotomamonjy](https://sites.google.com/site/alainrakotomamonjy/home) This toolbox benefit a lot from open source research and we would like to thank the following persons for providing some code (in various languages): diff --git a/ot/bregman.py b/ot/bregman.py index faa6365..97027e8 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -536,7 +536,7 @@ def greenkhorn(a, b, M, reg, numItermax=10000, stopThr=1e-9, verbose=False, log= G[i_1, :] = u[i_1] * K[i_1, :] * v viol[i_1] = u[i_1] * K[i_1, :].dot(v) - a[i_1] - viol_2 = viol_2 + (K[i_1, :].T * (u[i_1] - old_u) * v) + viol_2 += (K[i_1, :].T * (u[i_1] - old_u) * v) else: old_v = v[i_2] @@ -544,7 +544,7 @@ def greenkhorn(a, b, M, reg, numItermax=10000, stopThr=1e-9, verbose=False, log= G[:, i_2] = u * K[:, i_2] * v[i_2] #aviol = (G@one_m - a) #aviol_2 = (G.T@one_n - b) - viol = viol + (-old_v + v[i_2]) * K[:, i_2] * u + viol += (-old_v + v[i_2]) * K[:, i_2] * u viol_2[i_2] = v[i_2] * K[:, i_2].dot(u) - b[i_2] #print('b',np.max(abs(aviol -viol)),np.max(abs(aviol_2 - viol_2))) -- cgit v1.2.3 From 55e8392993919d3c67538756663abd943d3bb491 Mon Sep 17 00:00:00 2001 From: Rémi Flamary Date: Mon, 24 Sep 2018 10:19:18 +0200 Subject: remove unused variable --- ot/bregman.py | 2 -- 1 file changed, 2 deletions(-) (limited to 'ot/bregman.py') diff --git a/ot/bregman.py b/ot/bregman.py index 97027e8..6e446a1 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -513,8 +513,6 @@ def greenkhorn(a, b, M, reg, numItermax=10000, stopThr=1e-9, verbose=False, log= v = np.full(m, 1. / m) G = u[:, np.newaxis] * K * v[np.newaxis, :] - one_n = np.ones(n) - one_m = np.ones(m) viol = G.sum(1) - a viol_2 = G.sum(0) - b stopThr_val = 1 -- cgit v1.2.3 From 1d494107611c2e6e2249b7a624e64cec6357b4bd Mon Sep 17 00:00:00 2001 From: Rémi Flamary Date: Mon, 24 Sep 2018 10:23:02 +0200 Subject: implement for loop --- ot/bregman.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'ot/bregman.py') diff --git a/ot/bregman.py b/ot/bregman.py index 6e446a1..05f7c75 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -520,7 +520,7 @@ def greenkhorn(a, b, M, reg, numItermax=10000, stopThr=1e-9, verbose=False, log= log['u'] = u log['v'] = v - while i < numItermax and stopThr_val > stopThr: + for i in range(numItermax): i += 1 i_1 = np.argmax(np.abs(viol)) i_2 = np.argmax(np.abs(viol_2)) @@ -547,6 +547,11 @@ def greenkhorn(a, b, M, reg, numItermax=10000, stopThr=1e-9, verbose=False, log= #print('b',np.max(abs(aviol -viol)),np.max(abs(aviol_2 - viol_2))) + if stopThr_val <= stopThr: + break + else: + print('Warning: Algorithm did not converge') + if log: log['u'] = u log['v'] = v -- cgit v1.2.3 From 75fe96c183852971bb7be1da39af202b9f7d6e6c Mon Sep 17 00:00:00 2001 From: Rémi Flamary Date: Mon, 24 Sep 2018 10:25:25 +0200 Subject: remove i+1 --- ot/bregman.py | 1 - 1 file changed, 1 deletion(-) (limited to 'ot/bregman.py') diff --git a/ot/bregman.py b/ot/bregman.py index 05f7c75..1f5150a 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -521,7 +521,6 @@ def greenkhorn(a, b, M, reg, numItermax=10000, stopThr=1e-9, verbose=False, log= log['v'] = v for i in range(numItermax): - i += 1 i_1 = np.argmax(np.abs(viol)) i_2 = np.argmax(np.abs(viol_2)) m_viol_1 = np.abs(viol[i_1]) -- cgit v1.2.3 From dee6d6e16f6e5d328bc590089cf99ef586d7ca0f Mon Sep 17 00:00:00 2001 From: Rémi Flamary Date: Mon, 24 Sep 2018 10:34:32 +0200 Subject: correct reference number in doc --- README.md | 2 +- ot/bregman.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'ot/bregman.py') diff --git a/README.md b/README.md index 1c8114a..16fa153 100644 --- a/README.md +++ b/README.md @@ -232,4 +232,4 @@ You can also post bug reports and feature requests in Github issues. Make sure t [21] Solomon, J., De Goes, F., Peyré, G., Cuturi, M., Butscher, A., Nguyen, A. & Guibas, L. (2015). [Convolutional wasserstein distances: Efficient optimal transportation on geometric domains](https://dl.acm.org/citation.cfm?id=2766963). ACM Transactions on Graphics (TOG), 34(4), 66. -[21] J. Altschuler, J.Weed, P. Rigollet, (2017) [Near-linear time approximation algorithms for optimal transport via Sinkhorn iteration](https://papers.nips.cc/paper/6792-near-linear-time-approximation-algorithms-for-optimal-transport-via-sinkhorn-iteration.pdf), Advances in Neural Information Processing Systems (NIPS) 31 +[22] J. Altschuler, J.Weed, P. Rigollet, (2017) [Near-linear time approximation algorithms for optimal transport via Sinkhorn iteration](https://papers.nips.cc/paper/6792-near-linear-time-approximation-algorithms-for-optimal-transport-via-sinkhorn-iteration.pdf), Advances in Neural Information Processing Systems (NIPS) 31 diff --git a/ot/bregman.py b/ot/bregman.py index 418de57..fd04fa4 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -489,7 +489,7 @@ def greenkhorn(a, b, M, reg, numItermax=10000, stopThr=1e-9, verbose=False, log= ---------- .. [2] M. Cuturi, Sinkhorn Distances : Lightspeed Computation of Optimal Transport, Advances in Neural Information Processing Systems (NIPS) 26, 2013 - [21] J. Altschuler, J.Weed, P. Rigollet : Near-linear time approximation algorithms for optimal transport via Sinkhorn iteration, Advances in Neural Information Processing Systems (NIPS) 31, 2017 + [22] J. Altschuler, J.Weed, P. Rigollet : Near-linear time approximation algorithms for optimal transport via Sinkhorn iteration, Advances in Neural Information Processing Systems (NIPS) 31, 2017 See Also -- cgit v1.2.3 From 1b24b1fd60a7126cd1646525ac5d7cf25f382a3a Mon Sep 17 00:00:00 2001 From: Rémi Flamary Date: Mon, 24 Sep 2018 10:40:15 +0200 Subject: remove variable i initialization --- ot/bregman.py | 2 -- 1 file changed, 2 deletions(-) (limited to 'ot/bregman.py') diff --git a/ot/bregman.py b/ot/bregman.py index fd04fa4..d1057ff 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -499,8 +499,6 @@ def greenkhorn(a, b, M, reg, numItermax=10000, stopThr=1e-9, verbose=False, log= """ - i = 0 - n = a.shape[0] m = b.shape[0] -- cgit v1.2.3 From 93db239e1156ad1db8edbb13c1ecde973ce009c0 Mon Sep 17 00:00:00 2001 From: Rémi Flamary Date: Mon, 19 Nov 2018 11:17:07 +0100 Subject: remove W605 errors --- ot/bregman.py | 18 +++++++++--------- ot/externals/funcsigs.py | 46 +++++++++++++++++++++++----------------------- ot/gpu/bregman.py | 6 +++--- ot/stochastic.py | 20 ++++++++++---------- setup.cfg | 2 +- 5 files changed, 46 insertions(+), 46 deletions(-) (limited to 'ot/bregman.py') diff --git a/ot/bregman.py b/ot/bregman.py index d1057ff..43340f7 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -370,9 +370,9 @@ def sinkhorn_knopp(a, b, M, reg, numItermax=1000, v = np.divide(b, KtransposeU) u = 1. / np.dot(Kp, v) - if (np.any(KtransposeU == 0) or - np.any(np.isnan(u)) or np.any(np.isnan(v)) or - np.any(np.isinf(u)) or np.any(np.isinf(v))): + if (np.any(KtransposeU == 0) + or np.any(np.isnan(u)) or np.any(np.isnan(v)) + or np.any(np.isinf(u)) or np.any(np.isinf(v))): # we have reached the machine precision # come back to previous solution and quit loop print('Warning: numerical errors at iteration', cpt) @@ -683,13 +683,13 @@ def sinkhorn_stabilized(a, b, M, reg, numItermax=1000, tau=1e3, stopThr=1e-9, def get_K(alpha, beta): """log space computation""" - return np.exp(-(M - alpha.reshape((na, 1)) - - beta.reshape((1, nb))) / reg) + return np.exp(-(M - alpha.reshape((na, 1)) + - beta.reshape((1, nb))) / reg) def get_Gamma(alpha, beta, u, v): """log space gamma computation""" - return np.exp(-(M - alpha.reshape((na, 1)) - beta.reshape((1, nb))) / - reg + np.log(u.reshape((na, 1))) + np.log(v.reshape((1, nb)))) + return np.exp(-(M - alpha.reshape((na, 1)) - beta.reshape((1, nb))) + / reg + np.log(u.reshape((na, 1))) + np.log(v.reshape((1, nb)))) # print(np.min(K)) @@ -899,8 +899,8 @@ def sinkhorn_epsilon_scaling(a, b, M, reg, numItermax=100, epsilon0=1e4, numInne def get_K(alpha, beta): """log space computation""" - return np.exp(-(M - alpha.reshape((na, 1)) - - beta.reshape((1, nb))) / reg) + return np.exp(-(M - alpha.reshape((na, 1)) + - beta.reshape((1, nb))) / reg) # print(np.min(K)) def get_reg(n): # exponential decreasing diff --git a/ot/externals/funcsigs.py b/ot/externals/funcsigs.py index c73fdc9..106bde7 100644 --- a/ot/externals/funcsigs.py +++ b/ot/externals/funcsigs.py @@ -126,8 +126,8 @@ def signature(obj): new_params[arg_name] = param.replace(default=arg_value, _partial_kwarg=True) - elif (param.kind not in (_VAR_KEYWORD, _VAR_POSITIONAL) and - not param._partial_kwarg): + elif (param.kind not in (_VAR_KEYWORD, _VAR_POSITIONAL) + and not param._partial_kwarg): new_params.pop(arg_name) return sig.replace(parameters=new_params.values()) @@ -333,11 +333,11 @@ class Parameter(object): raise TypeError(msg) def __eq__(self, other): - return (issubclass(other.__class__, Parameter) and - self._name == other._name and - self._kind == other._kind and - self._default == other._default and - self._annotation == other._annotation) + return (issubclass(other.__class__, Parameter) + and self._name == other._name + and self._kind == other._kind + and self._default == other._default + and self._annotation == other._annotation) def __ne__(self, other): return not self.__eq__(other) @@ -372,8 +372,8 @@ class BoundArguments(object): def args(self): args = [] for param_name, param in self._signature.parameters.items(): - if (param.kind in (_VAR_KEYWORD, _KEYWORD_ONLY) or - param._partial_kwarg): + if (param.kind in (_VAR_KEYWORD, _KEYWORD_ONLY) + or param._partial_kwarg): # Keyword arguments mapped by 'functools.partial' # (Parameter._partial_kwarg is True) are mapped # in 'BoundArguments.kwargs', along with VAR_KEYWORD & @@ -402,8 +402,8 @@ class BoundArguments(object): kwargs_started = False for param_name, param in self._signature.parameters.items(): if not kwargs_started: - if (param.kind in (_VAR_KEYWORD, _KEYWORD_ONLY) or - param._partial_kwarg): + if (param.kind in (_VAR_KEYWORD, _KEYWORD_ONLY) + or param._partial_kwarg): kwargs_started = True else: if param_name not in self.arguments: @@ -432,9 +432,9 @@ class BoundArguments(object): raise TypeError(msg) def __eq__(self, other): - return (issubclass(other.__class__, BoundArguments) and - self.signature == other.signature and - self.arguments == other.arguments) + return (issubclass(other.__class__, BoundArguments) + and self.signature == other.signature + and self.arguments == other.arguments) def __ne__(self, other): return not self.__eq__(other) @@ -612,9 +612,9 @@ class Signature(object): raise TypeError(msg) def __eq__(self, other): - if (not issubclass(type(other), Signature) or - self.return_annotation != other.return_annotation or - len(self.parameters) != len(other.parameters)): + if (not issubclass(type(other), Signature) + or self.return_annotation != other.return_annotation + or len(self.parameters) != len(other.parameters)): return False other_positions = dict((param, idx) @@ -635,8 +635,8 @@ class Signature(object): except KeyError: return False else: - if (idx != other_idx or - param != other.parameters[param_name]): + if (idx != other_idx + or param != other.parameters[param_name]): return False return True @@ -688,8 +688,8 @@ class Signature(object): raise TypeError(msg) parameters_ex = (param,) break - elif (param.kind == _VAR_KEYWORD or - param.default is not _empty): + elif (param.kind == _VAR_KEYWORD + or param.default is not _empty): # That's fine too - we have a default value for this # parameter. So, lets start parsing `kwargs`, starting # with the current parameter @@ -755,8 +755,8 @@ class Signature(object): # if it has a default value, or it is an '*args'-like # parameter, left alone by the processing of positional # arguments. - if (not partial and param.kind != _VAR_POSITIONAL and - param.default is _empty): + if (not partial and param.kind != _VAR_POSITIONAL + and param.default is _empty): raise TypeError('{arg!r} parameter lacking default value'. format(arg=param_name)) diff --git a/ot/gpu/bregman.py b/ot/gpu/bregman.py index 978b307..3031ed9 100644 --- a/ot/gpu/bregman.py +++ b/ot/gpu/bregman.py @@ -146,9 +146,9 @@ def sinkhorn_knopp(a, b, M, reg, numItermax=1000, stopThr=1e-9, v = np.divide(b, KtransposeU) u = 1. / np.dot(Kp, v) - if (np.any(KtransposeU == 0) or - np.any(np.isnan(u)) or np.any(np.isnan(v)) or - np.any(np.isinf(u)) or np.any(np.isinf(v))): + if (np.any(KtransposeU == 0) + or np.any(np.isnan(u)) or np.any(np.isnan(v)) + or np.any(np.isinf(u)) or np.any(np.isinf(v))): # we have reached the machine precision # come back to previous solution and quit loop print('Warning: numerical errors at iteration', cpt) diff --git a/ot/stochastic.py b/ot/stochastic.py index ec53015..1376884 100644 --- a/ot/stochastic.py +++ b/ot/stochastic.py @@ -418,8 +418,8 @@ def solve_semi_dual_entropic(a, b, M, reg, method, numItermax=10000, lr=None, return None opt_alpha = c_transform_entropic(b, M, reg, opt_beta) - pi = (np.exp((opt_alpha[:, None] + opt_beta[None, :] - M[:, :]) / reg) * - a[:, None] * b[None, :]) + pi = (np.exp((opt_alpha[:, None] + opt_beta[None, :] - M[:, :]) / reg) + * a[:, None] * b[None, :]) if log: log = {} @@ -520,15 +520,15 @@ def batch_grad_dual(a, b, M, reg, alpha, beta, batch_size, batch_alpha, arXiv preprint arxiv:1711.02283. ''' - G = - (np.exp((alpha[batch_alpha, None] + beta[None, batch_beta] - - M[batch_alpha, :][:, batch_beta]) / reg) * + G = - (np.exp((alpha[batch_alpha, None] + beta[None, batch_beta] + - M[batch_alpha, :][:, batch_beta]) / reg) * a[batch_alpha, None] * b[None, batch_beta]) grad_beta = np.zeros(np.shape(M)[1]) grad_alpha = np.zeros(np.shape(M)[0]) - grad_beta[batch_beta] = (b[batch_beta] * len(batch_alpha) / np.shape(M)[0] + - G.sum(0)) - grad_alpha[batch_alpha] = (a[batch_alpha] * len(batch_beta) / - np.shape(M)[1] + G.sum(1)) + grad_beta[batch_beta] = (b[batch_beta] * len(batch_alpha) / np.shape(M)[0] + + G.sum(0)) + grad_alpha[batch_alpha] = (a[batch_alpha] * len(batch_beta) + / np.shape(M)[1] + G.sum(1)) return grad_alpha, grad_beta @@ -702,8 +702,8 @@ def solve_dual_entropic(a, b, M, reg, batch_size, numItermax=10000, lr=1, opt_alpha, opt_beta = sgd_entropic_regularization(a, b, M, reg, batch_size, numItermax, lr) - pi = (np.exp((opt_alpha[:, None] + opt_beta[None, :] - M[:, :]) / reg) * - a[:, None] * b[None, :]) + pi = (np.exp((opt_alpha[:, None] + opt_beta[None, :] - M[:, :]) / reg) + * a[:, None] * b[None, :]) if log: log = {} log['alpha'] = opt_alpha diff --git a/setup.cfg b/setup.cfg index b2a2415..24512d2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -3,4 +3,4 @@ description-file = README.md [flake8] exclude = __init__.py -ignore = E265,E501 +ignore = E265,E501,W605 -- cgit v1.2.3 From 42a501c5d839c010bbfa3a4440b43cb4f9775fc7 Mon Sep 17 00:00:00 2001 From: Rémi Flamary Date: Mon, 11 Mar 2019 10:39:03 +0100 Subject: add test sinkhorn+log --- ot/bregman.py | 14 +++++++++++++- test/test_bregman.py | 25 +++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) (limited to 'ot/bregman.py') diff --git a/ot/bregman.py b/ot/bregman.py index 43340f7..013bc33 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -120,7 +120,8 @@ def sinkhorn(a, b, M, reg, method='sinkhorn', numItermax=1000, print('Warning : unknown method using classic Sinkhorn Knopp') def sink(): - return sinkhorn_knopp(a, b, M, reg, **kwargs) + return sinkhorn_knopp(a, b, M, reg, numItermax=numItermax, + stopThr=stopThr, verbose=verbose, log=log, **kwargs) return sink() @@ -499,6 +500,15 @@ def greenkhorn(a, b, M, reg, numItermax=10000, stopThr=1e-9, verbose=False, log= """ + a = np.asarray(a, dtype=np.float64) + b = np.asarray(b, dtype=np.float64) + M = np.asarray(M, dtype=np.float64) + + if len(a) == 0: + a = np.ones((M.shape[0],), dtype=np.float64) / M.shape[0] + if len(b) == 0: + b = np.ones((M.shape[1],), dtype=np.float64) / M.shape[1] + n = a.shape[0] m = b.shape[0] @@ -514,7 +524,9 @@ def greenkhorn(a, b, M, reg, numItermax=10000, stopThr=1e-9, verbose=False, log= viol = G.sum(1) - a viol_2 = G.sum(0) - b stopThr_val = 1 + if log: + log = dict() log['u'] = u log['v'] = v diff --git a/test/test_bregman.py b/test/test_bregman.py index 14edaf5..90eaf27 100644 --- a/test/test_bregman.py +++ b/test/test_bregman.py @@ -81,6 +81,31 @@ def test_sinkhorn_variants(): print(G0, G_green) +def test_sinkhorn_variants_log(): + # test sinkhorn + n = 100 + rng = np.random.RandomState(0) + + x = rng.randn(n, 2) + u = ot.utils.unif(n) + + M = ot.dist(x, x) + + G0, log0 = ot.sinkhorn(u, u, M, 1, method='sinkhorn', stopThr=1e-10, log=True) + Gs, logs = ot.sinkhorn(u, u, M, 1, method='sinkhorn_stabilized', stopThr=1e-10, log=True) + Ges, loges = ot.sinkhorn( + u, u, M, 1, method='sinkhorn_epsilon_scaling', stopThr=1e-10, log=True) + Gerr, logerr = ot.sinkhorn(u, u, M, 1, method='do_not_exists', stopThr=1e-10, log=True) + G_green, loggreen = ot.sinkhorn(u, u, M, 1, method='greenkhorn', stopThr=1e-10, log=True) + + # check values + np.testing.assert_allclose(G0, Gs, atol=1e-05) + np.testing.assert_allclose(G0, Ges, atol=1e-05) + np.testing.assert_allclose(G0, Gerr) + np.testing.assert_allclose(G0, G_green, atol=1e-5) + print(G0, G_green) + + def test_bary(): n_bins = 100 # nb bins -- cgit v1.2.3 From a2545b5a503c95c9bf07948929b77e9c3f4f28d3 Mon Sep 17 00:00:00 2001 From: Kilian Fatras Date: Fri, 29 Mar 2019 12:41:43 +0100 Subject: add empirical sinkhorn and sikhorn divergence functions --- README.md | 2 + examples/plot_OT_2D_samples.py | 26 ++++ ot/bregman.py | 269 +++++++++++++++++++++++++++++++++++++++++ test/test_bregman.py | 57 +++++++++ 4 files changed, 354 insertions(+) (limited to 'ot/bregman.py') diff --git a/README.md b/README.md index b068131..dbd93fc 100644 --- a/README.md +++ b/README.md @@ -230,3 +230,5 @@ You can also post bug reports and feature requests in Github issues. Make sure t [21] Solomon, J., De Goes, F., Peyré, G., Cuturi, M., Butscher, A., Nguyen, A. & Guibas, L. (2015). [Convolutional wasserstein distances: Efficient optimal transportation on geometric domains](https://dl.acm.org/citation.cfm?id=2766963). ACM Transactions on Graphics (TOG), 34(4), 66. [22] J. Altschuler, J.Weed, P. Rigollet, (2017) [Near-linear time approximation algorithms for optimal transport via Sinkhorn iteration](https://papers.nips.cc/paper/6792-near-linear-time-approximation-algorithms-for-optimal-transport-via-sinkhorn-iteration.pdf), Advances in Neural Information Processing Systems (NIPS) 31 + +[23] Aude, G., Peyré, G., Cuturi, M., [Learning Generative Models with Sinkhorn Divergences](https://arxiv.org/abs/1706.00292), Proceedings of the Twenty-First International Conference on Artficial Intelligence and Statistics, (AISTATS) 21, 2018 diff --git a/examples/plot_OT_2D_samples.py b/examples/plot_OT_2D_samples.py index bb952a0..63126ba 100644 --- a/examples/plot_OT_2D_samples.py +++ b/examples/plot_OT_2D_samples.py @@ -10,6 +10,7 @@ sum of diracs. The OT matrix is plotted with the samples. """ # Author: Remi Flamary +# Kilian Fatras # # License: MIT License @@ -100,3 +101,28 @@ pl.legend(loc=0) pl.title('OT matrix Sinkhorn with samples') pl.show() + + +############################################################################## +# Emprirical Sinkhorn +# ---------------- + +#%% sinkhorn + +# reg term +lambd = 1e-3 + +Ges = ot.bregman.empirical_sinkhorn(xs, xt, lambd) + +pl.figure(7) +pl.imshow(Ges, interpolation='nearest') +pl.title('OT matrix empirical sinkhorn') + +pl.figure(8) +ot.plot.plot2D_samples_mat(xs, xt, Ges, color=[.5, .5, 1]) +pl.plot(xs[:, 0], xs[:, 1], '+b', label='Source samples') +pl.plot(xt[:, 0], xt[:, 1], 'xr', label='Target samples') +pl.legend(loc=0) +pl.title('OT matrix Sinkhorn from samples') + +pl.show() diff --git a/ot/bregman.py b/ot/bregman.py index 013bc33..f1b18f8 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -5,6 +5,7 @@ Bregman projections for regularized OT # Author: Remi Flamary # Nicolas Courty +# Kilian Fatras # # License: MIT License @@ -1296,3 +1297,271 @@ def unmix(a, D, M, M0, h0, reg, reg0, alpha, numItermax=1000, return np.sum(K0, axis=1), log else: return np.sum(K0, axis=1) + + +def empirical_sinkhorn(X_s, X_t, reg, a=None, b=None, metric='sqeuclidean', numIterMax=10000, stopThr=1e-9, verbose=False, log=False, **kwargs): + ''' + Solve the entropic regularization optimal transport problem and return the + OT matrix from empirical data + + The function solves the following optimization problem: + + .. math:: + \gamma = arg\min_\gamma <\gamma,M>_F + reg\cdot\Omega(\gamma) + + s.t. \gamma 1 = a + + \gamma^T 1= b + + \gamma\geq 0 + where : + + - M is the (ns,nt) metric cost matrix + - :math:`\Omega` is the entropic regularization term :math:`\Omega(\gamma)=\sum_{i,j} \gamma_{i,j}\log(\gamma_{i,j})` + - a and b are source and target weights (sum to 1) + + + Parameters + ---------- + X_s : np.ndarray (ns, d) + samples in the source domain + X_t : np.ndarray (nt, d) + samples in the target domain + reg : float + Regularization term >0 + a : np.ndarray (ns,) + samples weights in the source domain + b : np.ndarray (nt,) + samples weights in the target domain + numItermax : int, optional + Max number of iterations + stopThr : float, optional + Stop threshol on error (>0) + verbose : bool, optional + Print information along iterations + log : bool, optional + record log if True + + + Returns + ------- + gamma : (ns x nt) ndarray + Regularized optimal transportation matrix for the given parameters + log : dict + log dictionary return only if log==True in parameters + + Examples + -------- + + >>> n_s = 2 + >>> n_t = 2 + >>> reg = 0.1 + >>> X_s = np.reshape(np.arange(n_s), (n_s, 1)) + >>> X_t = np.reshape(np.arange(0, n_t), (n_t, 1)) + >>> emp_sinkhorn = empirical_sinkhorn(X_s, X_t, reg, verbose=False) + >>> print(emp_sinkhorn) + >>> [[4.99977301e-01 2.26989344e-05] + [2.26989344e-05 4.99977301e-01]] + + + References + ---------- + + .. [2] M. Cuturi, Sinkhorn Distances : Lightspeed Computation of Optimal Transport, Advances in Neural Information Processing Systems (NIPS) 26, 2013 + + .. [9] Schmitzer, B. (2016). Stabilized Sparse Scaling Algorithms for Entropy Regularized Transport Problems. arXiv preprint arXiv:1610.06519. + + .. [10] Chizat, L., Peyré, G., Schmitzer, B., & Vialard, F. X. (2016). Scaling algorithms for unbalanced transport problems. arXiv preprint arXiv:1607.05816. + ''' + + if a is None: + a = ot.unif(np.shape(X_s)[0]) + if b is None: + b = ot.unif(np.shape(X_t)[0]) + M = ot.dist(X_s, X_t, metric=metric) + if log == False: + pi = ot.sinkhorn(a, b, M, reg, numItermax=numIterMax, stopThr=stopThr, verbose=verbose, log=False, **kwargs) + return pi + + if log == True: + pi, log = ot.sinkhorn(a, b, M, reg, numItermax=numIterMax, stopThr=stopThr, verbose=verbose, log=True, **kwargs) + return pi, log + + +def empirical_sinkhorn2(X_s, X_t, reg, a=None, b=None, metric='sqeuclidean', numIterMax=10000, stopThr=1e-9, verbose=False, log=False, **kwargs): + ''' + Solve the entropic regularization optimal transport problem from empirical + data and return the OT loss + + + The function solves the following optimization problem: + + .. math:: + W = \min_\gamma_\gamma <\gamma,M>_F + reg\cdot\Omega(\gamma) + + s.t. \gamma 1 = a + + \gamma^T 1= b + + \gamma\geq 0 + where : + + - M is the (ns,nt) metric cost matrix + - :math:`\Omega` is the entropic regularization term :math:`\Omega(\gamma)=\sum_{i,j} \gamma_{i,j}\log(\gamma_{i,j})` + - a and b are source and target weights (sum to 1) + + + Parameters + ---------- + X_s : np.ndarray (ns, d) + samples in the source domain + X_t : np.ndarray (nt, d) + samples in the target domain + reg : float + Regularization term >0 + a : np.ndarray (ns,) + samples weights in the source domain + b : np.ndarray (nt,) + samples weights in the target domain + numItermax : int, optional + Max number of iterations + stopThr : float, optional + Stop threshol on error (>0) + verbose : bool, optional + Print information along iterations + log : bool, optional + record log if True + + + Returns + ------- + gamma : (ns x nt) ndarray + Regularized optimal transportation matrix for the given parameters + log : dict + log dictionary return only if log==True in parameters + + Examples + -------- + + >>> n_s = 2 + >>> n_t = 2 + >>> reg = 0.1 + >>> X_s = np.reshape(np.arange(n_s), (n_s, 1)) + >>> X_t = np.reshape(np.arange(0, n_t), (n_t, 1)) + >>> loss_sinkhorn = empirical_sinkhorn2(X_s, X_t, reg, verbose=False) + >>> print(loss_sinkhorn) + >>> [4.53978687e-05] + + + References + ---------- + + .. [2] M. Cuturi, Sinkhorn Distances : Lightspeed Computation of Optimal Transport, Advances in Neural Information Processing Systems (NIPS) 26, 2013 + + .. [9] Schmitzer, B. (2016). Stabilized Sparse Scaling Algorithms for Entropy Regularized Transport Problems. arXiv preprint arXiv:1610.06519. + + .. [10] Chizat, L., Peyré, G., Schmitzer, B., & Vialard, F. X. (2016). Scaling algorithms for unbalanced transport problems. arXiv preprint arXiv:1607.05816. + ''' + + if a is None: + a = ot.unif(np.shape(X_s)[0]) + if b is None: + b = ot.unif(np.shape(X_t)[0]) + + M = ot.dist(X_s, X_t, metric=metric) + if log == False: + sinkhorn_loss = ot.sinkhorn2(a, b, M, reg, numItermax=numIterMax, stopThr=stopThr, verbose=verbose, log=log, **kwargs) + return sinkhorn_loss + + if log == True: + sinkhorn_loss, log = ot.sinkhorn2(a, b, M, reg, numItermax=numIterMax, stopThr=stopThr, verbose=verbose, log=log, **kwargs) + return sinkhorn_loss, log + + +def empirical_sinkhorn_divergence(X_s, X_t, reg, a=None, b=None, metric='sqeuclidean', numIterMax=10000, stopThr=1e-9, verbose=False, log=False, **kwargs): + ''' + Compute the sinkhorn divergence loss from empirical data + + The function solves the following optimization problem: + + .. math:: + S = \min_\gamma <\gamma,M>_F + reg\cdot\Omega(\gamma) - + \min_\gamma_a <\gamma_a,M_a>_F + reg\cdot\Omega(\gamma_a) - + \min_\gamma_b <\gamma_b,M_b>_F + reg\cdot\Omega(\gamma_b) + + s.t. \gamma 1 = a + + \gamma^T 1= b + + \gamma\geq 0 + + \gamma_a 1 = a + + \gamma_a^T 1= a + + \gamma_a\geq 0 + + \gamma_b 1 = b + + \gamma_b^T 1= b + + \gamma_b\geq 0 + where : + + - M (resp. :math:`M_a, M_b) is the (ns,nt) metric cost matrix (resp (ns, ns) and (nt, nt)) + - :math:`\Omega` is the entropic regularization term :math:`\Omega(\gamma)=\sum_{i,j} \gamma_{i,j}\log(\gamma_{i,j})` + - a and b are source and target weights (sum to 1) + + + Parameters + ---------- + X_s : np.ndarray (ns, d) + samples in the source domain + X_t : np.ndarray (nt, d) + samples in the target domain + reg : float + Regularization term >0 + a : np.ndarray (ns,) + samples weights in the source domain + b : np.ndarray (nt,) + samples weights in the target domain + numItermax : int, optional + Max number of iterations + stopThr : float, optional + Stop threshol on error (>0) + verbose : bool, optional + Print information along iterations + log : bool, optional + record log if True + + + Returns + ------- + gamma : (ns x nt) ndarray + Regularized optimal transportation matrix for the given parameters + log : dict + log dictionary return only if log==True in parameters + + Examples + -------- + + >>> n_s = 2 + >>> n_t = 4 + >>> reg = 0.1 + >>> X_s = np.reshape(np.arange(n_s), (n_s, 1)) + >>> X_t = np.reshape(np.arange(0, n_t), (n_t, 1)) + >>> emp_sinkhorn_div = empirical_sinkhorn_divergence(X_s, X_t, reg) + >>> print(emp_sinkhorn_div) + >>> [2.99977435] + + + References + ---------- + + .. [23] Aude Genevay, Gabriel Peyré, Marco Cuturi, Learning Generative Models with Sinkhorn Divergences, Proceedings of the Twenty-First International Conference on Artficial Intelligence and Statistics, (AISTATS) 21, 2018 + ''' + + sinkhorn_div = (2 * empirical_sinkhorn2(X_s, X_t, reg, a, b, metric=metric, numIterMax=numIterMax, stopThr=1e-9, verbose=verbose, log=log, **kwargs) - + empirical_sinkhorn2(X_s, X_s, reg, a, b, metric=metric, numIterMax=numIterMax, stopThr=1e-9, verbose=verbose, log=log, **kwargs) - + empirical_sinkhorn2(X_t, X_t, reg, a, b, metric=metric, numIterMax=numIterMax, stopThr=1e-9, verbose=verbose, log=log, **kwargs)) + return max(0, sinkhorn_div) diff --git a/test/test_bregman.py b/test/test_bregman.py index 90eaf27..b890df1 100644 --- a/test/test_bregman.py +++ b/test/test_bregman.py @@ -1,6 +1,7 @@ """Tests for module bregman on OT with bregman projections """ # Author: Remi Flamary +# Kilian Fatras # # License: MIT License @@ -187,3 +188,59 @@ def test_unmix(): ot.bregman.unmix(a, D, M, M0, h0, reg, 1, alpha=0.01, log=True, verbose=True) + + +def test_empirical_sinkhorn(): + # test sinkhorn + n = 100 + a = ot.unif(n) + b = ot.unif(n) + M = ot.dist(X_s, X_t) + M_e = ot.dist(X_s, X_t, metric='euclidean') + + rng = np.random.RandomState(0) + + X_s = np.reshape(np.arange(n), (n, 1)) + X_t = np.reshape(np.arange(0, n), (n, 1)) + + G_sqe = ot.bregman.empirical_sinkhorn(X_s, X_t, 1) + sinkhorn_sqe = ot.sinkhorn(a, b, M, 1) + + G_e = ot.bregman.empirical_sinkhorn(X_s, X_t, 1) + sinkhorn_e = ot.sinkhorn(a, b, M_e, 1) + + loss_emp_sinkhorn = ot.bregman.empirical_sinkhorn2(X_s, X_t, 1) + loss_sinkhorn = ot.sinkhorn2(a, b, M, 1) + + # check constratints + np.testing.assert_allclose( + sinkhorn_sqe.sum(1), G_sqe.sum(1), atol=1e-05) # metric sqeuclidian + np.testing.assert_allclose( + sinkhorn_sqe.sum(0), G_sqe.sum(0), atol=1e-05) # metric sqeuclidian + np.testing.assert_allclose( + sinkhorn_e.sum(1), G_e.sum(1), atol=1e-05) # metric euclidian + np.testing.assert_allclose( + sinkhorn_e.sum(0), G_e.sum(0), atol=1e-05) # metric euclidian + np.testing.assert_allclose(loss_emp_sinkhorn, loss_sinkhorn, atol=1e-05) + + +def test_empirical_sinkhorn_divergence(): + #Test sinkhorn divergence + n = 10 + a = ot.unif(n) + b = ot.unif(n) + X_s = np.reshape(np.arange(n), (n, 1)) + X_t = np.reshape(np.arange(0, n * 2, 2), (n, 1)) + M = ot.dist(X_s, X_t) + M_s = ot.dist(X_s, X_s) + M_t = ot.dist(X_t, X_t) + + emp_sinkhorn_div = empirical_sinkhorn_divergence(X_s, X_t, 1) + sinkhorn_div = (2 * ot.sinkhorn2(a, b, M, 1) - ot.sinkhorn2(a, a, M_s, 1) - + ot.sinkhorn2(b, b, M_t, 1)) + + # check constratints + np.testing.assert_allclose( + emp_sinkhorn_div, sinkhorn_div, atol=1e-05) # cf conv emp sinkhorn + np.testing.assert_allclose( + emp_sinkhorn_div, sinkhorn_div, atol=1e-05) # cf conv emp sinkhorn -- cgit v1.2.3 From 9569f893defa8e712a4f3199770a0df745d4cfff Mon Sep 17 00:00:00 2001 From: Kilian Fatras Date: Fri, 29 Mar 2019 13:06:01 +0100 Subject: fix pep8 --- ot/bregman.py | 29 +++++++++++++++-------------- test/test_bregman.py | 6 ++---- 2 files changed, 17 insertions(+), 18 deletions(-) (limited to 'ot/bregman.py') diff --git a/ot/bregman.py b/ot/bregman.py index f1b18f8..f6aa339 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -1375,17 +1375,18 @@ def empirical_sinkhorn(X_s, X_t, reg, a=None, b=None, metric='sqeuclidean', numI ''' if a is None: - a = ot.unif(np.shape(X_s)[0]) + a = utils.unif(np.shape(X_s)[0]) if b is None: - b = ot.unif(np.shape(X_t)[0]) + b = utils.unif(np.shape(X_t)[0]) + M = ot.dist(X_s, X_t, metric=metric) - if log == False: - pi = ot.sinkhorn(a, b, M, reg, numItermax=numIterMax, stopThr=stopThr, verbose=verbose, log=False, **kwargs) - return pi - if log == True: - pi, log = ot.sinkhorn(a, b, M, reg, numItermax=numIterMax, stopThr=stopThr, verbose=verbose, log=True, **kwargs) + if log: + pi, log = sinkhorn(a, b, M, reg, numItermax=numIterMax, stopThr=stopThr, verbose=verbose, log=True, **kwargs) return pi, log + else: + pi = sinkhorn(a, b, M, reg, numItermax=numIterMax, stopThr=stopThr, verbose=verbose, log=False, **kwargs) + return pi def empirical_sinkhorn2(X_s, X_t, reg, a=None, b=None, metric='sqeuclidean', numIterMax=10000, stopThr=1e-9, verbose=False, log=False, **kwargs): @@ -1464,18 +1465,18 @@ def empirical_sinkhorn2(X_s, X_t, reg, a=None, b=None, metric='sqeuclidean', num ''' if a is None: - a = ot.unif(np.shape(X_s)[0]) + a = utils.unif(np.shape(X_s)[0]) if b is None: - b = ot.unif(np.shape(X_t)[0]) + b = utils.unif(np.shape(X_t)[0]) M = ot.dist(X_s, X_t, metric=metric) - if log == False: - sinkhorn_loss = ot.sinkhorn2(a, b, M, reg, numItermax=numIterMax, stopThr=stopThr, verbose=verbose, log=log, **kwargs) - return sinkhorn_loss - if log == True: - sinkhorn_loss, log = ot.sinkhorn2(a, b, M, reg, numItermax=numIterMax, stopThr=stopThr, verbose=verbose, log=log, **kwargs) + if log: + sinkhorn_loss, log = sinkhorn2(a, b, M, reg, numItermax=numIterMax, stopThr=stopThr, verbose=verbose, log=log, **kwargs) return sinkhorn_loss, log + else: + sinkhorn_loss = sinkhorn2(a, b, M, reg, numItermax=numIterMax, stopThr=stopThr, verbose=verbose, log=log, **kwargs) + return sinkhorn_loss def empirical_sinkhorn_divergence(X_s, X_t, reg, a=None, b=None, metric='sqeuclidean', numIterMax=10000, stopThr=1e-9, verbose=False, log=False, **kwargs): diff --git a/test/test_bregman.py b/test/test_bregman.py index b890df1..8b001a7 100644 --- a/test/test_bregman.py +++ b/test/test_bregman.py @@ -195,13 +195,11 @@ def test_empirical_sinkhorn(): n = 100 a = ot.unif(n) b = ot.unif(n) - M = ot.dist(X_s, X_t) - M_e = ot.dist(X_s, X_t, metric='euclidean') - - rng = np.random.RandomState(0) X_s = np.reshape(np.arange(n), (n, 1)) X_t = np.reshape(np.arange(0, n), (n, 1)) + M = ot.dist(X_s, X_t) + M_e = ot.dist(X_s, X_t, metric='euclidean') G_sqe = ot.bregman.empirical_sinkhorn(X_s, X_t, 1) sinkhorn_sqe = ot.sinkhorn(a, b, M, 1) -- cgit v1.2.3 From f63712f4df213bbfe0a2665390d51974305de705 Mon Sep 17 00:00:00 2001 From: Kilian Fatras Date: Fri, 29 Mar 2019 13:44:07 +0100 Subject: call ot.unif and ot.dist --- ot/bregman.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'ot/bregman.py') diff --git a/ot/bregman.py b/ot/bregman.py index f6aa339..9e9989f 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -1375,9 +1375,9 @@ def empirical_sinkhorn(X_s, X_t, reg, a=None, b=None, metric='sqeuclidean', numI ''' if a is None: - a = utils.unif(np.shape(X_s)[0]) + a = ot.unif(np.shape(X_s)[0]) if b is None: - b = utils.unif(np.shape(X_t)[0]) + b = ot.unif(np.shape(X_t)[0]) M = ot.dist(X_s, X_t, metric=metric) @@ -1465,9 +1465,9 @@ def empirical_sinkhorn2(X_s, X_t, reg, a=None, b=None, metric='sqeuclidean', num ''' if a is None: - a = utils.unif(np.shape(X_s)[0]) + a = ot.unif(np.shape(X_s)[0]) if b is None: - b = utils.unif(np.shape(X_t)[0]) + b = ot.unif(np.shape(X_t)[0]) M = ot.dist(X_s, X_t, metric=metric) -- cgit v1.2.3 From 24b268c5d4ee3a03e5a1742168236755cfba9ed5 Mon Sep 17 00:00:00 2001 From: Kilian Fatras Date: Fri, 29 Mar 2019 13:59:25 +0100 Subject: import unif and dist in bregman file --- ot/bregman.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) (limited to 'ot/bregman.py') diff --git a/ot/bregman.py b/ot/bregman.py index 9e9989f..f873a85 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -10,6 +10,7 @@ Bregman projections for regularized OT # License: MIT License import numpy as np +from .utils import unif, dist def sinkhorn(a, b, M, reg, method='sinkhorn', numItermax=1000, @@ -1375,11 +1376,11 @@ def empirical_sinkhorn(X_s, X_t, reg, a=None, b=None, metric='sqeuclidean', numI ''' if a is None: - a = ot.unif(np.shape(X_s)[0]) + a = unif(np.shape(X_s)[0]) if b is None: - b = ot.unif(np.shape(X_t)[0]) + b = unif(np.shape(X_t)[0]) - M = ot.dist(X_s, X_t, metric=metric) + M = dist(X_s, X_t, metric=metric) if log: pi, log = sinkhorn(a, b, M, reg, numItermax=numIterMax, stopThr=stopThr, verbose=verbose, log=True, **kwargs) @@ -1465,11 +1466,11 @@ def empirical_sinkhorn2(X_s, X_t, reg, a=None, b=None, metric='sqeuclidean', num ''' if a is None: - a = ot.unif(np.shape(X_s)[0]) + a = unif(np.shape(X_s)[0]) if b is None: - b = ot.unif(np.shape(X_t)[0]) + b = unif(np.shape(X_t)[0]) - M = ot.dist(X_s, X_t, metric=metric) + M = dist(X_s, X_t, metric=metric) if log: sinkhorn_loss, log = sinkhorn2(a, b, M, reg, numItermax=numIterMax, stopThr=stopThr, verbose=verbose, log=log, **kwargs) -- cgit v1.2.3 From 7c02007919596dedf9d4555737900e717c3d31a8 Mon Sep 17 00:00:00 2001 From: Kilian Fatras Date: Thu, 4 Apr 2019 11:46:49 +0200 Subject: fix doc --- ot/bregman.py | 29 ++++++++++++++++++----------- ot/stochastic.py | 3 +++ 2 files changed, 21 insertions(+), 11 deletions(-) (limited to 'ot/bregman.py') diff --git a/ot/bregman.py b/ot/bregman.py index f873a85..47554fb 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -1317,9 +1317,9 @@ def empirical_sinkhorn(X_s, X_t, reg, a=None, b=None, metric='sqeuclidean', numI \gamma\geq 0 where : - - M is the (ns,nt) metric cost matrix + - :math:`M` is the (ns,nt) metric cost matrix - :math:`\Omega` is the entropic regularization term :math:`\Omega(\gamma)=\sum_{i,j} \gamma_{i,j}\log(\gamma_{i,j})` - - a and b are source and target weights (sum to 1) + - :math:`a` and :math:`b` are source and target weights (sum to 1) Parameters @@ -1399,7 +1399,7 @@ def empirical_sinkhorn2(X_s, X_t, reg, a=None, b=None, metric='sqeuclidean', num The function solves the following optimization problem: .. math:: - W = \min_\gamma_\gamma <\gamma,M>_F + reg\cdot\Omega(\gamma) + W = \min_\gamma <\gamma,M>_F + reg\cdot\Omega(\gamma) s.t. \gamma 1 = a @@ -1408,9 +1408,9 @@ def empirical_sinkhorn2(X_s, X_t, reg, a=None, b=None, metric='sqeuclidean', num \gamma\geq 0 where : - - M is the (ns,nt) metric cost matrix + - :math:`M` is the (ns,nt) metric cost matrix - :math:`\Omega` is the entropic regularization term :math:`\Omega(\gamma)=\sum_{i,j} \gamma_{i,j}\log(\gamma_{i,j})` - - a and b are source and target weights (sum to 1) + - :math:`a` and :math:`b` are source and target weights (sum to 1) Parameters @@ -1484,13 +1484,20 @@ def empirical_sinkhorn_divergence(X_s, X_t, reg, a=None, b=None, metric='sqeucli ''' Compute the sinkhorn divergence loss from empirical data - The function solves the following optimization problem: + The function solves the following optimization problems and return the + sinkhorn divergence :math:`S`: .. math:: - S = \min_\gamma <\gamma,M>_F + reg\cdot\Omega(\gamma) - - \min_\gamma_a <\gamma_a,M_a>_F + reg\cdot\Omega(\gamma_a) - - \min_\gamma_b <\gamma_b,M_b>_F + reg\cdot\Omega(\gamma_b) + W &= \min_\gamma <\gamma,M>_F + reg\cdot\Omega(\gamma) + + W_a &= \min_{\gamma_a} <\gamma_a,M_a>_F + reg\cdot\Omega(\gamma_a) + + W_b &= \min_{\gamma_b} <\gamma_b,M_b>_F + reg\cdot\Omega(\gamma_b) + + S &= W - 1/2 * (W_a + W_b) + + .. math:: s.t. \gamma 1 = a \gamma^T 1= b @@ -1510,9 +1517,9 @@ def empirical_sinkhorn_divergence(X_s, X_t, reg, a=None, b=None, metric='sqeucli \gamma_b\geq 0 where : - - M (resp. :math:`M_a, M_b) is the (ns,nt) metric cost matrix (resp (ns, ns) and (nt, nt)) + - :math:`M` (resp. :math:`M_a, M_b`) is the (ns,nt) metric cost matrix (resp (ns, ns) and (nt, nt)) - :math:`\Omega` is the entropic regularization term :math:`\Omega(\gamma)=\sum_{i,j} \gamma_{i,j}\log(\gamma_{i,j})` - - a and b are source and target weights (sum to 1) + - :math:`a` and :math:`b` are source and target weights (sum to 1) Parameters diff --git a/ot/stochastic.py b/ot/stochastic.py index 0db39c8..85c4230 100644 --- a/ot/stochastic.py +++ b/ot/stochastic.py @@ -348,8 +348,11 @@ def solve_semi_dual_entropic(a, b, M, reg, method, numItermax=10000, lr=None, .. math:: \gamma = arg\min_\gamma <\gamma,M>_F + reg\cdot\Omega(\gamma) + s.t. \gamma 1 = a + \gamma^T 1= b + \gamma \geq 0 Where : -- cgit v1.2.3 From 780bdfee3c622698dc9b18a02fa06381314aa56d Mon Sep 17 00:00:00 2001 From: Kilian Fatras Date: Thu, 4 Apr 2019 13:45:33 +0200 Subject: fix log in sinkhorn div and add log tests --- ot/bregman.py | 26 ++++++++++++++++++++++---- test/test_bregman.py | 12 +++++++++++- 2 files changed, 33 insertions(+), 5 deletions(-) (limited to 'ot/bregman.py') diff --git a/ot/bregman.py b/ot/bregman.py index 47554fb..7acfcf1 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -1569,8 +1569,26 @@ def empirical_sinkhorn_divergence(X_s, X_t, reg, a=None, b=None, metric='sqeucli .. [23] Aude Genevay, Gabriel Peyré, Marco Cuturi, Learning Generative Models with Sinkhorn Divergences, Proceedings of the Twenty-First International Conference on Artficial Intelligence and Statistics, (AISTATS) 21, 2018 ''' + if log: + sinkhorn_loss_ab, log_ab = empirical_sinkhorn2(X_s, X_t, reg, a, b, metric=metric, numIterMax=numIterMax, stopThr=1e-9, verbose=verbose, log=log, **kwargs) + + sinkhorn_loss_a, log_a = empirical_sinkhorn2(X_s, X_s, reg, a, b, metric=metric, numIterMax=numIterMax, stopThr=1e-9, verbose=verbose, log=log, **kwargs) + + sinkhorn_loss_b, log_b = empirical_sinkhorn2(X_t, X_t, reg, a, b, metric=metric, numIterMax=numIterMax, stopThr=1e-9, verbose=verbose, log=log, **kwargs) - sinkhorn_div = (2 * empirical_sinkhorn2(X_s, X_t, reg, a, b, metric=metric, numIterMax=numIterMax, stopThr=1e-9, verbose=verbose, log=log, **kwargs) - - empirical_sinkhorn2(X_s, X_s, reg, a, b, metric=metric, numIterMax=numIterMax, stopThr=1e-9, verbose=verbose, log=log, **kwargs) - - empirical_sinkhorn2(X_t, X_t, reg, a, b, metric=metric, numIterMax=numIterMax, stopThr=1e-9, verbose=verbose, log=log, **kwargs)) - return max(0, sinkhorn_div) + sinkhorn_div = sinkhorn_loss_ab - 1 / 2 * (sinkhorn_loss_a + sinkhorn_loss_b) + + log = {} + log['sinkhorn_loss_ab'] = sinkhorn_loss_ab + log['sinkhorn_loss_a'] = sinkhorn_loss_a + log['sinkhorn_loss_b'] = sinkhorn_loss_b + log['log_sinkhorn_ab'] = log_ab + log['log_sinkhorn_a'] = log_a + log['log_sinkhorn_b'] = log_b + + return max(0, sinkhorn_div), log + else: + sinkhorn_div = (empirical_sinkhorn2(X_s, X_t, reg, a, b, metric=metric, numIterMax=numIterMax, stopThr=1e-9, verbose=verbose, log=log, **kwargs) - + 1 / 2 * empirical_sinkhorn2(X_s, X_s, reg, a, b, metric=metric, numIterMax=numIterMax, stopThr=1e-9, verbose=verbose, log=log, **kwargs) - + 1 / 2 * empirical_sinkhorn2(X_t, X_t, reg, a, b, metric=metric, numIterMax=numIterMax, stopThr=1e-9, verbose=verbose, log=log, **kwargs)) + return max(0, sinkhorn_div) diff --git a/test/test_bregman.py b/test/test_bregman.py index 0ebd546..68d3595 100644 --- a/test/test_bregman.py +++ b/test/test_bregman.py @@ -204,6 +204,9 @@ def test_empirical_sinkhorn(): G_sqe = ot.bregman.empirical_sinkhorn(X_s, X_t, 1) sinkhorn_sqe = ot.sinkhorn(a, b, M, 1) + G_log, log_es = ot.bregman.empirical_sinkhorn(X_s, X_t, 0.1, log=True) + sinkhorn_log, log_s = ot.sinkhorn(a, b, M, 0.1, log=True) + G_m = ot.bregman.empirical_sinkhorn(X_s, X_t, 1, metric='minkowski') sinkhorn_m = ot.sinkhorn(a, b, M_m, 1) @@ -215,6 +218,10 @@ def test_empirical_sinkhorn(): sinkhorn_sqe.sum(1), G_sqe.sum(1), atol=1e-05) # metric sqeuclidian np.testing.assert_allclose( sinkhorn_sqe.sum(0), G_sqe.sum(0), atol=1e-05) # metric sqeuclidian + np.testing.assert_allclose( + sinkhorn_log.sum(1), G_log.sum(1), atol=1e-05) # log + np.testing.assert_allclose( + sinkhorn_log.sum(0), G_log.sum(0), atol=1e-05) # log np.testing.assert_allclose( sinkhorn_m.sum(1), G_m.sum(1), atol=1e-05) # metric euclidian np.testing.assert_allclose( @@ -237,8 +244,11 @@ def test_empirical_sinkhorn_divergence(): sinkhorn_div = (2 * ot.sinkhorn2(a, b, M, 1) - ot.sinkhorn2(a, a, M_s, 1) - ot.sinkhorn2(b, b, M_t, 1)) + emp_sinkhorn_div_log, log_es = ot.bregman.empirical_sinkhorn_divergence(X_s, X_t, 0.1, log=True) + sinkhorn_div_log, log_s = ot.sinkhorn(a, b, M, 0.1, log=True) + # check constratints np.testing.assert_allclose( emp_sinkhorn_div, sinkhorn_div, atol=1e-05) # cf conv emp sinkhorn np.testing.assert_allclose( - emp_sinkhorn_div, sinkhorn_div, atol=1e-05) # cf conv emp sinkhorn + emp_sinkhorn_div_log, sinkhorn_div_log, atol=1e-05) # cf conv emp sinkhorn -- cgit v1.2.3 From 69186a6f4259d32fecac370f59efe16e2e460d04 Mon Sep 17 00:00:00 2001 From: Kilian Fatras Date: Thu, 4 Apr 2019 13:58:50 +0200 Subject: fix test sinkhorn div --- ot/bregman.py | 11 ++++++++--- test/test_bregman.py | 8 +++++--- 2 files changed, 13 insertions(+), 6 deletions(-) (limited to 'ot/bregman.py') diff --git a/ot/bregman.py b/ot/bregman.py index 7acfcf1..dc43834 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -1587,8 +1587,13 @@ def empirical_sinkhorn_divergence(X_s, X_t, reg, a=None, b=None, metric='sqeucli log['log_sinkhorn_b'] = log_b return max(0, sinkhorn_div), log + else: - sinkhorn_div = (empirical_sinkhorn2(X_s, X_t, reg, a, b, metric=metric, numIterMax=numIterMax, stopThr=1e-9, verbose=verbose, log=log, **kwargs) - - 1 / 2 * empirical_sinkhorn2(X_s, X_s, reg, a, b, metric=metric, numIterMax=numIterMax, stopThr=1e-9, verbose=verbose, log=log, **kwargs) - - 1 / 2 * empirical_sinkhorn2(X_t, X_t, reg, a, b, metric=metric, numIterMax=numIterMax, stopThr=1e-9, verbose=verbose, log=log, **kwargs)) + sinkhorn_loss_ab = empirical_sinkhorn2(X_s, X_t, reg, a, b, metric=metric, numIterMax=numIterMax, stopThr=1e-9, verbose=verbose, log=log, **kwargs) + + sinkhorn_loss_a = empirical_sinkhorn2(X_s, X_s, reg, a, b, metric=metric, numIterMax=numIterMax, stopThr=1e-9, verbose=verbose, log=log, **kwargs) + + sinkhorn_loss_b = empirical_sinkhorn2(X_t, X_t, reg, a, b, metric=metric, numIterMax=numIterMax, stopThr=1e-9, verbose=verbose, log=log, **kwargs) + + sinkhorn_div = sinkhorn_loss_ab - 1 / 2 * (sinkhorn_loss_a + sinkhorn_loss_b) return max(0, sinkhorn_div) diff --git a/test/test_bregman.py b/test/test_bregman.py index 68d3595..58700e2 100644 --- a/test/test_bregman.py +++ b/test/test_bregman.py @@ -241,11 +241,13 @@ def test_empirical_sinkhorn_divergence(): M_t = ot.dist(X_t, X_t) emp_sinkhorn_div = ot.bregman.empirical_sinkhorn_divergence(X_s, X_t, 1) - sinkhorn_div = (2 * ot.sinkhorn2(a, b, M, 1) - ot.sinkhorn2(a, a, M_s, 1) - - ot.sinkhorn2(b, b, M_t, 1)) + sinkhorn_div = (ot.sinkhorn2(a, b, M, 1) - 1 / 2 * ot.sinkhorn2(a, a, M_s, 1) - 1 / 2 * ot.sinkhorn2(b, b, M_t, 1)) emp_sinkhorn_div_log, log_es = ot.bregman.empirical_sinkhorn_divergence(X_s, X_t, 0.1, log=True) - sinkhorn_div_log, log_s = ot.sinkhorn(a, b, M, 0.1, log=True) + sink_div_log, log_s = ot.sinkhorn2(a, b, M, 1) + sink_div_log_a, log_s_a = ot.sinkhorn2(a, a, M_s, 1) + sink_div_log_b, log_s_b = ot.sinkhorn2(b, b, M_t, 1) + sink_div_log = sink_div_log - 1 / 2 * (sink_div_log_a + sink_div_log_b) # check constratints np.testing.assert_allclose( -- cgit v1.2.3 From 549b95b5736b42f3fe74daf9805303a08b1ae01d Mon Sep 17 00:00:00 2001 From: tvayer Date: Tue, 28 May 2019 16:08:41 +0200 Subject: FGW+gromov changes --- README.md | 2 + examples/plot_fgw.py | 152 +++++++++++++++++++++++++ ot/bregman.py | 2 +- ot/gromov.py | 310 ++++++++++++++++++++++++++++++++++++++++++++++++--- ot/optim.py | 102 ++++++++++++++++- 5 files changed, 546 insertions(+), 22 deletions(-) create mode 100644 examples/plot_fgw.py (limited to 'ot/bregman.py') diff --git a/README.md b/README.md index a22306d..be88f65 100644 --- a/README.md +++ b/README.md @@ -219,3 +219,5 @@ You can also post bug reports and feature requests in Github issues. Make sure t [16] Agueh, M., & Carlier, G. (2011). [Barycenters in the Wasserstein space](https://hal.archives-ouvertes.fr/hal-00637399/document). SIAM Journal on Mathematical Analysis, 43(2), 904-924. [17] Blondel, M., Seguy, V., & Rolet, A. (2018). [Smooth and Sparse Optimal Transport](https://arxiv.org/abs/1710.06276). Proceedings of the Twenty-First International Conference on Artificial Intelligence and Statistics (AISTATS). + +[18] Vayer, T., Chapel, L., Flamary, R., Tavenard, R. and Courty, N. (2019). [Optimal Transport for structured data with application on graphs](http://proceedings.mlr.press/v97/titouan19a.html) Proceedings of the 36th International Conference on Machine Learning (ICML). diff --git a/examples/plot_fgw.py b/examples/plot_fgw.py new file mode 100644 index 0000000..5c2d0e1 --- /dev/null +++ b/examples/plot_fgw.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +============================== +Plot Fused-gromov-Wasserstein +============================== + +This example illustrates the computation of FGW for 1D measures[18]. + +.. [18] Vayer Titouan, Chapel Laetitia, Flamary R{\'e}mi, Tavenard Romain + and Courty Nicolas + "Optimal Transport for structured data with application on graphs" + International Conference on Machine Learning (ICML). 2019. + +""" + +# Author: Titouan Vayer +# +# License: MIT License + +import matplotlib.pyplot as pl +import numpy as np +import ot +from ot.gromov import gromov_wasserstein,fused_gromov_wasserstein + +#%% parameters +# We create two 1D random measures +n=20 +n2=30 +sig=1 +sig2=0.1 + +np.random.seed(0) + +phi=np.arange(n)[:,None] +xs=phi+sig*np.random.randn(n,1) +ys=np.vstack((np.ones((n//2,1)),0*np.ones((n//2,1))))+sig2*np.random.randn(n,1) + +phi2=np.arange(n2)[:,None] +xt=phi2+sig*np.random.randn(n2,1) +yt=np.vstack((np.ones((n2//2,1)),0*np.ones((n2//2,1))))+sig2*np.random.randn(n2,1) +yt= yt[::-1,:] + +p=ot.unif(n) +q=ot.unif(n2) + +#%% plot the distributions + +pl.close(10) +pl.figure(10,(7,7)) + +pl.subplot(2,1,1) + +pl.scatter(ys,xs,c=phi,s=70) +pl.ylabel('Feature value a',fontsize=20) +pl.title('$\mu=\sum_i \delta_{x_i,a_i}$',fontsize=25, usetex=True, y=1) +pl.xticks(()) +pl.yticks(()) +pl.subplot(2,1,2) +pl.scatter(yt,xt,c=phi2,s=70) +pl.xlabel('coordinates x/y',fontsize=25) +pl.ylabel('Feature value b',fontsize=20) +pl.title('$\\nu=\sum_j \delta_{y_j,b_j}$',fontsize=25, usetex=True, y=1) +pl.yticks(()) +pl.tight_layout() +pl.show() + + +#%% Structure matrices and across-features distance matrix +C1=ot.dist(xs) +C2=ot.dist(xt).T +M=ot.dist(ys,yt) +w1=ot.unif(C1.shape[0]) +w2=ot.unif(C2.shape[0]) +Got=ot.emd([],[],M) + +#%% +cmap='Reds' +pl.close(10) +pl.figure(10,(5,5)) +fs=15 +l_x=[0,5,10,15] +l_y=[0,5,10,15,20,25] +gs = pl.GridSpec(5, 5) + +ax1=pl.subplot(gs[3:,:2]) + +pl.imshow(C1,cmap=cmap,interpolation='nearest') +pl.title("$C_1$",fontsize=fs) +pl.xlabel("$k$",fontsize=fs) +pl.ylabel("$i$",fontsize=fs) +pl.xticks(l_x) +pl.yticks(l_x) + +ax2=pl.subplot(gs[:3,2:]) + +pl.imshow(C2,cmap=cmap,interpolation='nearest') +pl.title("$C_2$",fontsize=fs) +pl.ylabel("$l$",fontsize=fs) +#pl.ylabel("$l$",fontsize=fs) +pl.xticks(()) +pl.yticks(l_y) +ax2.set_aspect('auto') + +ax3=pl.subplot(gs[3:,2:],sharex=ax2,sharey=ax1) +pl.imshow(M,cmap=cmap,interpolation='nearest') +pl.yticks(l_x) +pl.xticks(l_y) +pl.ylabel("$i$",fontsize=fs) +pl.title("$M_{AB}$",fontsize=fs) +pl.xlabel("$j$",fontsize=fs) +pl.tight_layout() +ax3.set_aspect('auto') +pl.show() + + +#%% Computing FGW and GW +alpha=1e-3 + +ot.tic() +Gwg,logw=fused_gromov_wasserstein(M,C1,C2,p,q,loss_fun='square_loss',alpha=alpha,verbose=True,log=True) +ot.toc() + +#%reload_ext WGW +Gg,log=gromov_wasserstein(C1,C2,p,q,loss_fun='square_loss',verbose=True,log=True) + +#%% visu OT matrix +cmap='Blues' +fs=15 +pl.figure(2,(13,5)) +pl.clf() +pl.subplot(1,3,1) +pl.imshow(Got,cmap=cmap,interpolation='nearest') +#pl.xlabel("$y$",fontsize=fs) +pl.ylabel("$i$",fontsize=fs) +pl.xticks(()) + +pl.title('Wasserstein ($M$ only)') + +pl.subplot(1,3,2) +pl.imshow(Gg,cmap=cmap,interpolation='nearest') +pl.title('Gromov ($C_1,C_2$ only)') +pl.xticks(()) +pl.subplot(1,3,3) +pl.imshow(Gwg,cmap=cmap,interpolation='nearest') +pl.title('FGW ($M+C_1,C_2$)') + +pl.xlabel("$j$",fontsize=fs) +pl.ylabel("$i$",fontsize=fs) + +pl.tight_layout() +pl.show() \ No newline at end of file diff --git a/ot/bregman.py b/ot/bregman.py index b017c1a..9040429 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -5,7 +5,7 @@ Bregman projections for regularized OT # Author: Remi Flamary # Nicolas Courty -# +# Titouan Vayer # License: MIT License import numpy as np diff --git a/ot/gromov.py b/ot/gromov.py index 0278e99..7491664 100644 --- a/ot/gromov.py +++ b/ot/gromov.py @@ -9,17 +9,18 @@ Gromov-Wasserstein transport method # Author: Erwan Vautier # Nicolas Courty # Rémi Flamary -# +# Titouan Vayer # License: MIT License import numpy as np + from .bregman import sinkhorn from .utils import dist from .optim import cg -def init_matrix(C1, C2, T, p, q, loss_fun='square_loss'): +def init_matrix(C1, C2, p, q, loss_fun='square_loss'): """ Return loss matrices and tensors for Gromov-Wasserstein fast computation Returns the value of \mathcal{L}(C1,C2) \otimes T with the selected loss @@ -77,16 +78,16 @@ def init_matrix(C1, C2, T, p, q, loss_fun='square_loss'): if loss_fun == 'square_loss': def f1(a): - return (a**2) / 2 + return (a**2) def f2(b): - return (b**2) / 2 + return (b**2) def h1(a): return a def h2(b): - return b + return 2*b elif loss_fun == 'kl_loss': def f1(a): return a * np.log(a + 1e-15) - a @@ -268,7 +269,7 @@ def update_kl_loss(p, lambdas, T, Cs): return np.exp(np.divide(tmpsum, ppt)) -def gromov_wasserstein(C1, C2, p, q, loss_fun, log=False, **kwargs): +def gromov_wasserstein(C1, C2, p, q, loss_fun, log=False,amijo=False, **kwargs): """ Returns the gromov-wasserstein transport between (C1,p) and (C2,q) @@ -306,6 +307,9 @@ def gromov_wasserstein(C1, C2, p, q, loss_fun, log=False, **kwargs): Print information along iterations log : bool, optional record log if True + amijo : bool, optional + If True the steps of the line-search is found via an amijo research. Else closed form is used. + If there is convergence issues use False. **kwargs : dict parameters can be directly pased to the ot.optim.cg solver @@ -329,9 +333,7 @@ def gromov_wasserstein(C1, C2, p, q, loss_fun, log=False, **kwargs): """ - T = np.eye(len(p), len(q)) - - constC, hC1, hC2 = init_matrix(C1, C2, T, p, q, loss_fun) + constC, hC1, hC2 = init_matrix(C1, C2, p, q, loss_fun) G0 = p[:, None] * q[None, :] @@ -342,14 +344,79 @@ def gromov_wasserstein(C1, C2, p, q, loss_fun, log=False, **kwargs): return gwggrad(constC, hC1, hC2, G) if log: - res, log = cg(p, q, 0, 1, f, df, G0, log=True, **kwargs) + res, log = cg(p, q, 0, 1, f, df, G0,log=True,amijo=amijo,C1=C1,C2=C2,constC=constC, **kwargs) log['gw_dist'] = gwloss(constC, hC1, hC2, res) return res, log else: - return cg(p, q, 0, 1, f, df, G0, **kwargs) + return cg(p, q, 0, 1, f, df, G0,amijo=amijo, **kwargs) + +def fused_gromov_wasserstein(M,C1,C2,p,q,loss_fun='square_loss',alpha=0.5,amijo=False,**kwargs): + """ + Computes the FGW distance between two graphs see [3] + .. math:: + \gamma = arg\min_\gamma (1-\alpha)*<\gamma,M>_F + alpha* \sum_{i,j,k,l} L(C1_{i,k},C2_{j,l})*T_{i,j}*T_{k,l} + s.t. \gamma 1 = p + \gamma^T 1= q + \gamma\geq 0 + where : + - M is the (ns,nt) metric cost matrix + - :math:`f` is the regularization term ( and df is its gradient) + - a and b are source and target weights (sum to 1) + The algorithm used for solving the problem is conditional gradient as discussed in [1]_ + Parameters + ---------- + M : ndarray, shape (ns, nt) + Metric cost matrix between features across domains + C1 : ndarray, shape (ns, ns) + Metric cost matrix respresentative of the structure in the source space + C2 : ndarray, shape (nt, nt) + Metric cost matrix espresentative of the structure in the target space + p : ndarray, shape (ns,) + distribution in the source space + q : ndarray, shape (nt,) + distribution in the target space + loss_fun : string,optionnal + loss function used for the solver + max_iter : int, optional + Max number of iterations + tol : float, optional + Stop threshold on error (>0) + verbose : bool, optional + Print information along iterations + log : bool, optional + record log if True + amijo : bool, optional + If True the steps of the line-search is found via an amijo research. Else closed form is used. + If there is convergence issues use False. + **kwargs : dict + parameters can be directly pased to the ot.optim.cg solver + Returns + ------- + gamma : (ns x nt) ndarray + Optimal transportation matrix for the given parameters + log : dict + log dictionary return only if log==True in parameters + References + ---------- + .. [18] Vayer Titouan, Chapel Laetitia, Flamary R{\'e}mi, Tavenard Romain + and Courty Nicolas + "Optimal Transport for structured data with application on graphs" + International Conference on Machine Learning (ICML). 2019. + """ + + constC,hC1,hC2=init_matrix(C1,C2,p,q,loss_fun) + + G0=p[:,None]*q[None,:] + + def f(G): + return gwloss(constC,hC1,hC2,G) + def df(G): + return gwggrad(constC,hC1,hC2,G) + + return cg(p,q,M,alpha,f,df,G0,amijo=amijo,C1=C1,C2=C2,constC=constC,**kwargs) -def gromov_wasserstein2(C1, C2, p, q, loss_fun, log=False, **kwargs): +def gromov_wasserstein2(C1, C2, p, q, loss_fun, log=False,amijo=False, **kwargs): """ Returns the gromov-wasserstein discrepancy between (C1,p) and (C2,q) @@ -387,7 +454,9 @@ def gromov_wasserstein2(C1, C2, p, q, loss_fun, log=False, **kwargs): Print information along iterations log : bool, optional record log if True - + amijo : bool, optional + If True the steps of the line-search is found via an amijo research. Else closed form is used. + If there is convergence issues use False. Returns ------- gw_dist : float @@ -407,9 +476,7 @@ def gromov_wasserstein2(C1, C2, p, q, loss_fun, log=False, **kwargs): """ - T = np.eye(len(p), len(q)) - - constC, hC1, hC2 = init_matrix(C1, C2, T, p, q, loss_fun) + constC, hC1, hC2 = init_matrix(C1, C2, p, q, loss_fun) G0 = p[:, None] * q[None, :] @@ -418,7 +485,7 @@ def gromov_wasserstein2(C1, C2, p, q, loss_fun, log=False, **kwargs): def df(G): return gwggrad(constC, hC1, hC2, G) - res, log = cg(p, q, 0, 1, f, df, G0, log=True, **kwargs) + res, log = cg(p, q, 0, 1, f, df, G0, log=True,amijo=amijo,C1=C1,C2=C2,constC=constC, **kwargs) log['gw_dist'] = gwloss(constC, hC1, hC2, res) log['T'] = res if log: @@ -495,7 +562,7 @@ def entropic_gromov_wasserstein(C1, C2, p, q, loss_fun, epsilon, T = np.outer(p, q) # Initialization - constC, hC1, hC2 = init_matrix(C1, C2, T, p, q, loss_fun) + constC, hC1, hC2 = init_matrix(C1, C2, p, q, loss_fun) cpt = 0 err = 1 @@ -815,3 +882,210 @@ def gromov_barycenters(N, Cs, ps, p, lambdas, loss_fun, cpt += 1 return C + +def fgw_barycenters(N,Ys,Cs,ps,lambdas,alpha,fixed_structure=False,fixed_features=False,p=None,loss_fun='square_loss', + max_iter=100, tol=1e-9,verbose=False,log=True,init_C=None,init_X=None): + + """ + Compute the fgw barycenter as presented eq (5) in [3]. + ---------- + N : integer + Desired number of samples of the target barycenter + Ys: list of ndarray, each element has shape (ns,d) + Features of all samples + Cs : list of ndarray, each element has shape (ns,ns) + Structure matrices of all samples + ps : list of ndarray, each element has shape (ns,) + masses of all samples + lambdas : list of float + list of the S spaces' weights + alpha : float + Alpha parameter for the fgw distance + fixed_structure : bool + Wether to fix the structure of the barycenter during the updates + fixed_features : bool + Wether to fix the feature of the barycenter during the updates + init_C : ndarray, shape (N,N), optional + initialization for the barycenters' structure matrix. If not set random init + init_X : ndarray, shape (N,d), optional + initialization for the barycenters' features. If not set random init + Returns + ---------- + X : ndarray, shape (N,d) + Barycenters' features + C : ndarray, shape (N,N) + Barycenters' structure matrix + log_: + T : list of (N,ns) transport matrices + Ms : all distance matrices between the feature of the barycenter and the other features dist(X,Ys) shape (N,ns) + References + ---------- + .. [18] Vayer Titouan, Chapel Laetitia, Flamary R{\'e}mi, Tavenard Romain + and Courty Nicolas + "Optimal Transport for structured data with application on graphs" + International Conference on Machine Learning (ICML). 2019. + """ + S = len(Cs) + d = Ys[0].shape[1] #dimension on the node features + if p is None: + p = np.ones(N)/N + + Cs = [np.asarray(Cs[s], dtype=np.float64) for s in range(S)] + Ys = [np.asarray(Ys[s], dtype=np.float64) for s in range(S)] + + lambdas = np.asarray(lambdas, dtype=np.float64) + + if fixed_structure: + if init_C is None: + C=Cs[0] + else: + C=init_C + else: + if init_C is None: + xalea = np.random.randn(N, 2) + C = dist(xalea, xalea) + else: + C = init_C + + if fixed_features: + if init_X is None: + X=Ys[0] + else : + X= init_X + else: + if init_X is None: + X=np.zeros((N,d)) + else: + X = init_X + + T=[np.outer(p,q) for q in ps] + + # X is N,d + # Ys is ns,d + Ms = [np.asarray(dist(X,Ys[s]), dtype=np.float64) for s in range(len(Ys))] + # Ms is N,ns + + cpt = 0 + err_feature = 1 + err_structure = 1 + + if log: + log_={} + log_['err_feature']=[] + log_['err_structure']=[] + log_['Ts_iter']=[] + + while((err_feature > tol or err_structure > tol) and cpt < max_iter): + Cprev = C + Xprev = X + + if not fixed_features: + Ys_temp=[y.T for y in Ys] + X=update_feature_matrix(lambdas,Ys_temp,T,p) + + # X must be N,d + # Ys must be ns,d + Ms=[np.asarray(dist(X,Ys[s]), dtype=np.float64) for s in range(len(Ys))] + + if not fixed_structure: + if loss_fun == 'square_loss': + # T must be ns,N + # Cs must be ns,ns + # p must be N,1 + T_temp=[t.T for t in T] + C = update_sructure_matrix(p, lambdas, T_temp, Cs) + + # Ys must be d,ns + # Ts must be N,ns + # p must be N,1 + # Ms is N,ns + # C is N,N + # Cs is ns,ns + # p is N,1 + # ps is ns,1 + + T = [fused_gromov_wasserstein((1-alpha)*Ms[s],C,Cs[s],p,ps[s],loss_fun,alpha,numItermax=max_iter, stopThr=1e-5, verbose=verbose) for s in range(S)] + + # T is N,ns + + log_['Ts_iter'].append(T) + err_feature = np.linalg.norm(X - Xprev.reshape(d,N)) + err_structure = np.linalg.norm(C - Cprev) + + if log: + log_['err_feature'].append(err_feature) + log_['err_structure'].append(err_structure) + + if verbose: + if cpt % 200 == 0: + print('{:5s}|{:12s}'.format( + 'It.', 'Err') + '\n' + '-' * 19) + print('{:5d}|{:8e}|'.format(cpt, err_structure)) + print('{:5d}|{:8e}|'.format(cpt, err_feature)) + + cpt += 1 + log_['T']=T # ce sont les matrices du barycentre de la target vers les Ys + log_['p']=p + log_['Ms']=Ms #Ms sont de tailles N,ns + + return X.T,C,log_ + + +def update_sructure_matrix(p, lambdas, T, Cs): + """ + Updates C according to the L2 Loss kernel with the S Ts couplings + calculated at each iteration + Parameters + ---------- + p : ndarray, shape (N,) + masses in the targeted barycenter + lambdas : list of float + list of the S spaces' weights + T : list of S np.ndarray(ns,N) + the S Ts couplings calculated at each iteration + Cs : list of S ndarray, shape(ns,ns) + Metric cost matrices + Returns + ---------- + C : ndarray, shape (nt,nt) + updated C matrix + """ + tmpsum = sum([lambdas[s] * np.dot(T[s].T, Cs[s]).dot(T[s]) for s in range(len(T))]) + ppt = np.outer(p, p) + + return np.divide(tmpsum, ppt) + +def update_feature_matrix(lambdas,Ys,Ts,p): + + """ + Updates the feature with respect to the S Ts couplings. See "Solving the barycenter problem with Block Coordinate Descent (BCD)" in [3] + calculated at each iteration + Parameters + ---------- + p : ndarray, shape (N,) + masses in the targeted barycenter + lambdas : list of float + list of the S spaces' weights + Ts : list of S np.ndarray(ns,N) + the S Ts couplings calculated at each iteration + Ys : list of S ndarray, shape(d,ns) + The features + Returns + ---------- + X : ndarray, shape (d,N) + + References + ---------- + .. [18] Vayer Titouan, Chapel Laetitia, Flamary R{\'e}mi, Tavenard Romain + and Courty Nicolas + "Optimal Transport for structured data with application on graphs" + International Conference on Machine Learning (ICML). 2019. + """ + + p=np.diag(np.array(1/p).reshape(-1,)) + + tmpsum = sum([lambdas[s] * np.dot(Ys[s],Ts[s].T).dot(p) for s in range(len(Ts))]) + + return tmpsum + + diff --git a/ot/optim.py b/ot/optim.py index f31fae2..a774865 100644 --- a/ot/optim.py +++ b/ot/optim.py @@ -4,7 +4,7 @@ Optimization algorithms for OT """ # Author: Remi Flamary -# +# Titouan Vayer # License: MIT License import numpy as np @@ -71,9 +71,70 @@ def line_search_armijo(f, xk, pk, gfk, old_fval, return alpha, fc[0], phi1 +def do_linesearch(cost,G,deltaG,Mi,f_val, + amijo=False,C1=None,C2=None,reg=None,Gc=None,constC=None,M=None): + """ + Solve the linesearch in the FW iterations + Parameters + ---------- + cost : method + The FGW cost + G : ndarray, shape(ns,nt) + The transport map at a given iteration of the FW + deltaG : ndarray (ns,nt) + Difference between the optimal map found by linearization in the FW algorithm and the value at a given iteration + Mi : ndarray (ns,nt) + Cost matrix of the linearized transport problem. Corresponds to the gradient of the cost + f_val : float + Value of the cost at G + amijo : bool, optionnal + If True the steps of the line-search is found via an amijo research. Else closed form is used. + If there is convergence issues use False. + C1 : ndarray (ns,ns), optionnal + Structure matrix in the source domain. Only used when amijo=False + C2 : ndarray (nt,nt), optionnal + Structure matrix in the target domain. Only used when amijo=False + reg : float, optionnal + Regularization parameter. Corresponds to the alpha parameter of FGW. Only used when amijo=False + Gc : ndarray (ns,nt) + Optimal map found by linearization in the FW algorithm. Only used when amijo=False + constC : ndarray (ns,nt) + Constant for the gromov cost. See [3]. Only used when amijo=False + M : ndarray (ns,nt), optionnal + Cost matrix between the features. Only used when amijo=False + Returns + ------- + alpha : float + The optimal step size of the FW + fc : int + nb of function call. Useless here + f_val : float + The value of the cost for the next iteration + References + ---------- + .. [18] Vayer Titouan, Chapel Laetitia, Flamary R{\'e}mi, Tavenard Romain + and Courty Nicolas + "Optimal Transport for structured data with application on graphs" + International Conference on Machine Learning (ICML). 2019. + """ + if amijo: + alpha, fc, f_val = line_search_armijo(cost, G, deltaG, Mi, f_val) + else: # requires symetric matrices + dot1=np.dot(C1,deltaG) + dot12=dot1.dot(C2) + a=-2*reg*np.sum(dot12*deltaG) + b=np.sum((M+reg*constC)*deltaG)-2*reg*(np.sum(dot12*G)+np.sum(np.dot(C1,G).dot(C2)*deltaG)) + c=cost(G) + + alpha=solve_1d_linesearch_quad_funct(a,b,c) + fc=None + f_val=cost(G+alpha*deltaG) + + return alpha,fc,f_val + def cg(a, b, M, reg, f, df, G0=None, numItermax=200, - stopThr=1e-9, verbose=False, log=False): + stopThr=1e-9, verbose=False, log=False,**kwargs): """ Solve the general regularized OT problem with conditional gradient @@ -116,6 +177,8 @@ def cg(a, b, M, reg, f, df, G0=None, numItermax=200, Print information along iterations log : bool, optional record log if True + kwargs : dict + Parameters for linesearch Returns ------- @@ -177,7 +240,7 @@ def cg(a, b, M, reg, f, df, G0=None, numItermax=200, deltaG = Gc - G # line search - alpha, fc, f_val = line_search_armijo(cost, G, deltaG, Mi, f_val) + alpha, fc, f_val = do_linesearch(cost, G, deltaG, Mi, f_val, reg=reg, M=M, Gc=Gc,**kwargs) G = G + alpha * deltaG @@ -339,3 +402,36 @@ def gcg(a, b, M, reg1, reg2, f, df, G0=None, numItermax=10, return G, log else: return G + +def solve_1d_linesearch_quad_funct(a,b,c): + """ + Solve on 0,1 the following problem: + .. math:: + \min f(x)=a*x^{2}+b*x+c + + Parameters + ---------- + a,b,c : float + The coefficients of the quadratic function + + Returns + ------- + x : float + The optimal value which leads to the minimal cost + + """ + f0=c + df0=b + f1=a+f0+df0 + + if a>0: # convex + minimum=min(1,max(0,-b/(2*a))) + #print('entrelesdeux') + return minimum + else: # non convexe donc sur les coins + if f0>f1: + #print('sur1 f(1)={}'.format(f(1))) + return 1 + else: + #print('sur0 f(0)={}'.format(f(0))) + return 0 -- cgit v1.2.3 From cd4b98c34f885176f33db3fab16530622f29ab42 Mon Sep 17 00:00:00 2001 From: tvayer Date: Tue, 28 May 2019 17:13:21 +0200 Subject: solve conlict --- README.md | 15 ++++++++++++++- ot/bregman.py | 1 + ot/gromov.py | 6 +++--- ot/optim.py | 2 +- 4 files changed, 19 insertions(+), 5 deletions(-) (limited to 'ot/bregman.py') diff --git a/README.md b/README.md index be88f65..13e1013 100644 --- a/README.md +++ b/README.md @@ -220,4 +220,17 @@ You can also post bug reports and feature requests in Github issues. Make sure t [17] Blondel, M., Seguy, V., & Rolet, A. (2018). [Smooth and Sparse Optimal Transport](https://arxiv.org/abs/1710.06276). Proceedings of the Twenty-First International Conference on Artificial Intelligence and Statistics (AISTATS). -[18] Vayer, T., Chapel, L., Flamary, R., Tavenard, R. and Courty, N. (2019). [Optimal Transport for structured data with application on graphs](http://proceedings.mlr.press/v97/titouan19a.html) Proceedings of the 36th International Conference on Machine Learning (ICML). +[18] Genevay, A., Cuturi, M., Peyré, G. & Bach, F. (2016) [Stochastic Optimization for Large-scale Optimal Transport](https://arxiv.org/abs/1605.08527). Advances in Neural Information Processing Systems (2016). + +[19] Seguy, V., Bhushan Damodaran, B., Flamary, R., Courty, N., Rolet, A.& Blondel, M. [Large-scale Optimal Transport and Mapping Estimation](https://arxiv.org/pdf/1711.02283.pdf). International Conference on Learning Representation (2018) + +[20] Cuturi, M. and Doucet, A. (2014) [Fast Computation of Wasserstein Barycenters](http://proceedings.mlr.press/v32/cuturi14.html). International Conference in Machine Learning + +[21] Solomon, J., De Goes, F., Peyré, G., Cuturi, M., Butscher, A., Nguyen, A. & Guibas, L. (2015). [Convolutional wasserstein distances: Efficient optimal transportation on geometric domains](https://dl.acm.org/citation.cfm?id=2766963). ACM Transactions on Graphics (TOG), 34(4), 66. + +[22] J. Altschuler, J.Weed, P. Rigollet, (2017) [Near-linear time approximation algorithms for optimal transport via Sinkhorn iteration](https://papers.nips.cc/paper/6792-near-linear-time-approximation-algorithms-for-optimal-transport-via-sinkhorn-iteration.pdf), Advances in Neural Information Processing Systems (NIPS) 31 + +[23] Aude, G., Peyré, G., Cuturi, M., [Learning Generative Models with Sinkhorn Divergences](https://arxiv.org/abs/1706.00292), Proceedings of the Twenty-First International Conference on Artficial Intelligence and Statistics, (AISTATS) 21, 2018 + +[24] Vayer, T., Chapel, L., Flamary, R., Tavenard, R. and Courty, N. (2019). [Optimal Transport for structured data with application on graphs](http://proceedings.mlr.press/v97/titouan19a.html) Proceedings of the 36th International Conference on Machine Learning (ICML). + diff --git a/ot/bregman.py b/ot/bregman.py index 9040429..7be67b8 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -5,6 +5,7 @@ Bregman projections for regularized OT # Author: Remi Flamary # Nicolas Courty +# Kilian Fatras # Titouan Vayer # License: MIT License diff --git a/ot/gromov.py b/ot/gromov.py index 31bd657..ad68a1c 100644 --- a/ot/gromov.py +++ b/ot/gromov.py @@ -398,7 +398,7 @@ def fused_gromov_wasserstein(M,C1,C2,p,q,loss_fun='square_loss',alpha=0.5,amijo= log dictionary return only if log==True in parameters References ---------- - .. [18] Vayer Titouan, Chapel Laetitia, Flamary R{\'e}mi, Tavenard Romain + .. [24] Vayer Titouan, Chapel Laetitia, Flamary R{\'e}mi, Tavenard Romain and Courty Nicolas "Optimal Transport for structured data with application on graphs" International Conference on Machine Learning (ICML). 2019. @@ -921,7 +921,7 @@ def fgw_barycenters(N,Ys,Cs,ps,lambdas,alpha,fixed_structure=False,fixed_feature Ms : all distance matrices between the feature of the barycenter and the other features dist(X,Ys) shape (N,ns) References ---------- - .. [18] Vayer Titouan, Chapel Laetitia, Flamary R{\'e}mi, Tavenard Romain + .. [24] Vayer Titouan, Chapel Laetitia, Flamary R{\'e}mi, Tavenard Romain and Courty Nicolas "Optimal Transport for structured data with application on graphs" International Conference on Machine Learning (ICML). 2019. @@ -1077,7 +1077,7 @@ def update_feature_matrix(lambdas,Ys,Ts,p): References ---------- - .. [18] Vayer Titouan, Chapel Laetitia, Flamary R{\'e}mi, Tavenard Romain + .. [24] Vayer Titouan, Chapel Laetitia, Flamary R{\'e}mi, Tavenard Romain and Courty Nicolas "Optimal Transport for structured data with application on graphs" International Conference on Machine Learning (ICML). 2019. diff --git a/ot/optim.py b/ot/optim.py index a774865..9fce21e 100644 --- a/ot/optim.py +++ b/ot/optim.py @@ -112,7 +112,7 @@ def do_linesearch(cost,G,deltaG,Mi,f_val, The value of the cost for the next iteration References ---------- - .. [18] Vayer Titouan, Chapel Laetitia, Flamary R{\'e}mi, Tavenard Romain + .. [24] Vayer Titouan, Chapel Laetitia, Flamary R{\'e}mi, Tavenard Romain and Courty Nicolas "Optimal Transport for structured data with application on graphs" International Conference on Machine Learning (ICML). 2019. -- cgit v1.2.3 From 11c2c26ff897e5763e714546e7021cffa8d673a7 Mon Sep 17 00:00:00 2001 From: tvayer Date: Tue, 28 May 2019 17:19:40 +0200 Subject: solve 2 --- README.md | 20 +++++++++++--------- ot/bregman.py | 1 + 2 files changed, 12 insertions(+), 9 deletions(-) (limited to 'ot/bregman.py') diff --git a/README.md b/README.md index 13e1013..9951773 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,7 @@ [![Anaconda Cloud](https://anaconda.org/conda-forge/pot/badges/version.svg)](https://anaconda.org/conda-forge/pot) [![Build Status](https://travis-ci.org/rflamary/POT.svg?branch=master)](https://travis-ci.org/rflamary/POT) [![Documentation Status](https://readthedocs.org/projects/pot/badge/?version=latest)](http://pot.readthedocs.io/en/latest/?badge=latest) +[![Downloads](https://pepy.tech/badge/pot)](https://pepy.tech/project/pot) [![Anaconda downloads](https://anaconda.org/conda-forge/pot/badges/downloads.svg)](https://anaconda.org/conda-forge/pot) [![License](https://anaconda.org/conda-forge/pot/badges/license.svg)](https://github.com/rflamary/POT/blob/master/LICENSE) @@ -14,15 +15,18 @@ This open source Python library provide several solvers for optimization problem It provides the following solvers: * OT Network Flow solver for the linear program/ Earth Movers Distance [1]. -* Entropic regularization OT solver with Sinkhorn Knopp Algorithm [2] and stabilized version [9][10] with optional GPU implementation (requires cudamat). +* Entropic regularization OT solver with Sinkhorn Knopp Algorithm [2], stabilized version [9][10] and greedy Sinkhorn [22] with optional GPU implementation (requires cupy). +* Sinkhorn divergence [23] and entropic regularization OT from empirical data. * Smooth optimal transport solvers (dual and semi-dual) for KL and squared L2 regularizations [17]. * Non regularized Wasserstein barycenters [16] with LP solver (only small scale). -* Bregman projections for Wasserstein barycenter [3] and unmixing [4]. +* Bregman projections for Wasserstein barycenter [3], convolutional barycenter [21] and unmixing [4]. * Optimal transport for domain adaptation with group lasso regularization [5] * Conditional gradient [6] and Generalized conditional gradient for regularized OT [7]. * Linear OT [14] and Joint OT matrix and mapping estimation [8]. * Wasserstein Discriminant Analysis [11] (requires autograd + pymanopt). * Gromov-Wasserstein distances and barycenters ([13] and regularized [12]) +* Stochastic Optimization for Large-scale Optimal Transport (semi-dual problem [18] and dual problem [19]) +* Non regularized free support Wasserstein barycenters [20]. Some demonstrations (both in Python and Jupyter Notebook format) are available in the examples folder. @@ -77,16 +81,12 @@ Note that for easier access the module is name ot instead of pot. Some sub-modules require additional dependences which are discussed below -* **ot.dr** (Wasserstein dimensionality rediuction) depends on autograd and pymanopt that can be installed with: +* **ot.dr** (Wasserstein dimensionality reduction) depends on autograd and pymanopt that can be installed with: ``` pip install pymanopt autograd ``` -* **ot.gpu** (GPU accelerated OT) depends on cudamat that have to be installed with: -``` -git clone https://github.com/cudamat/cudamat.git -cd cudamat -python setup.py install --user # for user install (no root) -``` +* **ot.gpu** (GPU accelerated OT) depends on cupy that have to be installed following instructions on [this page](https://docs-cupy.chainer.org/en/stable/install.html). + obviously you need CUDA installed and a compatible GPU. @@ -162,6 +162,8 @@ The contributors to this library are: * [Stanislas Chambon](https://slasnista.github.io/) * [Antoine Rolet](https://arolet.github.io/) * Erwan Vautier (Gromov-Wasserstein) +* [Kilian Fatras](https://kilianfatras.github.io/) +* [Alain Rakotomamonjy](https://sites.google.com/site/alainrakotomamonjy/home) This toolbox benefit a lot from open source research and we would like to thank the following persons for providing some code (in various languages): diff --git a/ot/bregman.py b/ot/bregman.py index 7be67b8..ffa6202 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -7,6 +7,7 @@ Bregman projections for regularized OT # Nicolas Courty # Kilian Fatras # Titouan Vayer +# # License: MIT License import numpy as np -- cgit v1.2.3 From 897982718a5fd81a9a591d80a7d50839399fc088 Mon Sep 17 00:00:00 2001 From: Hicham Janati Date: Tue, 18 Jun 2019 16:40:06 +0200 Subject: fix func names + add more tests --- ot/__init__.py | 2 +- ot/bregman.py | 2 +- ot/unbalanced.py | 79 ++++++++++++++++++++++++++++++------------------- test/test_unbalanced.py | 79 +++++++++++++++++++++++++++++++++++++++++++++++-- 4 files changed, 127 insertions(+), 35 deletions(-) (limited to 'ot/bregman.py') diff --git a/ot/__init__.py b/ot/__init__.py index 361be02..acb05e6 100644 --- a/ot/__init__.py +++ b/ot/__init__.py @@ -25,7 +25,7 @@ from . import unbalanced # OT functions from .lp import emd, emd2 from .bregman import sinkhorn, sinkhorn2, barycenter -from .unbalanced import sinkhorn_unbalanced +from .unbalanced import sinkhorn_unbalanced, barycenter_unbalanced from .da import sinkhorn_lpl1_mm # utils functions diff --git a/ot/bregman.py b/ot/bregman.py index 321712b..09716e6 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -241,7 +241,7 @@ def sinkhorn2(a, b, M, reg, method='sinkhorn', numItermax=1000, b = np.asarray(b, dtype=np.float64) if len(b.shape) < 2: - b = b.reshape((-1, 1)) + b = b[:, None] return sink() diff --git a/ot/unbalanced.py b/ot/unbalanced.py index a30fc18..97e2576 100644 --- a/ot/unbalanced.py +++ b/ot/unbalanced.py @@ -73,8 +73,9 @@ def sinkhorn_unbalanced(a, b, M, reg, alpha, method='sinkhorn', numItermax=1000, >>> a=[.5, .5] >>> b=[.5, .5] >>> M=[[0., 1.], [1., 0.]] - >>> ot.sinkhorn2(a, b, M, 1, 1) - array([0.26894142]) + >>> ot.sinkhorn_unbalanced(a, b, M, 1, 1) + array([[0.51122823, 0.18807035], + [0.18807035, 0.51122823]]) References @@ -91,28 +92,36 @@ def sinkhorn_unbalanced(a, b, M, reg, alpha, method='sinkhorn', numItermax=1000, See Also -------- - ot.unbalanced.sinkhorn_knopp : Unbalanced Classic Sinkhorn [10] - ot.unbalanced.sinkhorn_stabilized: Unbalanced Stabilized sinkhorn [9][10] - ot.unbalanced.sinkhorn_epsilon_scaling: Unbalanced Sinkhorn with epslilon scaling [9][10] + ot.unbalanced.sinkhorn_knopp_unbalanced : Unbalanced Classic Sinkhorn [10] + ot.unbalanced.sinkhorn_stabilized_unbalanced: Unbalanced Stabilized sinkhorn [9][10] + ot.unbalanced.sinkhorn_epsilon_scaling_unbalanced: Unbalanced Sinkhorn with epslilon scaling [9][10] """ if method.lower() == 'sinkhorn': def sink(): - return sinkhorn_knopp(a, b, M, reg, alpha, numItermax=numItermax, - stopThr=stopThr, verbose=verbose, log=log, **kwargs) - else: - warnings.warn('Unknown method. Falling back to classic Sinkhorn Knopp') + return sinkhorn_knopp_unbalanced(a, b, M, reg, alpha, + numItermax=numItermax, + stopThr=stopThr, verbose=verbose, + log=log, **kwargs) + + elif method.lower() in ['sinkhorn_stabilized', 'sinkhorn_epsilon_scaling']: + warnings.warn('Method not implemented yet. Using classic Sinkhorn Knopp') def sink(): - return sinkhorn_knopp(a, b, M, reg, alpha, numItermax=numItermax, - stopThr=stopThr, verbose=verbose, log=log, **kwargs) + return sinkhorn_knopp_unbalanced(a, b, M, reg, alpha, + numItermax=numItermax, + stopThr=stopThr, verbose=verbose, + log=log, **kwargs) + else: + raise ValueError('Unknown method. Using classic Sinkhorn Knopp') return sink() -def sinkhorn2(a, b, M, reg, alpha, method='sinkhorn', numItermax=1000, - stopThr=1e-9, verbose=False, log=False, **kwargs): +def sinkhorn_unbalanced2(a, b, M, reg, alpha, method='sinkhorn', + numItermax=1000, stopThr=1e-9, verbose=False, + log=False, **kwargs): u""" Solve the entropic regularization unbalanced optimal transport problem and return the loss @@ -173,8 +182,8 @@ def sinkhorn2(a, b, M, reg, alpha, method='sinkhorn', numItermax=1000, >>> a=[.5, .10] >>> b=[.5, .5] >>> M=[[0., 1.],[1., 0.]] - >>> ot.sinkhorn2(a, b, M, 1., 1.) - array([ 0.26894142]) + >>> ot.unbalanced.sinkhorn_unbalanced2(a, b, M, 1., 1.) + array([0.31912866]) @@ -199,23 +208,31 @@ def sinkhorn2(a, b, M, reg, alpha, method='sinkhorn', numItermax=1000, if method.lower() == 'sinkhorn': def sink(): - return sinkhorn_knopp(a, b, M, reg, alpha, numItermax=numItermax, - stopThr=stopThr, verbose=verbose, log=log, **kwargs) - else: - warnings.warn('Unknown method using classic Sinkhorn Knopp') + return sinkhorn_knopp_unbalanced(a, b, M, reg, alpha, + numItermax=numItermax, + stopThr=stopThr, verbose=verbose, + log=log, **kwargs) + + elif method.lower() in ['sinkhorn_stabilized', 'sinkhorn_epsilon_scaling']: + warnings.warn('Method not implemented yet. Using classic Sinkhorn Knopp') def sink(): - return sinkhorn_knopp(a, b, M, reg, alpha, **kwargs) + return sinkhorn_knopp_unbalanced(a, b, M, reg, alpha, + numItermax=numItermax, + stopThr=stopThr, verbose=verbose, + log=log, **kwargs) + else: + raise ValueError('Unknown method. Using classic Sinkhorn Knopp') b = np.asarray(b, dtype=np.float64) if len(b.shape) < 2: - b = b[None, :] + b = b[:, None] return sink() -def sinkhorn_knopp(a, b, M, reg, alpha, numItermax=1000, - stopThr=1e-9, verbose=False, log=False, **kwargs): +def sinkhorn_knopp_unbalanced(a, b, M, reg, alpha, numItermax=1000, + stopThr=1e-9, verbose=False, log=False, **kwargs): """ Solve the entropic regularization unbalanced optimal transport problem and return the loss @@ -273,10 +290,9 @@ def sinkhorn_knopp(a, b, M, reg, alpha, numItermax=1000, >>> a=[.5, .15] >>> b=[.5, .5] >>> M=[[0., 1.],[1., 0.]] - >>> ot.sinkhorn(a, b, M, 1., 1.) - array([[ 0.36552929, 0.13447071], - [ 0.13447071, 0.36552929]]) - + >>> ot.sinkhorn_knopp_unbalanced(a, b, M, 1., 1.) + array([[0.52761554, 0.22392482], + [0.10286295, 0.32257641]]) References ---------- @@ -303,8 +319,7 @@ def sinkhorn_knopp(a, b, M, reg, alpha, numItermax=1000, if len(b) == 0: b = np.ones(n_b, dtype=np.float64) / n_b - assert n_a == len(a) and n_b == len(b) - if b.ndim > 1: + if len(b.shape) > 1: n_hists = b.shape[1] else: n_hists = 0 @@ -315,8 +330,9 @@ def sinkhorn_knopp(a, b, M, reg, alpha, numItermax=1000, # we assume that no distances are null except those of the diagonal of # distances if n_hists: - u = np.ones((n_a, n_hists)) / n_a + u = np.ones((n_a, 1)) / n_a v = np.ones((n_b, n_hists)) / n_b + a = a.reshape(n_a, 1) else: u = np.ones(n_a) / n_a v = np.ones(n_b) / n_b @@ -332,6 +348,7 @@ def sinkhorn_knopp(a, b, M, reg, alpha, numItermax=1000, cpt = 0 err = 1. + while (err > stopThr and cpt < numItermax): uprev = u vprev = v @@ -473,7 +490,7 @@ def barycenter_unbalanced(A, M, reg, alpha, weights=None, numItermax=1000, or np.any(np.isinf(u)) or np.any(np.isinf(v))): # we have reached the machine precision # come back to previous solution and quit loop - warnings.warn('Numerical errors at iteration', cpt) + warnings.warn('Numerical errors at iteration %s' % cpt) u = uprev v = vprev break diff --git a/test/test_unbalanced.py b/test/test_unbalanced.py index b39e457..1395fe1 100644 --- a/test/test_unbalanced.py +++ b/test/test_unbalanced.py @@ -29,7 +29,8 @@ def test_unbalanced_convergence(method): G, log = ot.unbalanced.sinkhorn_unbalanced(a, b, M, reg=epsilon, alpha=alpha, stopThr=1e-10, method=method, log=True) - + loss = ot.unbalanced.sinkhorn_unbalanced2(a, b, M, epsilon, alpha, + method=method) # check fixed point equations fi = alpha / (alpha + epsilon) v_final = (b / K.T.dot(log["u"])) ** fi @@ -40,6 +41,44 @@ def test_unbalanced_convergence(method): np.testing.assert_allclose( v_final, log["v"], atol=1e-05) + # check if sinkhorn_unbalanced2 returns the correct loss + np.testing.assert_allclose((G * M).sum(), loss, atol=1e-5) + + +@pytest.mark.parametrize("method", ["sinkhorn"]) +def test_unbalanced_multiple_inputs(method): + # test generalized sinkhorn for unbalanced OT + n = 100 + rng = np.random.RandomState(42) + + x = rng.randn(n, 2) + a = ot.utils.unif(n) + + # make dists unbalanced + b = rng.rand(n, 2) + + M = ot.dist(x, x) + epsilon = 1. + alpha = 1. + K = np.exp(- M / epsilon) + + loss, log = ot.unbalanced.sinkhorn_unbalanced(a, b, M, reg=epsilon, + alpha=alpha, + stopThr=1e-10, method=method, + log=True) + # check fixed point equations + fi = alpha / (alpha + epsilon) + v_final = (b / K.T.dot(log["u"])) ** fi + + u_final = (a[:, None] / K.dot(log["v"])) ** fi + + np.testing.assert_allclose( + u_final, log["u"], atol=1e-05) + np.testing.assert_allclose( + v_final, log["v"], atol=1e-05) + + assert len(loss) == b.shape[1] + def test_unbalanced_barycenter(): # test generalized sinkhorn for unbalanced OT barycenter @@ -59,7 +98,6 @@ def test_unbalanced_barycenter(): q, log = ot.unbalanced.barycenter_unbalanced(A, M, reg=epsilon, alpha=alpha, stopThr=1e-10, log=True) - # check fixed point equations fi = alpha / (alpha + epsilon) v_final = (q[:, None] / K.T.dot(log["u"])) ** fi @@ -69,3 +107,40 @@ def test_unbalanced_barycenter(): u_final, log["u"], atol=1e-05) np.testing.assert_allclose( v_final, log["v"], atol=1e-05) + + +def test_implemented_methods(): + IMPLEMENTED_METHODS = ['sinkhorn'] + TO_BE_IMPLEMENTED_METHODS = ['sinkhorn_stabilized', + 'sinkhorn_epsilon_scaling'] + NOT_VALID_TOKENS = ['foo'] + # test generalized sinkhorn for unbalanced OT barycenter + n = 3 + rng = np.random.RandomState(42) + + x = rng.randn(n, 2) + a = ot.utils.unif(n) + + # make dists unbalanced + b = ot.utils.unif(n) * 1.5 + + M = ot.dist(x, x) + epsilon = 1. + alpha = 1. + for method in IMPLEMENTED_METHODS: + ot.unbalanced.sinkhorn_unbalanced(a, b, M, epsilon, alpha, + method=method) + ot.unbalanced.sinkhorn_unbalanced2(a, b, M, epsilon, alpha, + method=method) + with pytest.warns(UserWarning, match='not implemented'): + for method in set(TO_BE_IMPLEMENTED_METHODS): + ot.unbalanced.sinkhorn_unbalanced(a, b, M, epsilon, alpha, + method=method) + ot.unbalanced.sinkhorn_unbalanced2(a, b, M, epsilon, alpha, + method=method) + with pytest.raises(ValueError): + for method in set(NOT_VALID_TOKENS): + ot.unbalanced.sinkhorn_unbalanced(a, b, M, epsilon, alpha, + method=method) + ot.unbalanced.sinkhorn_unbalanced2(a, b, M, epsilon, alpha, + method=method) -- cgit v1.2.3 From 1fe13ed9cdda363c95c84f95fc70dcd95ac276f1 Mon Sep 17 00:00:00 2001 From: Romain Tavenard Date: Mon, 1 Jul 2019 12:16:04 +0200 Subject: Fixed doctests --- .travis.yml | 2 +- ot/bregman.py | 17 +++++++--------- ot/stochastic.py | 59 +++++++++++++++++++++++--------------------------------- ot/utils.py | 6 +++--- 4 files changed, 35 insertions(+), 49 deletions(-) (limited to 'ot/bregman.py') diff --git a/.travis.yml b/.travis.yml index d6b4232..da68c96 100644 --- a/.travis.yml +++ b/.travis.yml @@ -32,5 +32,5 @@ install: script: - python setup.py develop - flake8 examples/ ot/ test/ - - python -m pytest -v test/ ot/ --doctest-modules --cov=ot + - python -m pytest -v test/ ot/ --doctest-modules --ignore ot/gpu/ --cov=ot # - py.test ot test diff --git a/ot/bregman.py b/ot/bregman.py index 09716e6..8225967 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -1360,10 +1360,9 @@ def empirical_sinkhorn(X_s, X_t, reg, a=None, b=None, metric='sqeuclidean', numI >>> reg = 0.1 >>> X_s = np.reshape(np.arange(n_s), (n_s, 1)) >>> X_t = np.reshape(np.arange(0, n_t), (n_t, 1)) - >>> emp_sinkhorn = empirical_sinkhorn(X_s, X_t, reg, verbose=False) - >>> print(emp_sinkhorn) - >>> [[4.99977301e-01 2.26989344e-05] - [2.26989344e-05 4.99977301e-01]] + >>> empirical_sinkhorn(X_s, X_t, reg, verbose=False) # doctest: +NORMALIZE_WHITESPACE + array([[4.99977301e-01, 2.26989344e-05], + [2.26989344e-05, 4.99977301e-01]]) References @@ -1451,9 +1450,8 @@ def empirical_sinkhorn2(X_s, X_t, reg, a=None, b=None, metric='sqeuclidean', num >>> reg = 0.1 >>> X_s = np.reshape(np.arange(n_s), (n_s, 1)) >>> X_t = np.reshape(np.arange(0, n_t), (n_t, 1)) - >>> loss_sinkhorn = empirical_sinkhorn2(X_s, X_t, reg, verbose=False) - >>> print(loss_sinkhorn) - >>> [4.53978687e-05] + >>> empirical_sinkhorn2(X_s, X_t, reg, verbose=False) + array([4.53978687e-05]) References @@ -1560,9 +1558,8 @@ def empirical_sinkhorn_divergence(X_s, X_t, reg, a=None, b=None, metric='sqeucli >>> reg = 0.1 >>> X_s = np.reshape(np.arange(n_s), (n_s, 1)) >>> X_t = np.reshape(np.arange(0, n_t), (n_t, 1)) - >>> emp_sinkhorn_div = empirical_sinkhorn_divergence(X_s, X_t, reg) - >>> print(emp_sinkhorn_div) - >>> [2.99977435] + >>> empirical_sinkhorn_divergence(X_s, X_t, reg) + array([2.99977435]) References diff --git a/ot/stochastic.py b/ot/stochastic.py index 85c4230..762eb3e 100644 --- a/ot/stochastic.py +++ b/ot/stochastic.py @@ -11,7 +11,7 @@ import numpy as np def coordinate_grad_semi_dual(b, M, reg, beta, i): - ''' + r''' Compute the coordinate gradient update for regularized discrete distributions for (i, :) The function computes the gradient of the semi dual problem: @@ -51,7 +51,7 @@ def coordinate_grad_semi_dual(b, M, reg, beta, i): Examples -------- - + >>> import ot >>> n_source = 7 >>> n_target = 4 >>> reg = 1 @@ -63,8 +63,7 @@ def coordinate_grad_semi_dual(b, M, reg, beta, i): >>> Y_target = rng.randn(n_target, 2) >>> M = ot.dist(X_source, Y_target) >>> method = "ASGD" - >>> asgd_pi = stochastic.solve_semi_dual_entropic(a, b, M, reg, - method, numItermax) + >>> asgd_pi = ot.stochastic.solve_semi_dual_entropic(a, b, M, reg, method, numItermax) >>> print(asgd_pi) References @@ -84,7 +83,7 @@ def coordinate_grad_semi_dual(b, M, reg, beta, i): def sag_entropic_transport(a, b, M, reg, numItermax=10000, lr=None): - ''' + r''' Compute the SAG algorithm to solve the regularized discrete measures optimal transport max problem @@ -133,7 +132,7 @@ def sag_entropic_transport(a, b, M, reg, numItermax=10000, lr=None): Examples -------- - + >>> import ot >>> n_source = 7 >>> n_target = 4 >>> reg = 1 @@ -145,8 +144,7 @@ def sag_entropic_transport(a, b, M, reg, numItermax=10000, lr=None): >>> Y_target = rng.randn(n_target, 2) >>> M = ot.dist(X_source, Y_target) >>> method = "ASGD" - >>> asgd_pi = stochastic.solve_semi_dual_entropic(a, b, M, reg, - method, numItermax) + >>> asgd_pi = ot.stochastic.solve_semi_dual_entropic(a, b, M, reg, method, numItermax) >>> print(asgd_pi) References @@ -176,7 +174,7 @@ def sag_entropic_transport(a, b, M, reg, numItermax=10000, lr=None): def averaged_sgd_entropic_transport(a, b, M, reg, numItermax=300000, lr=None): - ''' + r''' Compute the ASGD algorithm to solve the regularized semi continous measures optimal transport max problem The function solves the following optimization problem: @@ -223,7 +221,7 @@ def averaged_sgd_entropic_transport(a, b, M, reg, numItermax=300000, lr=None): Examples -------- - + >>> import ot >>> n_source = 7 >>> n_target = 4 >>> reg = 1 @@ -235,8 +233,7 @@ def averaged_sgd_entropic_transport(a, b, M, reg, numItermax=300000, lr=None): >>> Y_target = rng.randn(n_target, 2) >>> M = ot.dist(X_source, Y_target) >>> method = "ASGD" - >>> asgd_pi = stochastic.solve_semi_dual_entropic(a, b, M, reg, - method, numItermax) + >>> asgd_pi = ot.stochastic.solve_semi_dual_entropic(a, b, M, reg, method, numItermax) >>> print(asgd_pi) References @@ -264,7 +261,7 @@ def averaged_sgd_entropic_transport(a, b, M, reg, numItermax=300000, lr=None): def c_transform_entropic(b, M, reg, beta): - ''' + r''' The goal is to recover u from the c-transform. The function computes the c_transform of a dual variable from the other @@ -303,7 +300,7 @@ def c_transform_entropic(b, M, reg, beta): Examples -------- - + >>> import ot >>> n_source = 7 >>> n_target = 4 >>> reg = 1 @@ -315,8 +312,7 @@ def c_transform_entropic(b, M, reg, beta): >>> Y_target = rng.randn(n_target, 2) >>> M = ot.dist(X_source, Y_target) >>> method = "ASGD" - >>> asgd_pi = stochastic.solve_semi_dual_entropic(a, b, M, reg, - method, numItermax) + >>> asgd_pi = ot.stochastic.solve_semi_dual_entropic(a, b, M, reg, method, numItermax) >>> print(asgd_pi) References @@ -340,7 +336,7 @@ def c_transform_entropic(b, M, reg, beta): def solve_semi_dual_entropic(a, b, M, reg, method, numItermax=10000, lr=None, log=False): - ''' + r''' Compute the transportation matrix to solve the regularized discrete measures optimal transport max problem @@ -398,7 +394,7 @@ def solve_semi_dual_entropic(a, b, M, reg, method, numItermax=10000, lr=None, Examples -------- - + >>> import ot >>> n_source = 7 >>> n_target = 4 >>> reg = 1 @@ -410,8 +406,7 @@ def solve_semi_dual_entropic(a, b, M, reg, method, numItermax=10000, lr=None, >>> Y_target = rng.randn(n_target, 2) >>> M = ot.dist(X_source, Y_target) >>> method = "ASGD" - >>> asgd_pi = stochastic.solve_semi_dual_entropic(a, b, M, reg, - method, numItermax) + >>> asgd_pi = ot.stochastic.solve_semi_dual_entropic(a, b, M, reg, method, numItermax) >>> print(asgd_pi) References @@ -451,7 +446,7 @@ def solve_semi_dual_entropic(a, b, M, reg, method, numItermax=10000, lr=None, def batch_grad_dual(a, b, M, reg, alpha, beta, batch_size, batch_alpha, batch_beta): - ''' + r''' Computes the partial gradient of the dual optimal transport problem. For each (i,j) in a batch of coordinates, the partial gradients are : @@ -506,7 +501,7 @@ def batch_grad_dual(a, b, M, reg, alpha, beta, batch_size, batch_alpha, Examples -------- - + >>> import ot >>> n_source = 7 >>> n_target = 4 >>> reg = 1 @@ -520,9 +515,7 @@ def batch_grad_dual(a, b, M, reg, alpha, beta, batch_size, batch_alpha, >>> X_source = rng.randn(n_source, 2) >>> Y_target = rng.randn(n_target, 2) >>> M = ot.dist(X_source, Y_target) - >>> sgd_dual_pi, log = stochastic.solve_dual_entropic(a, b, M, reg, - batch_size, - numItermax, lr, log) + >>> sgd_dual_pi, log = ot.stochastic.solve_dual_entropic(a, b, M, reg, batch_size, numItermax, lr, log) >>> print(log['alpha'], log['beta']) >>> print(sgd_dual_pi) @@ -548,7 +541,7 @@ def batch_grad_dual(a, b, M, reg, alpha, beta, batch_size, batch_alpha, def sgd_entropic_regularization(a, b, M, reg, batch_size, numItermax, lr): - ''' + r''' Compute the sgd algorithm to solve the regularized discrete measures optimal transport dual problem @@ -597,7 +590,7 @@ def sgd_entropic_regularization(a, b, M, reg, batch_size, numItermax, lr): Examples -------- - + >>> import ot >>> n_source = 7 >>> n_target = 4 >>> reg = 1 @@ -611,9 +604,7 @@ def sgd_entropic_regularization(a, b, M, reg, batch_size, numItermax, lr): >>> X_source = rng.randn(n_source, 2) >>> Y_target = rng.randn(n_target, 2) >>> M = ot.dist(X_source, Y_target) - >>> sgd_dual_pi, log = stochastic.solve_dual_entropic(a, b, M, reg, - batch_size, - numItermax, lr, log) + >>> sgd_dual_pi, log = ot.stochastic.solve_dual_entropic(a, b, M, reg, batch_size, numItermax, lr, log) >>> print(log['alpha'], log['beta']) >>> print(sgd_dual_pi) @@ -644,7 +635,7 @@ def sgd_entropic_regularization(a, b, M, reg, batch_size, numItermax, lr): def solve_dual_entropic(a, b, M, reg, batch_size, numItermax=10000, lr=1, log=False): - ''' + r''' Compute the transportation matrix to solve the regularized discrete measures optimal transport dual problem @@ -695,7 +686,7 @@ def solve_dual_entropic(a, b, M, reg, batch_size, numItermax=10000, lr=1, Examples -------- - + >>> import ot >>> n_source = 7 >>> n_target = 4 >>> reg = 1 @@ -709,9 +700,7 @@ def solve_dual_entropic(a, b, M, reg, batch_size, numItermax=10000, lr=1, >>> X_source = rng.randn(n_source, 2) >>> Y_target = rng.randn(n_target, 2) >>> M = ot.dist(X_source, Y_target) - >>> sgd_dual_pi, log = stochastic.solve_dual_entropic(a, b, M, reg, - batch_size, - numItermax, lr, log) + >>> sgd_dual_pi, log = ot.stochastic.solve_dual_entropic(a, b, M, reg, batch_size, numItermax, lr, log) >>> print(log['alpha'], log['beta']) >>> print(sgd_dual_pi) diff --git a/ot/utils.py b/ot/utils.py index efd1288..f21ceb9 100644 --- a/ot/utils.py +++ b/ot/utils.py @@ -285,9 +285,9 @@ class deprecated(object): The optional extra argument will be appended to the deprecation message and the docstring. Note: to use this with the default value for extra, put in an empty of parentheses: - >>> from ot.deprecation import deprecated - >>> @deprecated() - ... def some_function(): pass + >>> from ot.deprecation import deprecated # doctest: +SKIP + >>> @deprecated() # doctest: +SKIP + ... def some_function(): pass # doctest: +SKIP Parameters ---------- -- cgit v1.2.3 From 3a84263ffe4bbf2ac055dfd5a84e1b65c14f9cda Mon Sep 17 00:00:00 2001 From: Romain Tavenard Date: Mon, 1 Jul 2019 14:23:14 +0200 Subject: Set numpy array formatting version to post-1.13 --- .travis.yml | 2 ++ ot/bregman.py | 86 +++++++++++++++++++++++++++---------------------------- ot/lp/__init__.py | 22 +++++++------- ot/unbalanced.py | 10 +++---- 4 files changed, 61 insertions(+), 59 deletions(-) (limited to 'ot/bregman.py') diff --git a/.travis.yml b/.travis.yml index da68c96..f004a32 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,6 +26,8 @@ before_script: # configure a headless display to test plot generation # command to install dependencies install: - pip install -r requirements.txt + - pip install numpy>=1.14 # for numpy array formatting in doctests + - pip install scipy<1.3 # otherwise, pymanopt fails, cf - pip install flake8 pytest "pytest-cov<2.6" - pip install . # command to run tests + check syntax style diff --git a/ot/bregman.py b/ot/bregman.py index 8225967..caf4024 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -16,7 +16,7 @@ from .utils import unif, dist def sinkhorn(a, b, M, reg, method='sinkhorn', numItermax=1000, stopThr=1e-9, verbose=False, log=False, **kwargs): - u""" + r""" Solve the entropic regularization optimal transport problem and return the OT matrix The function solves the following optimization problem: @@ -73,12 +73,12 @@ def sinkhorn(a, b, M, reg, method='sinkhorn', numItermax=1000, -------- >>> import ot - >>> a=[.5,.5] - >>> b=[.5,.5] - >>> M=[[0.,1.],[1.,0.]] - >>> ot.sinkhorn(a,b,M,1) - array([[ 0.36552929, 0.13447071], - [ 0.13447071, 0.36552929]]) + >>> a=[.5, .5] + >>> b=[.5, .5] + >>> M=[[0., 1.], [1., 0.]] + >>> ot.sinkhorn(a, b, M, 1) + array([[0.36552929, 0.13447071], + [0.13447071, 0.36552929]]) References @@ -131,7 +131,7 @@ def sinkhorn(a, b, M, reg, method='sinkhorn', numItermax=1000, def sinkhorn2(a, b, M, reg, method='sinkhorn', numItermax=1000, stopThr=1e-9, verbose=False, log=False, **kwargs): - u""" + r""" Solve the entropic regularization optimal transport problem and return the loss The function solves the following optimization problem: @@ -188,11 +188,11 @@ def sinkhorn2(a, b, M, reg, method='sinkhorn', numItermax=1000, -------- >>> import ot - >>> a=[.5,.5] - >>> b=[.5,.5] - >>> M=[[0.,1.],[1.,0.]] - >>> ot.sinkhorn2(a,b,M,1) - array([ 0.26894142]) + >>> a=[.5, .5] + >>> b=[.5, .5] + >>> M=[[0., 1.], [1., 0.]] + >>> ot.sinkhorn2(a, b, M, 1) + array([0.26894142]) @@ -248,7 +248,7 @@ def sinkhorn2(a, b, M, reg, method='sinkhorn', numItermax=1000, def sinkhorn_knopp(a, b, M, reg, numItermax=1000, stopThr=1e-9, verbose=False, log=False, **kwargs): - """ + r""" Solve the entropic regularization optimal transport problem and return the OT matrix The function solves the following optimization problem: @@ -302,12 +302,12 @@ def sinkhorn_knopp(a, b, M, reg, numItermax=1000, -------- >>> import ot - >>> a=[.5,.5] - >>> b=[.5,.5] - >>> M=[[0.,1.],[1.,0.]] - >>> ot.sinkhorn(a,b,M,1) - array([[ 0.36552929, 0.13447071], - [ 0.13447071, 0.36552929]]) + >>> a=[.5, .5] + >>> b=[.5, .5] + >>> M=[[0., 1.], [1., 0.]] + >>> ot.sinkhorn(a, b, M, 1) + array([[0.36552929, 0.13447071], + [0.13447071, 0.36552929]]) References @@ -422,7 +422,7 @@ def sinkhorn_knopp(a, b, M, reg, numItermax=1000, def greenkhorn(a, b, M, reg, numItermax=10000, stopThr=1e-9, verbose=False, log=False): - """ + r""" Solve the entropic regularization optimal transport problem and return the OT matrix The algorithm used is based on the paper @@ -481,12 +481,12 @@ def greenkhorn(a, b, M, reg, numItermax=10000, stopThr=1e-9, verbose=False, log= -------- >>> import ot - >>> a=[.5,.5] - >>> b=[.5,.5] - >>> M=[[0.,1.],[1.,0.]] - >>> ot.bregman.greenkhorn(a,b,M,1) - array([[ 0.36552929, 0.13447071], - [ 0.13447071, 0.36552929]]) + >>> a=[.5, .5] + >>> b=[.5, .5] + >>> M=[[0., 1.], [1., 0.]] + >>> ot.bregman.greenkhorn(a, b, M, 1) + array([[0.36552929, 0.13447071], + [0.13447071, 0.36552929]]) References @@ -576,7 +576,7 @@ def greenkhorn(a, b, M, reg, numItermax=10000, stopThr=1e-9, verbose=False, log= def sinkhorn_stabilized(a, b, M, reg, numItermax=1000, tau=1e3, stopThr=1e-9, warmstart=None, verbose=False, print_period=20, log=False, **kwargs): - """ + r""" Solve the entropic regularization OT problem with log stabilization The function solves the following optimization problem: @@ -639,8 +639,8 @@ def sinkhorn_stabilized(a, b, M, reg, numItermax=1000, tau=1e3, stopThr=1e-9, >>> b=[.5,.5] >>> M=[[0.,1.],[1.,0.]] >>> ot.bregman.sinkhorn_stabilized(a,b,M,1) - array([[ 0.36552929, 0.13447071], - [ 0.13447071, 0.36552929]]) + array([[0.36552929, 0.13447071], + [0.13447071, 0.36552929]]) References @@ -796,7 +796,7 @@ def sinkhorn_stabilized(a, b, M, reg, numItermax=1000, tau=1e3, stopThr=1e-9, def sinkhorn_epsilon_scaling(a, b, M, reg, numItermax=100, epsilon0=1e4, numInnerItermax=100, tau=1e3, stopThr=1e-9, warmstart=None, verbose=False, print_period=10, log=False, **kwargs): - """ + r""" Solve the entropic regularization optimal transport problem with log stabilization and epsilon scaling. @@ -862,12 +862,12 @@ def sinkhorn_epsilon_scaling(a, b, M, reg, numItermax=100, epsilon0=1e4, numInne -------- >>> import ot - >>> a=[.5,.5] - >>> b=[.5,.5] - >>> M=[[0.,1.],[1.,0.]] - >>> ot.bregman.sinkhorn_epsilon_scaling(a,b,M,1) - array([[ 0.36552929, 0.13447071], - [ 0.13447071, 0.36552929]]) + >>> a=[.5, .5] + >>> b=[.5, .5] + >>> M=[[0., 1.], [1., 0.]] + >>> ot.bregman.sinkhorn_epsilon_scaling(a, b, M, 1) + array([[0.36552929, 0.13447071], + [0.13447071, 0.36552929]]) References @@ -989,7 +989,7 @@ def projC(gamma, q): def barycenter(A, M, reg, weights=None, numItermax=1000, stopThr=1e-4, verbose=False, log=False): - """Compute the entropic regularized wasserstein barycenter of distributions A + r"""Compute the entropic regularized wasserstein barycenter of distributions A The function solves the following optimization problem: @@ -1084,7 +1084,7 @@ def barycenter(A, M, reg, weights=None, numItermax=1000, def convolutional_barycenter2d(A, reg, weights=None, numItermax=10000, stopThr=1e-9, stabThr=1e-30, verbose=False, log=False): - """Compute the entropic regularized wasserstein barycenter of distributions A + r"""Compute the entropic regularized wasserstein barycenter of distributions A where A is a collection of 2D images. The function solves the following optimization problem: @@ -1195,7 +1195,7 @@ def convolutional_barycenter2d(A, reg, weights=None, numItermax=10000, stopThr=1 def unmix(a, D, M, M0, h0, reg, reg0, alpha, numItermax=1000, stopThr=1e-3, verbose=False, log=False): - """ + r""" Compute the unmixing of an observation with a given dictionary using Wasserstein distance The function solve the following optimization problem: @@ -1302,7 +1302,7 @@ def unmix(a, D, M, M0, h0, reg, reg0, alpha, numItermax=1000, def empirical_sinkhorn(X_s, X_t, reg, a=None, b=None, metric='sqeuclidean', numIterMax=10000, stopThr=1e-9, verbose=False, log=False, **kwargs): - ''' + r''' Solve the entropic regularization optimal transport problem and return the OT matrix from empirical data @@ -1391,7 +1391,7 @@ def empirical_sinkhorn(X_s, X_t, reg, a=None, b=None, metric='sqeuclidean', numI def empirical_sinkhorn2(X_s, X_t, reg, a=None, b=None, metric='sqeuclidean', numIterMax=10000, stopThr=1e-9, verbose=False, log=False, **kwargs): - ''' + r''' Solve the entropic regularization optimal transport problem from empirical data and return the OT loss @@ -1480,7 +1480,7 @@ def empirical_sinkhorn2(X_s, X_t, reg, a=None, b=None, metric='sqeuclidean', num def empirical_sinkhorn_divergence(X_s, X_t, reg, a=None, b=None, metric='sqeuclidean', numIterMax=10000, stopThr=1e-9, verbose=False, log=False, **kwargs): - ''' + r''' Compute the sinkhorn divergence loss from empirical data The function solves the following optimization problems and return the diff --git a/ot/lp/__init__.py b/ot/lp/__init__.py index a3f5b8d..8ec286b 100644 --- a/ot/lp/__init__.py +++ b/ot/lp/__init__.py @@ -25,7 +25,7 @@ __all__=['emd', 'emd2', 'barycenter', 'free_support_barycenter', 'cvx', def emd(a, b, M, numItermax=100000, log=False): - """Solves the Earth Movers distance problem and returns the OT matrix + r"""Solves the Earth Movers distance problem and returns the OT matrix .. math:: @@ -76,8 +76,8 @@ def emd(a, b, M, numItermax=100000, log=False): >>> b=[.5,.5] >>> M=[[0.,1.],[1.,0.]] >>> ot.emd(a,b,M) - array([[ 0.5, 0. ], - [ 0. , 0.5]]) + array([[0.5, 0. ], + [0. , 0.5]]) References ---------- @@ -117,7 +117,7 @@ def emd(a, b, M, numItermax=100000, log=False): def emd2(a, b, M, processes=multiprocessing.cpu_count(), numItermax=100000, log=False, return_matrix=False): - """Solves the Earth Movers distance problem and returns the loss + r"""Solves the Earth Movers distance problem and returns the loss .. math:: \gamma = arg\min_\gamma <\gamma,M>_F @@ -315,7 +315,7 @@ def free_support_barycenter(measures_locations, measures_weights, X_init, b=None def emd_1d(x_a, x_b, a=None, b=None, metric='sqeuclidean', p=1., dense=True, log=False): - """Solves the Earth Movers distance problem between 1d measures and returns + r"""Solves the Earth Movers distance problem between 1d measures and returns the OT matrix @@ -381,11 +381,11 @@ def emd_1d(x_a, x_b, a=None, b=None, metric='sqeuclidean', p=1., dense=True, >>> x_a = [2., 0.] >>> x_b = [0., 3.] >>> ot.emd_1d(x_a, x_b, a, b) - array([[0. , 0.5], - [0.5, 0. ]]) + array([[0. , 0.5], + [0.5, 0. ]]) >>> ot.emd_1d(x_a, x_b) - array([[0. , 0.5], - [0.5, 0. ]]) + array([[0. , 0.5], + [0.5, 0. ]]) References ---------- @@ -435,7 +435,7 @@ def emd_1d(x_a, x_b, a=None, b=None, metric='sqeuclidean', p=1., dense=True, def emd2_1d(x_a, x_b, a=None, b=None, metric='sqeuclidean', p=1., dense=True, log=False): - """Solves the Earth Movers distance problem between 1d measures and returns + r"""Solves the Earth Movers distance problem between 1d measures and returns the loss @@ -530,7 +530,7 @@ def emd2_1d(x_a, x_b, a=None, b=None, metric='sqeuclidean', p=1., dense=True, def wasserstein_1d(x_a, x_b, a=None, b=None, p=1.): - """Solves the p-Wasserstein distance problem between 1d measures and returns + r"""Solves the p-Wasserstein distance problem between 1d measures and returns the distance diff --git a/ot/unbalanced.py b/ot/unbalanced.py index 484ce95..b2b7b10 100644 --- a/ot/unbalanced.py +++ b/ot/unbalanced.py @@ -13,7 +13,7 @@ import numpy as np def sinkhorn_unbalanced(a, b, M, reg, alpha, method='sinkhorn', numItermax=1000, stopThr=1e-9, verbose=False, log=False, **kwargs): - u""" + r""" Solve the unbalanced entropic regularization optimal transport problem and return the loss The function solves the following optimization problem: @@ -75,7 +75,7 @@ def sinkhorn_unbalanced(a, b, M, reg, alpha, method='sinkhorn', numItermax=1000, >>> M=[[0., 1.], [1., 0.]] >>> ot.sinkhorn_unbalanced(a, b, M, 1, 1) array([[0.51122823, 0.18807035], - [0.18807035, 0.51122823]]) + [0.18807035, 0.51122823]]) References @@ -122,7 +122,7 @@ def sinkhorn_unbalanced(a, b, M, reg, alpha, method='sinkhorn', numItermax=1000, def sinkhorn_unbalanced2(a, b, M, reg, alpha, method='sinkhorn', numItermax=1000, stopThr=1e-9, verbose=False, log=False, **kwargs): - u""" + r""" Solve the entropic regularization unbalanced optimal transport problem and return the loss The function solves the following optimization problem: @@ -233,7 +233,7 @@ def sinkhorn_unbalanced2(a, b, M, reg, alpha, method='sinkhorn', def sinkhorn_knopp_unbalanced(a, b, M, reg, alpha, numItermax=1000, stopThr=1e-9, verbose=False, log=False, **kwargs): - """ + r""" Solve the entropic regularization unbalanced optimal transport problem and return the loss The function solves the following optimization problem: @@ -401,7 +401,7 @@ def sinkhorn_knopp_unbalanced(a, b, M, reg, alpha, numItermax=1000, def barycenter_unbalanced(A, M, reg, alpha, weights=None, numItermax=1000, stopThr=1e-4, verbose=False, log=False): - """Compute the entropic regularized unbalanced wasserstein barycenter of distributions A + r"""Compute the entropic regularized unbalanced wasserstein barycenter of distributions A The function solves the following optimization problem: -- cgit v1.2.3 From a08375c8dc7594e247e586fcc4d65a96771d25c7 Mon Sep 17 00:00:00 2001 From: Romain Tavenard Date: Mon, 1 Jul 2019 15:36:55 +0200 Subject: Fixed all doctests assuming functions are working properly (actually tested in tests/) --- .travis.yml | 5 +- ot/bregman.py | 2 +- ot/stochastic.py | 155 +++++++++++++++++++++++++++++++++++-------------------- ot/unbalanced.py | 2 +- 4 files changed, 104 insertions(+), 60 deletions(-) (limited to 'ot/bregman.py') diff --git a/.travis.yml b/.travis.yml index 275109c..2c6a5c8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,8 +26,9 @@ before_script: # configure a headless display to test plot generation # command to install dependencies install: - pip install -r requirements.txt - - pip install numpy>=1.14 # for numpy array formatting in doctests - - pip install "scipy<1.3" # otherwise, pymanopt fails, cf + - pip install numpy>=1.14 "scipy<1.3" # for numpy array formatting in doctests + # ^ scipy version: otherwise, pymanopt fails, cf + - python -c "import numpy; import scipy; print('numpy: ', numpy.__version__); print('scipy: ', scipy.__version__)" - pip install flake8 pytest "pytest-cov<2.6" - pip install . # command to run tests + check syntax style diff --git a/ot/bregman.py b/ot/bregman.py index caf4024..50f8389 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -1559,7 +1559,7 @@ def empirical_sinkhorn_divergence(X_s, X_t, reg, a=None, b=None, metric='sqeucli >>> X_s = np.reshape(np.arange(n_s), (n_s, 1)) >>> X_t = np.reshape(np.arange(0, n_t), (n_t, 1)) >>> empirical_sinkhorn_divergence(X_s, X_t, reg) - array([2.99977435]) + array([1.49988718]) References diff --git a/ot/stochastic.py b/ot/stochastic.py index 762eb3e..bf3e7a7 100644 --- a/ot/stochastic.py +++ b/ot/stochastic.py @@ -52,19 +52,23 @@ def coordinate_grad_semi_dual(b, M, reg, beta, i): Examples -------- >>> import ot + >>> np.random.seed(0) >>> n_source = 7 >>> n_target = 4 - >>> reg = 1 - >>> numItermax = 300000 >>> a = ot.utils.unif(n_source) >>> b = ot.utils.unif(n_target) - >>> rng = np.random.RandomState(0) - >>> X_source = rng.randn(n_source, 2) - >>> Y_target = rng.randn(n_target, 2) + >>> X_source = np.random.randn(n_source, 2) + >>> Y_target = np.random.randn(n_target, 2) >>> M = ot.dist(X_source, Y_target) - >>> method = "ASGD" - >>> asgd_pi = ot.stochastic.solve_semi_dual_entropic(a, b, M, reg, method, numItermax) - >>> print(asgd_pi) + >>> ot.stochastic.solve_semi_dual_entropic(a, b, M, reg=1, method="ASGD", numItermax=300000) + array([[2.53942342e-02, 9.98640673e-02, 1.75945647e-02, 4.27664307e-06], + [1.21556999e-01, 1.26350515e-02, 1.30491795e-03, 7.36017394e-03], + [3.54070702e-03, 7.63581358e-02, 6.29581672e-02, 1.32812798e-07], + [2.60578198e-02, 3.35916645e-02, 8.28023223e-02, 4.05336238e-04], + [9.86808864e-03, 7.59774324e-04, 1.08702729e-02, 1.21359007e-01], + [2.17218856e-02, 9.12931802e-04, 1.87962526e-03, 1.18342700e-01], + [4.14237512e-02, 2.67487857e-02, 7.23016955e-02, 2.38291052e-03]]) + References ---------- @@ -133,19 +137,22 @@ def sag_entropic_transport(a, b, M, reg, numItermax=10000, lr=None): Examples -------- >>> import ot + >>> np.random.seed(0) >>> n_source = 7 >>> n_target = 4 - >>> reg = 1 - >>> numItermax = 300000 >>> a = ot.utils.unif(n_source) >>> b = ot.utils.unif(n_target) - >>> rng = np.random.RandomState(0) - >>> X_source = rng.randn(n_source, 2) - >>> Y_target = rng.randn(n_target, 2) + >>> X_source = np.random.randn(n_source, 2) + >>> Y_target = np.random.randn(n_target, 2) >>> M = ot.dist(X_source, Y_target) - >>> method = "ASGD" - >>> asgd_pi = ot.stochastic.solve_semi_dual_entropic(a, b, M, reg, method, numItermax) - >>> print(asgd_pi) + >>> ot.stochastic.solve_semi_dual_entropic(a, b, M, reg=1, method="ASGD", numItermax=300000) + array([[2.53942342e-02, 9.98640673e-02, 1.75945647e-02, 4.27664307e-06], + [1.21556999e-01, 1.26350515e-02, 1.30491795e-03, 7.36017394e-03], + [3.54070702e-03, 7.63581358e-02, 6.29581672e-02, 1.32812798e-07], + [2.60578198e-02, 3.35916645e-02, 8.28023223e-02, 4.05336238e-04], + [9.86808864e-03, 7.59774324e-04, 1.08702729e-02, 1.21359007e-01], + [2.17218856e-02, 9.12931802e-04, 1.87962526e-03, 1.18342700e-01], + [4.14237512e-02, 2.67487857e-02, 7.23016955e-02, 2.38291052e-03]]) References ---------- @@ -222,19 +229,22 @@ def averaged_sgd_entropic_transport(a, b, M, reg, numItermax=300000, lr=None): Examples -------- >>> import ot + >>> np.random.seed(0) >>> n_source = 7 >>> n_target = 4 - >>> reg = 1 - >>> numItermax = 300000 >>> a = ot.utils.unif(n_source) >>> b = ot.utils.unif(n_target) - >>> rng = np.random.RandomState(0) - >>> X_source = rng.randn(n_source, 2) - >>> Y_target = rng.randn(n_target, 2) + >>> X_source = np.random.randn(n_source, 2) + >>> Y_target = np.random.randn(n_target, 2) >>> M = ot.dist(X_source, Y_target) - >>> method = "ASGD" - >>> asgd_pi = ot.stochastic.solve_semi_dual_entropic(a, b, M, reg, method, numItermax) - >>> print(asgd_pi) + >>> ot.stochastic.solve_semi_dual_entropic(a, b, M, reg=1, method="ASGD", numItermax=300000) + array([[2.53942342e-02, 9.98640673e-02, 1.75945647e-02, 4.27664307e-06], + [1.21556999e-01, 1.26350515e-02, 1.30491795e-03, 7.36017394e-03], + [3.54070702e-03, 7.63581358e-02, 6.29581672e-02, 1.32812798e-07], + [2.60578198e-02, 3.35916645e-02, 8.28023223e-02, 4.05336238e-04], + [9.86808864e-03, 7.59774324e-04, 1.08702729e-02, 1.21359007e-01], + [2.17218856e-02, 9.12931802e-04, 1.87962526e-03, 1.18342700e-01], + [4.14237512e-02, 2.67487857e-02, 7.23016955e-02, 2.38291052e-03]]) References ---------- @@ -301,19 +311,22 @@ def c_transform_entropic(b, M, reg, beta): Examples -------- >>> import ot + >>> np.random.seed(0) >>> n_source = 7 >>> n_target = 4 - >>> reg = 1 - >>> numItermax = 300000 >>> a = ot.utils.unif(n_source) >>> b = ot.utils.unif(n_target) - >>> rng = np.random.RandomState(0) - >>> X_source = rng.randn(n_source, 2) - >>> Y_target = rng.randn(n_target, 2) + >>> X_source = np.random.randn(n_source, 2) + >>> Y_target = np.random.randn(n_target, 2) >>> M = ot.dist(X_source, Y_target) - >>> method = "ASGD" - >>> asgd_pi = ot.stochastic.solve_semi_dual_entropic(a, b, M, reg, method, numItermax) - >>> print(asgd_pi) + >>> ot.stochastic.solve_semi_dual_entropic(a, b, M, reg=1, method="ASGD", numItermax=300000) + array([[2.53942342e-02, 9.98640673e-02, 1.75945647e-02, 4.27664307e-06], + [1.21556999e-01, 1.26350515e-02, 1.30491795e-03, 7.36017394e-03], + [3.54070702e-03, 7.63581358e-02, 6.29581672e-02, 1.32812798e-07], + [2.60578198e-02, 3.35916645e-02, 8.28023223e-02, 4.05336238e-04], + [9.86808864e-03, 7.59774324e-04, 1.08702729e-02, 1.21359007e-01], + [2.17218856e-02, 9.12931802e-04, 1.87962526e-03, 1.18342700e-01], + [4.14237512e-02, 2.67487857e-02, 7.23016955e-02, 2.38291052e-03]]) References ---------- @@ -395,19 +408,22 @@ def solve_semi_dual_entropic(a, b, M, reg, method, numItermax=10000, lr=None, Examples -------- >>> import ot + >>> np.random.seed(0) >>> n_source = 7 >>> n_target = 4 - >>> reg = 1 - >>> numItermax = 300000 >>> a = ot.utils.unif(n_source) >>> b = ot.utils.unif(n_target) - >>> rng = np.random.RandomState(0) - >>> X_source = rng.randn(n_source, 2) - >>> Y_target = rng.randn(n_target, 2) + >>> X_source = np.random.randn(n_source, 2) + >>> Y_target = np.random.randn(n_target, 2) >>> M = ot.dist(X_source, Y_target) - >>> method = "ASGD" - >>> asgd_pi = ot.stochastic.solve_semi_dual_entropic(a, b, M, reg, method, numItermax) - >>> print(asgd_pi) + >>> ot.stochastic.solve_semi_dual_entropic(a, b, M, reg=1, method="ASGD", numItermax=300000) + array([[2.53942342e-02, 9.98640673e-02, 1.75945647e-02, 4.27664307e-06], + [1.21556999e-01, 1.26350515e-02, 1.30491795e-03, 7.36017394e-03], + [3.54070702e-03, 7.63581358e-02, 6.29581672e-02, 1.32812798e-07], + [2.60578198e-02, 3.35916645e-02, 8.28023223e-02, 4.05336238e-04], + [9.86808864e-03, 7.59774324e-04, 1.08702729e-02, 1.21359007e-01], + [2.17218856e-02, 9.12931802e-04, 1.87962526e-03, 1.18342700e-01], + [4.14237512e-02, 2.67487857e-02, 7.23016955e-02, 2.38291052e-03]]) References ---------- @@ -502,22 +518,28 @@ def batch_grad_dual(a, b, M, reg, alpha, beta, batch_size, batch_alpha, Examples -------- >>> import ot + >>> np.random.seed(0) >>> n_source = 7 >>> n_target = 4 - >>> reg = 1 - >>> numItermax = 20000 - >>> lr = 0.1 - >>> batch_size = 3 - >>> log = True >>> a = ot.utils.unif(n_source) >>> b = ot.utils.unif(n_target) - >>> rng = np.random.RandomState(0) - >>> X_source = rng.randn(n_source, 2) - >>> Y_target = rng.randn(n_target, 2) + >>> X_source = np.random.randn(n_source, 2) + >>> Y_target = np.random.randn(n_target, 2) >>> M = ot.dist(X_source, Y_target) - >>> sgd_dual_pi, log = ot.stochastic.solve_dual_entropic(a, b, M, reg, batch_size, numItermax, lr, log) - >>> print(log['alpha'], log['beta']) - >>> print(sgd_dual_pi) + >>> sgd_dual_pi, log = ot.stochastic.solve_dual_entropic(a, b, M, reg=1, batch_size=3, numItermax=30000, lr=0.1, log=True) + >>> log['alpha'] + array([0.71759102, 1.57057384, 0.85576566, 0.1208211 , 0.59190466, + 1.197148 , 0.17805133]) + >>> log['beta'] + array([0.49741367, 0.57478564, 1.40075528, 2.75890102]) + >>> sgd_dual_pi + array([[2.09730063e-02, 8.38169324e-02, 7.50365455e-03, 8.72731415e-09], + [5.58432437e-03, 5.89881299e-04, 3.09558411e-05, 8.35469849e-07], + [3.26489515e-03, 7.15536035e-02, 2.99778211e-02, 3.02601593e-10], + [4.05390622e-02, 5.31085068e-02, 6.65191787e-02, 1.55812785e-06], + [7.82299812e-02, 6.12099102e-03, 4.44989098e-02, 2.37719187e-03], + [5.06266486e-02, 2.16230494e-03, 2.26215141e-03, 6.81514609e-04], + [6.06713990e-02, 3.98139808e-02, 5.46829338e-02, 8.62371424e-06]]) References ---------- @@ -526,7 +548,6 @@ def batch_grad_dual(a, b, M, reg, alpha, beta, batch_size, batch_alpha, International Conference on Learning Representation (2018), arXiv preprint arxiv:1711.02283. ''' - G = - (np.exp((alpha[batch_alpha, None] + beta[None, batch_beta] - M[batch_alpha, :][:, batch_beta]) / reg) * a[batch_alpha, None] * b[None, batch_beta]) @@ -605,8 +626,19 @@ def sgd_entropic_regularization(a, b, M, reg, batch_size, numItermax, lr): >>> Y_target = rng.randn(n_target, 2) >>> M = ot.dist(X_source, Y_target) >>> sgd_dual_pi, log = ot.stochastic.solve_dual_entropic(a, b, M, reg, batch_size, numItermax, lr, log) - >>> print(log['alpha'], log['beta']) - >>> print(sgd_dual_pi) + >>> log['alpha'] + array([0.64171798, 1.27932201, 0.78132257, 0.15638935, 0.54888354, + 1.03663469, 0.20595781]) + >>> log['beta'] + array([0.51207194, 0.58033189, 1.28922676, 2.26859736]) + >>> sgd_dual_pi + array([[1.97276541e-02, 7.81248547e-02, 6.22136048e-03, 4.95442423e-09], + [4.23494310e-03, 4.43286263e-04, 2.06927079e-05, 3.82389139e-07], + [3.07542414e-03, 6.67897769e-02, 2.48904999e-02, 1.72030247e-10], + [4.26271990e-02, 5.53375455e-02, 6.16535024e-02, 9.88812650e-07], + [7.60423265e-02, 5.89585256e-03, 3.81267087e-02, 1.39458256e-03], + [4.37557504e-02, 1.85189176e-03, 1.72335760e-03, 3.55491279e-04], + [6.33096109e-02, 4.11683954e-02, 5.02962051e-02, 5.43097516e-06]]) References ---------- @@ -701,8 +733,19 @@ def solve_dual_entropic(a, b, M, reg, batch_size, numItermax=10000, lr=1, >>> Y_target = rng.randn(n_target, 2) >>> M = ot.dist(X_source, Y_target) >>> sgd_dual_pi, log = ot.stochastic.solve_dual_entropic(a, b, M, reg, batch_size, numItermax, lr, log) - >>> print(log['alpha'], log['beta']) - >>> print(sgd_dual_pi) + >>> log['alpha'] + array([0.64057733, 1.2683513 , 0.75610161, 0.16024284, 0.54926534, + 1.0514201 , 0.19958936]) + >>> log['beta'] + array([0.51372571, 0.58843489, 1.27993921, 2.24344807]) + >>> sgd_dual_pi + array([[1.97377795e-02, 7.86706853e-02, 6.15682001e-03, 4.82586997e-09], + [4.19566963e-03, 4.42016865e-04, 2.02777272e-05, 3.68823708e-07], + [3.00379244e-03, 6.56562018e-02, 2.40462171e-02, 1.63579656e-10], + [4.28626062e-02, 5.60031599e-02, 6.13193826e-02, 9.67977735e-07], + [7.61972739e-02, 5.94609051e-03, 3.77886693e-02, 1.36046648e-03], + [4.44810042e-02, 1.89476742e-03, 1.73285847e-03, 3.51826036e-04], + [6.30118293e-02, 4.12398660e-02, 4.95148998e-02, 5.26247246e-06]]) References ---------- diff --git a/ot/unbalanced.py b/ot/unbalanced.py index b2b7b10..4a2af8a 100644 --- a/ot/unbalanced.py +++ b/ot/unbalanced.py @@ -290,7 +290,7 @@ def sinkhorn_knopp_unbalanced(a, b, M, reg, alpha, numItermax=1000, >>> a=[.5, .15] >>> b=[.5, .5] >>> M=[[0., 1.],[1., 0.]] - >>> ot.sinkhorn_knopp_unbalanced(a, b, M, 1., 1.) + >>> ot.unbalanced.sinkhorn_knopp_unbalanced(a, b, M, 1., 1.) array([[0.52761554, 0.22392482], [0.10286295, 0.32257641]]) -- cgit v1.2.3 From 5b6eb56f2d4bfdaeaa45970c386c42c21d7d1caf Mon Sep 17 00:00:00 2001 From: Romain Tavenard Date: Tue, 2 Jul 2019 09:53:15 +0200 Subject: Fix tolerance in doctest --- ot/bregman.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'ot/bregman.py') diff --git a/ot/bregman.py b/ot/bregman.py index 50f8389..13dfa3b 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -1558,8 +1558,8 @@ def empirical_sinkhorn_divergence(X_s, X_t, reg, a=None, b=None, metric='sqeucli >>> reg = 0.1 >>> X_s = np.reshape(np.arange(n_s), (n_s, 1)) >>> X_t = np.reshape(np.arange(0, n_t), (n_t, 1)) - >>> empirical_sinkhorn_divergence(X_s, X_t, reg) - array([1.49988718]) + >>> empirical_sinkhorn_divergence(X_s, X_t, reg) # doctest: +ELLIPSIS + array([1.499...]) References -- cgit v1.2.3 From 06fab4c1e5efbe79f91589917fba01c3fb300a87 Mon Sep 17 00:00:00 2001 From: Alexandre Gramfort Date: Tue, 9 Jul 2019 17:20:02 +0200 Subject: more --- ot/bregman.py | 112 ++++++++++++++++++++++++++------------------------------- ot/datasets.py | 32 +++++++---------- ot/optim.py | 32 ++++++++--------- ot/plot.py | 6 ++-- 4 files changed, 80 insertions(+), 102 deletions(-) (limited to 'ot/bregman.py') diff --git a/ot/bregman.py b/ot/bregman.py index 13dfa3b..b67074f 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -40,12 +40,12 @@ def sinkhorn(a, b, M, reg, method='sinkhorn', numItermax=1000, Parameters ---------- - a : np.ndarray (ns,) + a : ndarray, shape (ns,) samples weights in the source domain - b : np.ndarray (nt,) or np.ndarray (nt,nbb) + b : ndarray, shape (nt,) or ndarray, shape (nt, nbb) samples in the target domain, compute sinkhorn with multiple targets and fixed M if b is a matrix (return OT loss + dual variables in log) - M : np.ndarray (ns,nt) + M : ndarray, shape (ns, nt) loss matrix reg : float Regularization term >0 @@ -64,7 +64,7 @@ def sinkhorn(a, b, M, reg, method='sinkhorn', numItermax=1000, Returns ------- - gamma : (ns x nt) ndarray + gamma : ndarray, shape (ns, nt) Optimal transportation matrix for the given parameters log : dict log dictionary return only if log==True in parameters @@ -155,12 +155,12 @@ def sinkhorn2(a, b, M, reg, method='sinkhorn', numItermax=1000, Parameters ---------- - a : np.ndarray (ns,) + a : ndarray, shape (ns,) samples weights in the source domain - b : np.ndarray (nt,) or np.ndarray (nt,nbb) + b : ndarray, shape (nt,) or ndarray, shape (nt, nbb) samples in the target domain, compute sinkhorn with multiple targets and fixed M if b is a matrix (return OT loss + dual variables in log) - M : np.ndarray (ns,nt) + M : ndarray, shape (ns, nt) loss matrix reg : float Regularization term >0 @@ -176,7 +176,6 @@ def sinkhorn2(a, b, M, reg, method='sinkhorn', numItermax=1000, log : bool, optional record log if True - Returns ------- W : (nt) ndarray or float @@ -272,12 +271,12 @@ def sinkhorn_knopp(a, b, M, reg, numItermax=1000, Parameters ---------- - a : np.ndarray (ns,) + a : ndarray, shape (ns,) samples weights in the source domain - b : np.ndarray (nt,) or np.ndarray (nt,nbb) + b : ndarray, shape (nt,) or ndarray, shape (nt, nbb) samples in the target domain, compute sinkhorn with multiple targets and fixed M if b is a matrix (return OT loss + dual variables in log) - M : np.ndarray (ns,nt) + M : ndarray, shape (ns, nt) loss matrix reg : float Regularization term >0 @@ -290,10 +289,9 @@ def sinkhorn_knopp(a, b, M, reg, numItermax=1000, log : bool, optional record log if True - Returns ------- - gamma : (ns x nt) ndarray + gamma : ndarray, shape (ns, nt) Optimal transportation matrix for the given parameters log : dict log dictionary return only if log==True in parameters @@ -453,12 +451,12 @@ def greenkhorn(a, b, M, reg, numItermax=10000, stopThr=1e-9, verbose=False, log= Parameters ---------- - a : np.ndarray (ns,) + a : ndarray, shape (ns,) samples weights in the source domain - b : np.ndarray (nt,) or np.ndarray (nt,nbb) + b : ndarray, shape (nt,) or ndarray, shape (nt, nbb) samples in the target domain, compute sinkhorn with multiple targets and fixed M if b is a matrix (return OT loss + dual variables in log) - M : np.ndarray (ns,nt) + M : ndarray, shape (ns, nt) loss matrix reg : float Regularization term >0 @@ -469,10 +467,9 @@ def greenkhorn(a, b, M, reg, numItermax=10000, stopThr=1e-9, verbose=False, log= log : bool, optional record log if True - Returns ------- - gamma : (ns x nt) ndarray + gamma : ndarray, shape (ns, nt) Optimal transportation matrix for the given parameters log : dict log dictionary return only if log==True in parameters @@ -602,11 +599,11 @@ def sinkhorn_stabilized(a, b, M, reg, numItermax=1000, tau=1e3, stopThr=1e-9, Parameters ---------- - a : np.ndarray (ns,) + a : ndarray, shape (ns,) samples weights in the source domain - b : np.ndarray (nt,) + b : ndarray, shape (nt,) samples in the target domain - M : np.ndarray (ns,nt) + M : ndarray, shape (ns, nt) loss matrix reg : float Regularization term >0 @@ -623,10 +620,9 @@ def sinkhorn_stabilized(a, b, M, reg, numItermax=1000, tau=1e3, stopThr=1e-9, log : bool, optional record log if True - Returns ------- - gamma : (ns x nt) ndarray + gamma : ndarray, shape (ns, nt) Optimal transportation matrix for the given parameters log : dict log dictionary return only if log==True in parameters @@ -823,11 +819,11 @@ def sinkhorn_epsilon_scaling(a, b, M, reg, numItermax=100, epsilon0=1e4, numInne Parameters ---------- - a : np.ndarray (ns,) + a : ndarray, shape (ns,) samples weights in the source domain - b : np.ndarray (nt,) + b : ndarray, shape (nt,) samples in the target domain - M : np.ndarray (ns,nt) + M : ndarray, shape (ns, nt) loss matrix reg : float Regularization term >0 @@ -835,7 +831,7 @@ def sinkhorn_epsilon_scaling(a, b, M, reg, numItermax=100, epsilon0=1e4, numInne thershold for max value in u or v for log scaling tau : float thershold for max value in u or v for log scaling - warmstart : tible of vectors + warmstart : tuple of vectors if given then sarting values for alpha an beta log scalings numItermax : int, optional Max number of iterations @@ -850,10 +846,9 @@ def sinkhorn_epsilon_scaling(a, b, M, reg, numItermax=100, epsilon0=1e4, numInne log : bool, optional record log if True - Returns ------- - gamma : (ns x nt) ndarray + gamma : ndarray, shape (ns, nt) Optimal transportation matrix for the given parameters log : dict log dictionary return only if log==True in parameters @@ -1006,13 +1001,13 @@ def barycenter(A, M, reg, weights=None, numItermax=1000, Parameters ---------- - A : np.ndarray (d,n) + A : ndarray, shape (d,n) n training distributions a_i of size d - M : np.ndarray (d,d) + M : ndarray, shape (d,d) loss matrix for OT reg : float Regularization term >0 - weights : np.ndarray (n,) + weights : ndarray, shape (n,) Weights of each histogram a_i on the simplex (barycentric coodinates) numItermax : int, optional Max number of iterations @@ -1102,11 +1097,11 @@ def convolutional_barycenter2d(A, reg, weights=None, numItermax=10000, stopThr=1 Parameters ---------- - A : np.ndarray (n,w,h) + A : ndarray, shape (n, w, h) n distributions (2D images) of size w x h reg : float Regularization term >0 - weights : np.ndarray (n,) + weights : ndarray, shape (n,) Weights of each image on the simplex (barycentric coodinates) numItermax : int, optional Max number of iterations @@ -1119,15 +1114,13 @@ def convolutional_barycenter2d(A, reg, weights=None, numItermax=10000, stopThr=1 log : bool, optional record log if True - Returns ------- - a : (w,h) ndarray + a : ndarray, shape (w, h) 2D Wasserstein barycenter log : dict log dictionary return only if log==True in parameters - References ---------- @@ -1217,15 +1210,15 @@ def unmix(a, D, M, M0, h0, reg, reg0, alpha, numItermax=1000, Parameters ---------- - a : np.ndarray (d) + a : ndarray, shape (d) observed distribution - D : np.ndarray (d,n) + D : ndarray, shape (d, n) dictionary matrix - M : np.ndarray (d,d) + M : ndarray, shape (d, d) loss matrix - M0 : np.ndarray (n,n) + M0 : ndarray, shape (n, n) loss matrix - h0 : np.ndarray (n,) + h0 : ndarray, shape (n,) prior on h reg : float Regularization term >0 (Wasserstein data fitting) @@ -1245,7 +1238,7 @@ def unmix(a, D, M, M0, h0, reg, reg0, alpha, numItermax=1000, Returns ------- - a : (d,) ndarray + a : ndarray, shape (d,) Wasserstein barycenter log : dict log dictionary return only if log==True in parameters @@ -1325,15 +1318,15 @@ def empirical_sinkhorn(X_s, X_t, reg, a=None, b=None, metric='sqeuclidean', numI Parameters ---------- - X_s : np.ndarray (ns, d) + X_s : ndarray, shape (ns, d) samples in the source domain - X_t : np.ndarray (nt, d) + X_t : ndarray, shape (nt, d) samples in the target domain reg : float Regularization term >0 - a : np.ndarray (ns,) + a : ndarray, shape (ns,) samples weights in the source domain - b : np.ndarray (nt,) + b : ndarray, shape (nt,) samples weights in the target domain numItermax : int, optional Max number of iterations @@ -1347,7 +1340,7 @@ def empirical_sinkhorn(X_s, X_t, reg, a=None, b=None, metric='sqeuclidean', numI Returns ------- - gamma : (ns x nt) ndarray + gamma : ndarray, shape (ns, nt) Regularized optimal transportation matrix for the given parameters log : dict log dictionary return only if log==True in parameters @@ -1415,15 +1408,15 @@ def empirical_sinkhorn2(X_s, X_t, reg, a=None, b=None, metric='sqeuclidean', num Parameters ---------- - X_s : np.ndarray (ns, d) + X_s : ndarray, shape (ns, d) samples in the source domain - X_t : np.ndarray (nt, d) + X_t : ndarray, shape (nt, d) samples in the target domain reg : float Regularization term >0 - a : np.ndarray (ns,) + a : ndarray, shape (ns,) samples weights in the source domain - b : np.ndarray (nt,) + b : ndarray, shape (nt,) samples weights in the target domain numItermax : int, optional Max number of iterations @@ -1437,7 +1430,7 @@ def empirical_sinkhorn2(X_s, X_t, reg, a=None, b=None, metric='sqeuclidean', num Returns ------- - gamma : (ns x nt) ndarray + gamma : ndarray, shape (ns, nt) Regularized optimal transportation matrix for the given parameters log : dict log dictionary return only if log==True in parameters @@ -1523,15 +1516,15 @@ def empirical_sinkhorn_divergence(X_s, X_t, reg, a=None, b=None, metric='sqeucli Parameters ---------- - X_s : np.ndarray (ns, d) + X_s : ndarray, shape (ns, d) samples in the source domain - X_t : np.ndarray (nt, d) + X_t : ndarray, shape (nt, d) samples in the target domain reg : float Regularization term >0 - a : np.ndarray (ns,) + a : ndarray, shape (ns,) samples weights in the source domain - b : np.ndarray (nt,) + b : ndarray, shape (nt,) samples weights in the target domain numItermax : int, optional Max number of iterations @@ -1542,17 +1535,15 @@ def empirical_sinkhorn_divergence(X_s, X_t, reg, a=None, b=None, metric='sqeucli log : bool, optional record log if True - Returns ------- - gamma : (ns x nt) ndarray + gamma : ndarray, shape (ns, nt) Regularized optimal transportation matrix for the given parameters log : dict log dictionary return only if log==True in parameters Examples -------- - >>> n_s = 2 >>> n_t = 4 >>> reg = 0.1 @@ -1564,7 +1555,6 @@ def empirical_sinkhorn_divergence(X_s, X_t, reg, a=None, b=None, metric='sqeucli References ---------- - .. [23] Aude Genevay, Gabriel Peyré, Marco Cuturi, Learning Generative Models with Sinkhorn Divergences, Proceedings of the Twenty-First International Conference on Artficial Intelligence and Statistics, (AISTATS) 21, 2018 ''' if log: diff --git a/ot/datasets.py b/ot/datasets.py index e76e75d..ba0cfd9 100644 --- a/ot/datasets.py +++ b/ot/datasets.py @@ -17,7 +17,6 @@ def make_1D_gauss(n, m, s): Parameters ---------- - n : int number of bins in the histogram m : float @@ -25,12 +24,10 @@ def make_1D_gauss(n, m, s): s : float standard deviaton of the gaussian distribution - Returns ------- - h : np.array (n,) - 1D histogram for a gaussian distribution - + h : ndarray (n,) + 1D histogram for a gaussian distribution """ x = np.arange(n, dtype=np.float64) h = np.exp(-(x - m)**2 / (2 * s**2)) @@ -44,16 +41,15 @@ def get_1D_gauss(n, m, sigma): def make_2D_samples_gauss(n, m, sigma, random_state=None): - """return n samples drawn from 2D gaussian N(m,sigma) + """Return n samples drawn from 2D gaussian N(m,sigma) Parameters ---------- - n : int number of samples to make - m : np.array (2,) + m : ndarray, shape (2,) mean value of the gaussian distribution - sigma : np.array (2,2) + sigma : ndarray, shape (2, 2) covariance matrix of the gaussian distribution random_state : int, RandomState instance or None, optional (default=None) If int, random_state is the seed used by the random number generator; @@ -63,9 +59,8 @@ def make_2D_samples_gauss(n, m, sigma, random_state=None): Returns ------- - X : np.array (n,2) - n samples drawn from N(m,sigma) - + X : ndarray, shape (n, 2) + n samples drawn from N(m, sigma). """ generator = check_random_state(random_state) @@ -86,11 +81,10 @@ def get_2D_samples_gauss(n, m, sigma, random_state=None): def make_data_classif(dataset, n, nz=.5, theta=0, random_state=None, **kwargs): - """ dataset generation for classification problems + """Dataset generation for classification problems Parameters ---------- - dataset : str type of classification problem (see code) n : int @@ -105,13 +99,11 @@ def make_data_classif(dataset, n, nz=.5, theta=0, random_state=None, **kwargs): Returns ------- - X : np.array (n,d) - n observation of size d - y : np.array (n,) - labels of the samples - + X : ndarray, shape (n, d) + n observation of size d + y : ndarray, shape (n,) + labels of the samples. """ - generator = check_random_state(random_state) if dataset.lower() == '3gauss': diff --git a/ot/optim.py b/ot/optim.py index f94aceb..65baf9d 100644 --- a/ot/optim.py +++ b/ot/optim.py @@ -26,14 +26,13 @@ def line_search_armijo(f, xk, pk, gfk, old_fval, Parameters ---------- - - f : function + f : callable loss function - xk : np.ndarray + xk : ndarray initial position - pk : np.ndarray + pk : ndarray descent direction - gfk : np.ndarray + gfk : ndarray gradient of f at xk old_fval : float loss value at xk @@ -161,15 +160,15 @@ def cg(a, b, M, reg, f, df, G0=None, numItermax=200, Parameters ---------- - a : np.ndarray (ns,) + a : ndarray, shape (ns,) samples weights in the source domain - b : np.ndarray (nt,) + b : ndarray, shape (nt,) samples in the target domain - M : np.ndarray (ns,nt) + M : ndarray, shape (ns, nt) loss matrix reg : float Regularization term >0 - G0 : np.ndarray (ns,nt), optional + G0 : ndarray, shape (ns,nt), optional initial guess (default is indep joint density) numItermax : int, optional Max number of iterations @@ -299,17 +298,17 @@ def gcg(a, b, M, reg1, reg2, f, df, G0=None, numItermax=10, Parameters ---------- - a : np.ndarray (ns,) + a : ndarray, shape (ns,) samples weights in the source domain - b : np.ndarray (nt,) + b : ndarrayv (nt,) samples in the target domain - M : np.ndarray (ns,nt) + M : ndarray, shape (ns, nt) loss matrix reg1 : float Entropic Regularization term >0 reg2 : float Second Regularization term >0 - G0 : np.ndarray (ns,nt), optional + G0 : ndarray, shape (ns, nt), optional initial guess (default is indep joint density) numItermax : int, optional Max number of iterations @@ -326,15 +325,13 @@ def gcg(a, b, M, reg1, reg2, f, df, G0=None, numItermax=10, Returns ------- - gamma : (ns x nt) ndarray + gamma : ndarray, shape (ns, nt) Optimal transportation matrix for the given parameters log : dict log dictionary return only if log==True in parameters - References ---------- - .. [5] N. Courty; R. Flamary; D. Tuia; A. Rakotomamonjy, "Optimal Transport for Domain Adaptation," in IEEE Transactions on Pattern Analysis and Machine Intelligence , vol.PP, no.99, pp.1-1 .. [7] Rakotomamonjy, A., Flamary, R., & Courty, N. (2015). Generalized conditional gradient: analysis of convergence and applications. arXiv preprint arXiv:1510.06567. @@ -422,13 +419,12 @@ def solve_1d_linesearch_quad(a, b, c): Parameters ---------- a,b,c : float - The coefficients of the quadratic function + The coefficients of the quadratic function Returns ------- x : float The optimal value which leads to the minimal cost - """ f0 = c df0 = b diff --git a/ot/plot.py b/ot/plot.py index a409d4a..f403e98 100644 --- a/ot/plot.py +++ b/ot/plot.py @@ -26,11 +26,11 @@ def plot1D_mat(a, b, M, title=''): Parameters ---------- - a : np.array, shape (na,) + a : ndarray, shape (na,) Source distribution - b : np.array, shape (nb,) + b : ndarray, shape (nb,) Target distribution - M : np.array, shape (na,nb) + M : ndarray, shape (na, nb) Matrix to plot """ na, nb = M.shape -- cgit v1.2.3 From 0d9c65d39f7bf6a9c692ad8d5421ddb087ddcafc Mon Sep 17 00:00:00 2001 From: Alexandre Gramfort Date: Tue, 9 Jul 2019 18:09:30 +0200 Subject: trailing spaces --- ot/bregman.py | 14 +++++++------- ot/optim.py | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) (limited to 'ot/bregman.py') diff --git a/ot/bregman.py b/ot/bregman.py index b67074f..f39145d 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -291,7 +291,7 @@ def sinkhorn_knopp(a, b, M, reg, numItermax=1000, Returns ------- - gamma : ndarray, shape (ns, nt) + gamma : ndarray, shape (ns, nt) Optimal transportation matrix for the given parameters log : dict log dictionary return only if log==True in parameters @@ -469,7 +469,7 @@ def greenkhorn(a, b, M, reg, numItermax=10000, stopThr=1e-9, verbose=False, log= Returns ------- - gamma : ndarray, shape (ns, nt) + gamma : ndarray, shape (ns, nt) Optimal transportation matrix for the given parameters log : dict log dictionary return only if log==True in parameters @@ -622,7 +622,7 @@ def sinkhorn_stabilized(a, b, M, reg, numItermax=1000, tau=1e3, stopThr=1e-9, Returns ------- - gamma : ndarray, shape (ns, nt) + gamma : ndarray, shape (ns, nt) Optimal transportation matrix for the given parameters log : dict log dictionary return only if log==True in parameters @@ -848,7 +848,7 @@ def sinkhorn_epsilon_scaling(a, b, M, reg, numItermax=100, epsilon0=1e4, numInne Returns ------- - gamma : ndarray, shape (ns, nt) + gamma : ndarray, shape (ns, nt) Optimal transportation matrix for the given parameters log : dict log dictionary return only if log==True in parameters @@ -1340,7 +1340,7 @@ def empirical_sinkhorn(X_s, X_t, reg, a=None, b=None, metric='sqeuclidean', numI Returns ------- - gamma : ndarray, shape (ns, nt) + gamma : ndarray, shape (ns, nt) Regularized optimal transportation matrix for the given parameters log : dict log dictionary return only if log==True in parameters @@ -1430,7 +1430,7 @@ def empirical_sinkhorn2(X_s, X_t, reg, a=None, b=None, metric='sqeuclidean', num Returns ------- - gamma : ndarray, shape (ns, nt) + gamma : ndarray, shape (ns, nt) Regularized optimal transportation matrix for the given parameters log : dict log dictionary return only if log==True in parameters @@ -1537,7 +1537,7 @@ def empirical_sinkhorn_divergence(X_s, X_t, reg, a=None, b=None, metric='sqeucli Returns ------- - gamma : ndarray, shape (ns, nt) + gamma : ndarray, shape (ns, nt) Regularized optimal transportation matrix for the given parameters log : dict log dictionary return only if log==True in parameters diff --git a/ot/optim.py b/ot/optim.py index 65baf9d..0abd9e9 100644 --- a/ot/optim.py +++ b/ot/optim.py @@ -325,7 +325,7 @@ def gcg(a, b, M, reg1, reg2, f, df, G0=None, numItermax=10, Returns ------- - gamma : ndarray, shape (ns, nt) + gamma : ndarray, shape (ns, nt) Optimal transportation matrix for the given parameters log : dict log dictionary return only if log==True in parameters -- cgit v1.2.3 From 09f3f640fc46ba4905d5508b704f2e5a90dda295 Mon Sep 17 00:00:00 2001 From: Hicham Janati Date: Tue, 23 Jul 2019 21:28:30 +0200 Subject: fix issue 94 + add test --- ot/bregman.py | 10 +++++++--- test/test_bregman.py | 25 +++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 3 deletions(-) (limited to 'ot/bregman.py') diff --git a/ot/bregman.py b/ot/bregman.py index f39145d..70e4208 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -765,10 +765,14 @@ def sinkhorn_stabilized(a, b, M, reg, numItermax=1000, tau=1e3, stopThr=1e-9, cpt = cpt + 1 - # print('err=',err,' cpt=',cpt) if log: - log['logu'] = alpha / reg + np.log(u) - log['logv'] = beta / reg + np.log(v) + if nbb: + alpha = alpha[:, None] + beta = beta[:, None] + logu = alpha / reg + np.log(u) + logv = beta / reg + np.log(v) + log['logu'] = logu + log['logv'] = logv log['alpha'] = alpha + reg * np.log(u) log['beta'] = beta + reg * np.log(v) log['warmstart'] = (log['alpha'], log['beta']) diff --git a/test/test_bregman.py b/test/test_bregman.py index 7f4972c..83ebba8 100644 --- a/test/test_bregman.py +++ b/test/test_bregman.py @@ -254,3 +254,28 @@ def test_empirical_sinkhorn_divergence(): emp_sinkhorn_div, sinkhorn_div, atol=1e-05) # cf conv emp sinkhorn np.testing.assert_allclose( emp_sinkhorn_div_log, sink_div_log, atol=1e-05) # cf conv emp sinkhorn + + +def test_stabilized_vs_sinkhorn_multidim(): + # test if stable version matches sinkhorn + # for multidimensional inputs + n = 100 + + # Gaussian distributions + a = ot.datasets.make_1D_gauss(n, m=20, s=5) # m= mean, s= std + b1 = ot.datasets.make_1D_gauss(n, m=60, s=8) + b2 = ot.datasets.make_1D_gauss(n, m=30, s=4) + + # creating matrix A containing all distributions + b = np.vstack((b1, b2)).T + + M = ot.utils.dist0(n) + M /= np.median(M) + epsilon = 0.1 + G, log = ot.bregman.sinkhorn(a, b, M, reg=epsilon, + method="sinkhorn_stabilized", + log=True) + G2, log2 = ot.bregman.sinkhorn(a, b, M, epsilon, + method="sinkhorn", log=True) + + np.testing.assert_allclose(G, G2) -- cgit v1.2.3 From cfdbbd21642c6082164b84db78c2ead07499a113 Mon Sep 17 00:00:00 2001 From: Hicham Janati Date: Fri, 19 Jul 2019 17:04:14 +0200 Subject: remove square in convergence check add unbalanced with stabilization add unbalanced tests with stabilization fix doctest examples add xvfb in travis remove explicit call xvfb in travis change alpha to reg_m minor flake8 remove redundant sink definitions + better doc and naming add stabilized unbalanced barycenter + add not converged warnings add test for stable barycenter add generic barycenter func + make method funcs private fix typo + add method test for barycenters fix doc examples + add xml to gitignore fix whitespace in example change logsumexp import - scipy deprecation warning fix doctest improve naming + add stable barycenter in bregman add test for stable bar + test the method arg in bregman --- .gitignore | 3 + ot/__init__.py | 18 +- ot/bregman.py | 530 +++++++++++++++++++++----------- ot/unbalanced.py | 803 +++++++++++++++++++++++++++++++++++++++--------- pytest.ini | 0 test/test_bregman.py | 72 ++++- test/test_unbalanced.py | 163 +++++++--- 7 files changed, 1205 insertions(+), 384 deletions(-) create mode 100644 pytest.ini (limited to 'ot/bregman.py') diff --git a/.gitignore b/.gitignore index 42a9aad..dadf84c 100644 --- a/.gitignore +++ b/.gitignore @@ -59,6 +59,9 @@ coverage.xml *.mo *.pot +# xml +*.xml + # Django stuff: *.log local_settings.py diff --git a/ot/__init__.py b/ot/__init__.py index 35ae6fc..7d9615a 100644 --- a/ot/__init__.py +++ b/ot/__init__.py @@ -1,7 +1,7 @@ """ -This is the main module of the POT toolbox. It provides easy access to -a number of sub-modules and functions described below. +This is the main module of the POT toolbox. It provides easy access to +a number of sub-modules and functions described below. .. note:: @@ -14,27 +14,27 @@ a number of sub-modules and functions described below. - :any:`ot.lp` contains OT solvers for the exact (Linear Program) OT problems. - :any:`ot.smooth` contains OT solvers for the regularized (l2 and kl) smooth OT problems. - - :any:`ot.gromov` contains solvers for Gromov-Wasserstein and Fused Gromov + - :any:`ot.gromov` contains solvers for Gromov-Wasserstein and Fused Gromov Wasserstein problems. - - :any:`ot.optim` contains generic solvers OT based optimization problems + - :any:`ot.optim` contains generic solvers OT based optimization problems - :any:`ot.da` contains classes and function related to Monge mapping estimation and Domain Adaptation (DA). - :any:`ot.gpu` contains GPU (cupy) implementation of some OT solvers - - :any:`ot.dr` contains Dimension Reduction (DR) methods such as Wasserstein + - :any:`ot.dr` contains Dimension Reduction (DR) methods such as Wasserstein Discriminant Analysis. - - :any:`ot.utils` contains utility functions such as distance computation and - timing. + - :any:`ot.utils` contains utility functions such as distance computation and + timing. - :any:`ot.datasets` contains toy dataset generation functions. - :any:`ot.plot` contains visualization functions - :any:`ot.stochastic` contains stochastic solvers for regularized OT. - :any:`ot.unbalanced` contains solvers for regularized unbalanced OT. .. warning:: - The list of automatically imported sub-modules is as follows: + The list of automatically imported sub-modules is as follows: :py:mod:`ot.lp`, :py:mod:`ot.bregman`, :py:mod:`ot.optim` :py:mod:`ot.utils`, :py:mod:`ot.datasets`, :py:mod:`ot.gromov`, :py:mod:`ot.smooth` - :py:mod:`ot.stochastic` + :py:mod:`ot.stochastic` The following sub-modules are not imported due to additional dependencies: diff --git a/ot/bregman.py b/ot/bregman.py index 70e4208..2f27d58 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -7,10 +7,12 @@ Bregman projections for regularized OT # Nicolas Courty # Kilian Fatras # Titouan Vayer +# Hicham Janati # # License: MIT License import numpy as np +import warnings from .utils import unif, dist @@ -31,7 +33,7 @@ def sinkhorn(a, b, M, reg, method='sinkhorn', numItermax=1000, \gamma\geq 0 where : - - M is the (ns,nt) metric cost matrix + - M is the (dim_a, n_b) metric cost matrix - :math:`\Omega` is the entropic regularization term :math:`\Omega(\gamma)=\sum_{i,j} \gamma_{i,j}\log(\gamma_{i,j})` - a and b are source and target weights (sum to 1) @@ -40,12 +42,12 @@ def sinkhorn(a, b, M, reg, method='sinkhorn', numItermax=1000, Parameters ---------- - a : ndarray, shape (ns,) + a : ndarray, shape (dim_a,) samples weights in the source domain - b : ndarray, shape (nt,) or ndarray, shape (nt, nbb) + b : ndarray, shape (dim_b,) or ndarray, shape (dim_b, n_hists) samples in the target domain, compute sinkhorn with multiple targets and fixed M if b is a matrix (return OT loss + dual variables in log) - M : ndarray, shape (ns, nt) + M : ndarray, shape (dim_a, n_b) loss matrix reg : float Regularization term >0 @@ -64,7 +66,7 @@ def sinkhorn(a, b, M, reg, method='sinkhorn', numItermax=1000, Returns ------- - gamma : ndarray, shape (ns, nt) + gamma : ndarray, shape (dim_a, n_b) Optimal transportation matrix for the given parameters log : dict log dictionary return only if log==True in parameters @@ -103,30 +105,23 @@ def sinkhorn(a, b, M, reg, method='sinkhorn', numItermax=1000, """ if method.lower() == 'sinkhorn': - def sink(): - return sinkhorn_knopp(a, b, M, reg, numItermax=numItermax, - stopThr=stopThr, verbose=verbose, log=log, **kwargs) + return _sinkhorn_knopp(a, b, M, reg, numItermax=numItermax, + stopThr=stopThr, verbose=verbose, log=log, + **kwargs) elif method.lower() == 'greenkhorn': - def sink(): - return greenkhorn(a, b, M, reg, numItermax=numItermax, - stopThr=stopThr, verbose=verbose, log=log) + return _greenkhorn(a, b, M, reg, numItermax=numItermax, + stopThr=stopThr, verbose=verbose, log=log) elif method.lower() == 'sinkhorn_stabilized': - def sink(): - return sinkhorn_stabilized(a, b, M, reg, numItermax=numItermax, - stopThr=stopThr, verbose=verbose, log=log, **kwargs) + return _sinkhorn_stabilized(a, b, M, reg, numItermax=numItermax, + stopThr=stopThr, verbose=verbose, + log=log, **kwargs) elif method.lower() == 'sinkhorn_epsilon_scaling': - def sink(): - return sinkhorn_epsilon_scaling( - a, b, M, reg, numItermax=numItermax, - stopThr=stopThr, verbose=verbose, log=log, **kwargs) + return _sinkhorn_epsilon_scaling(a, b, M, reg, + numItermax=numItermax, + stopThr=stopThr, verbose=verbose, + log=log, **kwargs) else: - print('Warning : unknown method using classic Sinkhorn Knopp') - - def sink(): - return sinkhorn_knopp(a, b, M, reg, numItermax=numItermax, - stopThr=stopThr, verbose=verbose, log=log, **kwargs) - - return sink() + raise ValueError("Unknown method '%s'." % method) def sinkhorn2(a, b, M, reg, method='sinkhorn', numItermax=1000, @@ -146,7 +141,7 @@ def sinkhorn2(a, b, M, reg, method='sinkhorn', numItermax=1000, \gamma\geq 0 where : - - M is the (ns,nt) metric cost matrix + - M is the (dim_a, n_b) metric cost matrix - :math:`\Omega` is the entropic regularization term :math:`\Omega(\gamma)=\sum_{i,j} \gamma_{i,j}\log(\gamma_{i,j})` - a and b are source and target weights (sum to 1) @@ -155,12 +150,12 @@ def sinkhorn2(a, b, M, reg, method='sinkhorn', numItermax=1000, Parameters ---------- - a : ndarray, shape (ns,) + a : ndarray, shape (dim_a,) samples weights in the source domain - b : ndarray, shape (nt,) or ndarray, shape (nt, nbb) + b : ndarray, shape (dim_b,) or ndarray, shape (dim_b, n_hists) samples in the target domain, compute sinkhorn with multiple targets and fixed M if b is a matrix (return OT loss + dual variables in log) - M : ndarray, shape (ns, nt) + M : ndarray, shape (dim_a, n_b) loss matrix reg : float Regularization term >0 @@ -218,35 +213,25 @@ def sinkhorn2(a, b, M, reg, method='sinkhorn', numItermax=1000, ot.bregman.sinkhorn_epsilon_scaling: Sinkhorn with epslilon scaling [9][10] """ - + b = np.asarray(b, dtype=np.float64) + if len(b.shape) < 2: + b = b[:, None] if method.lower() == 'sinkhorn': - def sink(): - return sinkhorn_knopp(a, b, M, reg, numItermax=numItermax, - stopThr=stopThr, verbose=verbose, log=log, **kwargs) + return _sinkhorn_knopp(a, b, M, reg, numItermax=numItermax, + stopThr=stopThr, verbose=verbose, log=log, **kwargs) elif method.lower() == 'sinkhorn_stabilized': - def sink(): - return sinkhorn_stabilized(a, b, M, reg, numItermax=numItermax, - stopThr=stopThr, verbose=verbose, log=log, **kwargs) + return _sinkhorn_stabilized(a, b, M, reg, numItermax=numItermax, + stopThr=stopThr, verbose=verbose, log=log, **kwargs) elif method.lower() == 'sinkhorn_epsilon_scaling': - def sink(): - return sinkhorn_epsilon_scaling( - a, b, M, reg, numItermax=numItermax, - stopThr=stopThr, verbose=verbose, log=log, **kwargs) + return _sinkhorn_epsilon_scaling(a, b, M, reg, numItermax=numItermax, + stopThr=stopThr, verbose=verbose, + log=log, **kwargs) else: - print('Warning : unknown method using classic Sinkhorn Knopp') - - def sink(): - return sinkhorn_knopp(a, b, M, reg, **kwargs) + raise ValueError("Unknown method '%s'." % method) - b = np.asarray(b, dtype=np.float64) - if len(b.shape) < 2: - b = b[:, None] - return sink() - - -def sinkhorn_knopp(a, b, M, reg, numItermax=1000, - stopThr=1e-9, verbose=False, log=False, **kwargs): +def _sinkhorn_knopp(a, b, M, reg, numItermax=1000, + stopThr=1e-9, verbose=False, log=False, **kwargs): r""" Solve the entropic regularization optimal transport problem and return the OT matrix @@ -262,7 +247,7 @@ def sinkhorn_knopp(a, b, M, reg, numItermax=1000, \gamma\geq 0 where : - - M is the (ns,nt) metric cost matrix + - M is the (dim_a, n_b) metric cost matrix - :math:`\Omega` is the entropic regularization term :math:`\Omega(\gamma)=\sum_{i,j} \gamma_{i,j}\log(\gamma_{i,j})` - a and b are source and target weights (sum to 1) @@ -271,12 +256,12 @@ def sinkhorn_knopp(a, b, M, reg, numItermax=1000, Parameters ---------- - a : ndarray, shape (ns,) + a : ndarray, shape (dim_a,) samples weights in the source domain - b : ndarray, shape (nt,) or ndarray, shape (nt, nbb) + b : ndarray, shape (dim_b,) or ndarray, shape (dim_b, n_hists) samples in the target domain, compute sinkhorn with multiple targets and fixed M if b is a matrix (return OT loss + dual variables in log) - M : ndarray, shape (ns, nt) + M : ndarray, shape (dim_a, n_b) loss matrix reg : float Regularization term >0 @@ -291,7 +276,7 @@ def sinkhorn_knopp(a, b, M, reg, numItermax=1000, Returns ------- - gamma : ndarray, shape (ns, nt) + gamma : ndarray, shape (dim_a, n_b) Optimal transportation matrix for the given parameters log : dict log dictionary return only if log==True in parameters @@ -331,25 +316,25 @@ def sinkhorn_knopp(a, b, M, reg, numItermax=1000, b = np.ones((M.shape[1],), dtype=np.float64) / M.shape[1] # init data - Nini = len(a) - Nfin = len(b) + dim_a = len(a) + dim_b = len(b) if len(b.shape) > 1: - nbb = b.shape[1] + n_hists = b.shape[1] else: - nbb = 0 + n_hists = 0 if log: log = {'err': []} # we assume that no distances are null except those of the diagonal of # distances - if nbb: - u = np.ones((Nini, nbb)) / Nini - v = np.ones((Nfin, nbb)) / Nfin + if n_hists: + u = np.ones((dim_a, n_hists)) / dim_a + v = np.ones((dim_b, n_hists)) / dim_b else: - u = np.ones(Nini) / Nini - v = np.ones(Nfin) / Nfin + u = np.ones(dim_a) / dim_a + v = np.ones(dim_b) / dim_b # print(reg) @@ -384,13 +369,12 @@ def sinkhorn_knopp(a, b, M, reg, numItermax=1000, if cpt % 10 == 0: # we can speed up the process by checking for the error only all # the 10th iterations - if nbb: - err = np.sum((u - uprev)**2) / np.sum((u)**2) + \ - np.sum((v - vprev)**2) / np.sum((v)**2) + if n_hists: + np.einsum('ik,ij,jk->jk', u, K, v, out=tmp2) else: # compute right marginal tmp2= (diag(u)Kdiag(v))^T1 np.einsum('i,ij,j->j', u, K, v, out=tmp2) - err = np.linalg.norm(tmp2 - b)**2 # violation of marginal + err = np.linalg.norm(tmp2 - b) # violation of marginal if log: log['err'].append(err) @@ -404,7 +388,7 @@ def sinkhorn_knopp(a, b, M, reg, numItermax=1000, log['u'] = u log['v'] = v - if nbb: # return only loss + if n_hists: # return only loss res = np.einsum('ik,ij,jk,ij->k', u, K, v, M) if log: return res, log @@ -419,7 +403,7 @@ def sinkhorn_knopp(a, b, M, reg, numItermax=1000, return u.reshape((-1, 1)) * K * v.reshape((1, -1)) -def greenkhorn(a, b, M, reg, numItermax=10000, stopThr=1e-9, verbose=False, log=False): +def _greenkhorn(a, b, M, reg, numItermax=10000, stopThr=1e-9, verbose=False, log=False): r""" Solve the entropic regularization optimal transport problem and return the OT matrix @@ -443,7 +427,7 @@ def greenkhorn(a, b, M, reg, numItermax=10000, stopThr=1e-9, verbose=False, log= \gamma\geq 0 where : - - M is the (ns,nt) metric cost matrix + - M is the (dim_a, n_b) metric cost matrix - :math:`\Omega` is the entropic regularization term :math:`\Omega(\gamma)=\sum_{i,j} \gamma_{i,j}\log(\gamma_{i,j})` - a and b are source and target weights (sum to 1) @@ -451,12 +435,12 @@ def greenkhorn(a, b, M, reg, numItermax=10000, stopThr=1e-9, verbose=False, log= Parameters ---------- - a : ndarray, shape (ns,) + a : ndarray, shape (dim_a,) samples weights in the source domain - b : ndarray, shape (nt,) or ndarray, shape (nt, nbb) + b : ndarray, shape (dim_b,) or ndarray, shape (dim_b, n_hists) samples in the target domain, compute sinkhorn with multiple targets and fixed M if b is a matrix (return OT loss + dual variables in log) - M : ndarray, shape (ns, nt) + M : ndarray, shape (dim_a, n_b) loss matrix reg : float Regularization term >0 @@ -469,7 +453,7 @@ def greenkhorn(a, b, M, reg, numItermax=10000, stopThr=1e-9, verbose=False, log= Returns ------- - gamma : ndarray, shape (ns, nt) + gamma : ndarray, shape (dim_a, n_b) Optimal transportation matrix for the given parameters log : dict log dictionary return only if log==True in parameters @@ -481,7 +465,7 @@ def greenkhorn(a, b, M, reg, numItermax=10000, stopThr=1e-9, verbose=False, log= >>> a=[.5, .5] >>> b=[.5, .5] >>> M=[[0., 1.], [1., 0.]] - >>> ot.bregman.greenkhorn(a, b, M, 1) + >>> ot.bregman._greenkhorn(a, b, M, 1) array([[0.36552929, 0.13447071], [0.13447071, 0.36552929]]) @@ -509,16 +493,16 @@ def greenkhorn(a, b, M, reg, numItermax=10000, stopThr=1e-9, verbose=False, log= if len(b) == 0: b = np.ones((M.shape[1],), dtype=np.float64) / M.shape[1] - n = a.shape[0] - m = b.shape[0] + dim_a = a.shape[0] + dim_b = b.shape[0] # Next 3 lines equivalent to K= np.exp(-M/reg), but faster to compute K = np.empty_like(M) np.divide(M, -reg, out=K) np.exp(K, out=K) - u = np.full(n, 1. / n) - v = np.full(m, 1. / m) + u = np.full(dim_a, 1. / dim_a) + v = np.full(dim_b, 1. / dim_b) G = u[:, np.newaxis] * K * v[np.newaxis, :] viol = G.sum(1) - a @@ -571,8 +555,9 @@ def greenkhorn(a, b, M, reg, numItermax=10000, stopThr=1e-9, verbose=False, log= return G -def sinkhorn_stabilized(a, b, M, reg, numItermax=1000, tau=1e3, stopThr=1e-9, - warmstart=None, verbose=False, print_period=20, log=False, **kwargs): +def _sinkhorn_stabilized(a, b, M, reg, numItermax=1000, tau=1e3, stopThr=1e-9, + warmstart=None, verbose=False, print_period=20, + log=False, **kwargs): r""" Solve the entropic regularization OT problem with log stabilization @@ -588,7 +573,7 @@ def sinkhorn_stabilized(a, b, M, reg, numItermax=1000, tau=1e3, stopThr=1e-9, \gamma\geq 0 where : - - M is the (ns,nt) metric cost matrix + - M is the (dim_a, n_b) metric cost matrix - :math:`\Omega` is the entropic regularization term :math:`\Omega(\gamma)=\sum_{i,j} \gamma_{i,j}\log(\gamma_{i,j})` - a and b are source and target weights (sum to 1) @@ -599,11 +584,11 @@ def sinkhorn_stabilized(a, b, M, reg, numItermax=1000, tau=1e3, stopThr=1e-9, Parameters ---------- - a : ndarray, shape (ns,) + a : ndarray, shape (dim_a,) samples weights in the source domain - b : ndarray, shape (nt,) + b : ndarray, shape (dim_b,) samples in the target domain - M : ndarray, shape (ns, nt) + M : ndarray, shape (dim_a, n_b) loss matrix reg : float Regularization term >0 @@ -622,7 +607,7 @@ def sinkhorn_stabilized(a, b, M, reg, numItermax=1000, tau=1e3, stopThr=1e-9, Returns ------- - gamma : ndarray, shape (ns, nt) + gamma : ndarray, shape (dim_a, n_b) Optimal transportation matrix for the given parameters log : dict log dictionary return only if log==True in parameters @@ -634,7 +619,7 @@ def sinkhorn_stabilized(a, b, M, reg, numItermax=1000, tau=1e3, stopThr=1e-9, >>> a=[.5,.5] >>> b=[.5,.5] >>> M=[[0.,1.],[1.,0.]] - >>> ot.bregman.sinkhorn_stabilized(a,b,M,1) + >>> ot.bregman._sinkhorn_stabilized(a, b, M, 1) array([[0.36552929, 0.13447071], [0.13447071, 0.36552929]]) @@ -667,10 +652,10 @@ def sinkhorn_stabilized(a, b, M, reg, numItermax=1000, tau=1e3, stopThr=1e-9, # test if multiple target if len(b.shape) > 1: - nbb = b.shape[1] + n_hists = b.shape[1] a = a[:, np.newaxis] else: - nbb = 0 + n_hists = 0 # init data na = len(a) @@ -687,8 +672,8 @@ def sinkhorn_stabilized(a, b, M, reg, numItermax=1000, tau=1e3, stopThr=1e-9, else: alpha, beta = warmstart - if nbb: - u, v = np.ones((na, nbb)) / na, np.ones((nb, nbb)) / nb + if n_hists: + u, v = np.ones((na, n_hists)) / na, np.ones((nb, n_hists)) / nb else: u, v = np.ones(na) / na, np.ones(nb) / nb @@ -720,13 +705,13 @@ def sinkhorn_stabilized(a, b, M, reg, numItermax=1000, tau=1e3, stopThr=1e-9, # remove numerical problems and store them in K if np.abs(u).max() > tau or np.abs(v).max() > tau: - if nbb: + if n_hists: alpha, beta = alpha + reg * \ np.max(np.log(u), 1), beta + reg * np.max(np.log(v)) else: alpha, beta = alpha + reg * np.log(u), beta + reg * np.log(v) - if nbb: - u, v = np.ones((na, nbb)) / na, np.ones((nb, nbb)) / nb + if n_hists: + u, v = np.ones((na, n_hists)) / na, np.ones((nb, n_hists)) / nb else: u, v = np.ones(na) / na, np.ones(nb) / nb K = get_K(alpha, beta) @@ -734,12 +719,15 @@ def sinkhorn_stabilized(a, b, M, reg, numItermax=1000, tau=1e3, stopThr=1e-9, if cpt % print_period == 0: # we can speed up the process by checking for the error only all # the 10th iterations - if nbb: - err = np.sum((u - uprev)**2) / np.sum((u)**2) + \ - np.sum((v - vprev)**2) / np.sum((v)**2) + if n_hists: + err_u = abs(u - uprev).max() + err_u /= max(abs(u).max(), abs(uprev).max(), 1.) + err_v = abs(v - vprev).max() + err_v /= max(abs(v).max(), abs(vprev).max(), 1.) + err = 0.5 * (err_u + err_v) else: transp = get_Gamma(alpha, beta, u, v) - err = np.linalg.norm((np.sum(transp, axis=0) - b))**2 + err = np.linalg.norm((np.sum(transp, axis=0) - b)) if log: log['err'].append(err) @@ -766,7 +754,7 @@ def sinkhorn_stabilized(a, b, M, reg, numItermax=1000, tau=1e3, stopThr=1e-9, cpt = cpt + 1 if log: - if nbb: + if n_hists: alpha = alpha[:, None] beta = beta[:, None] logu = alpha / reg + np.log(u) @@ -776,26 +764,28 @@ def sinkhorn_stabilized(a, b, M, reg, numItermax=1000, tau=1e3, stopThr=1e-9, log['alpha'] = alpha + reg * np.log(u) log['beta'] = beta + reg * np.log(v) log['warmstart'] = (log['alpha'], log['beta']) - if nbb: - res = np.zeros((nbb)) - for i in range(nbb): + if n_hists: + res = np.zeros((n_hists)) + for i in range(n_hists): res[i] = np.sum(get_Gamma(alpha, beta, u[:, i], v[:, i]) * M) return res, log else: return get_Gamma(alpha, beta, u, v), log else: - if nbb: - res = np.zeros((nbb)) - for i in range(nbb): + if n_hists: + res = np.zeros((n_hists)) + for i in range(n_hists): res[i] = np.sum(get_Gamma(alpha, beta, u[:, i], v[:, i]) * M) return res else: return get_Gamma(alpha, beta, u, v) -def sinkhorn_epsilon_scaling(a, b, M, reg, numItermax=100, epsilon0=1e4, numInnerItermax=100, - tau=1e3, stopThr=1e-9, warmstart=None, verbose=False, print_period=10, log=False, **kwargs): +def _sinkhorn_epsilon_scaling(a, b, M, reg, numItermax=100, epsilon0=1e4, + numInnerItermax=100, tau=1e3, stopThr=1e-9, + warmstart=None, verbose=False, print_period=10, + log=False, **kwargs): r""" Solve the entropic regularization optimal transport problem with log stabilization and epsilon scaling. @@ -812,7 +802,7 @@ def sinkhorn_epsilon_scaling(a, b, M, reg, numItermax=100, epsilon0=1e4, numInne \gamma\geq 0 where : - - M is the (ns,nt) metric cost matrix + - M is the (dim_a, n_b) metric cost matrix - :math:`\Omega` is the entropic regularization term :math:`\Omega(\gamma)=\sum_{i,j} \gamma_{i,j}\log(\gamma_{i,j})` - a and b are source and target weights (sum to 1) @@ -823,18 +813,16 @@ def sinkhorn_epsilon_scaling(a, b, M, reg, numItermax=100, epsilon0=1e4, numInne Parameters ---------- - a : ndarray, shape (ns,) + a : ndarray, shape (dim_a,) samples weights in the source domain - b : ndarray, shape (nt,) + b : ndarray, shape (dim_b,) samples in the target domain - M : ndarray, shape (ns, nt) + M : ndarray, shape (dim_a, n_b) loss matrix reg : float Regularization term >0 tau : float thershold for max value in u or v for log scaling - tau : float - thershold for max value in u or v for log scaling warmstart : tuple of vectors if given then sarting values for alpha an beta log scalings numItermax : int, optional @@ -852,7 +840,7 @@ def sinkhorn_epsilon_scaling(a, b, M, reg, numItermax=100, epsilon0=1e4, numInne Returns ------- - gamma : ndarray, shape (ns, nt) + gamma : ndarray, shape (dim_a, n_b) Optimal transportation matrix for the given parameters log : dict log dictionary return only if log==True in parameters @@ -864,7 +852,7 @@ def sinkhorn_epsilon_scaling(a, b, M, reg, numItermax=100, epsilon0=1e4, numInne >>> a=[.5, .5] >>> b=[.5, .5] >>> M=[[0., 1.], [1., 0.]] - >>> ot.bregman.sinkhorn_epsilon_scaling(a, b, M, 1) + >>> ot.bregman._sinkhorn_epsilon_scaling(a, b, M, 1) array([[0.36552929, 0.13447071], [0.13447071, 0.36552929]]) @@ -893,8 +881,8 @@ def sinkhorn_epsilon_scaling(a, b, M, reg, numItermax=100, epsilon0=1e4, numInne b = np.ones((M.shape[1],), dtype=np.float64) / M.shape[1] # init data - na = len(a) - nb = len(b) + dim_a = len(a) + dim_b = len(b) # nrelative umerical precision with 64 bits numItermin = 35 @@ -907,14 +895,14 @@ def sinkhorn_epsilon_scaling(a, b, M, reg, numItermax=100, epsilon0=1e4, numInne # we assume that no distances are null except those of the diagonal of # distances if warmstart is None: - alpha, beta = np.zeros(na), np.zeros(nb) + alpha, beta = np.zeros(dim_a), np.zeros(dim_b) else: alpha, beta = warmstart def get_K(alpha, beta): """log space computation""" - return np.exp(-(M - alpha.reshape((na, 1)) - - beta.reshape((1, nb))) / reg) + return np.exp(-(M - alpha.reshape((dim_a, 1)) + - beta.reshape((1, dim_b))) / reg) # print(np.min(K)) def get_reg(n): # exponential decreasing @@ -927,7 +915,7 @@ def sinkhorn_epsilon_scaling(a, b, M, reg, numItermax=100, epsilon0=1e4, numInne regi = get_reg(cpt) - G, logi = sinkhorn_stabilized(a, b, M, regi, numItermax=numInnerItermax, stopThr=1e-9, warmstart=( + G, logi = _sinkhorn_stabilized(a, b, M, regi, numItermax=numInnerItermax, stopThr=1e-9, warmstart=( alpha, beta), verbose=False, print_period=20, tau=tau, log=True) alpha = logi['alpha'] @@ -986,8 +974,8 @@ def projC(gamma, q): return np.multiply(gamma, q / np.maximum(np.sum(gamma, axis=0), 1e-10)) -def barycenter(A, M, reg, weights=None, numItermax=1000, - stopThr=1e-4, verbose=False, log=False): +def barycenter(A, M, reg, weights=None, method="sinkhorn", numItermax=10000, + stopThr=1e-4, verbose=False, log=False, **kwargs): r"""Compute the entropic regularized wasserstein barycenter of distributions A The function solves the following optimization problem: @@ -1005,13 +993,15 @@ def barycenter(A, M, reg, weights=None, numItermax=1000, Parameters ---------- - A : ndarray, shape (d,n) - n training distributions a_i of size d - M : ndarray, shape (d,d) - loss matrix for OT + A : ndarray, shape (dim, n_hists) + n_hists training distributions a_i of size dim + M : ndarray, shape (dim, dim) + loss matrix for OT reg : float - Regularization term >0 - weights : ndarray, shape (n,) + Regularization term > 0 + method : str (optional) + method used for the solver either 'sinkhorn' or 'sinkhorn_stabilized' + weights : ndarray, shape (n_hists,) Weights of each histogram a_i on the simplex (barycentric coodinates) numItermax : int, optional Max number of iterations @@ -1025,7 +1015,7 @@ def barycenter(A, M, reg, weights=None, numItermax=1000, Returns ------- - a : (d,) ndarray + a : (dim,) ndarray Wasserstein barycenter log : dict log dictionary return only if log==True in parameters @@ -1036,8 +1026,70 @@ def barycenter(A, M, reg, weights=None, numItermax=1000, .. [3] Benamou, J. D., Carlier, G., Cuturi, M., Nenna, L., & Peyré, G. (2015). Iterative Bregman projections for regularized transportation problems. SIAM Journal on Scientific Computing, 37(2), A1111-A1138. + """ + + if method.lower() == 'sinkhorn': + return _barycenter(A, M, reg, numItermax=numItermax, + stopThr=stopThr, verbose=verbose, log=log, + **kwargs) + elif method.lower() == 'sinkhorn_stabilized': + return _barycenter_stabilized(A, M, reg, numItermax=numItermax, + stopThr=stopThr, verbose=verbose, + log=log, **kwargs) + else: + raise ValueError("Unknown method '%s'." % method) + + +def _barycenter(A, M, reg, weights=None, numItermax=1000, + stopThr=1e-4, verbose=False, log=False): + r"""Compute the entropic regularized wasserstein barycenter of distributions A + + The function solves the following optimization problem: + + .. math:: + \mathbf{a} = arg\min_\mathbf{a} \sum_i W_{reg}(\mathbf{a},\mathbf{a}_i) + + where : + + - :math:`W_{reg}(\cdot,\cdot)` is the entropic regularized Wasserstein distance (see ot.bregman.sinkhorn) + - :math:`\mathbf{a}_i` are training distributions in the columns of matrix :math:`\mathbf{A}` + - reg and :math:`\mathbf{M}` are respectively the regularization term and the cost matrix for OT + + The algorithm used for solving the problem is the Sinkhorn-Knopp matrix scaling algorithm as proposed in [3]_ + + Parameters + ---------- + A : ndarray, shape (dim, n_hists) + n_hists training distributions a_i of size dim + M : ndarray, shape (dim, dim) + loss matrix for OT + reg : float + Regularization term > 0 + weights : ndarray, shape (n_hists,) + Weights of each histogram a_i on the simplex (barycentric coodinates) + numItermax : int, optional + Max number of iterations + stopThr : float, optional + Stop threshol on error (>0) + verbose : bool, optional + Print information along iterations + log : bool, optional + record log if True + Returns + ------- + a : (dim,) ndarray + Wasserstein barycenter + log : dict + log dictionary return only if log==True in parameters + + + References + ---------- + + .. [3] Benamou, J. D., Carlier, G., Cuturi, M., Nenna, L., & Peyré, G. (2015). Iterative Bregman projections for regularized transportation problems. SIAM Journal on Scientific Computing, 37(2), A1111-A1138. + """ if weights is None: @@ -1082,6 +1134,136 @@ def barycenter(A, M, reg, weights=None, numItermax=1000, return geometricBar(weights, UKv) +def _barycenter_stabilized(A, M, reg, tau=1e10, weights=None, numItermax=1000, + stopThr=1e-4, verbose=False, log=False): + r"""Compute the entropic regularized wasserstein barycenter of distributions A + with stabilization. + + The function solves the following optimization problem: + + .. math:: + \mathbf{a} = arg\min_\mathbf{a} \sum_i W_{reg}(\mathbf{a},\mathbf{a}_i) + + where : + + - :math:`W_{reg}(\cdot,\cdot)` is the entropic regularized Wasserstein distance (see ot.bregman.sinkhorn) + - :math:`\mathbf{a}_i` are training distributions in the columns of matrix :math:`\mathbf{A}` + - reg and :math:`\mathbf{M}` are respectively the regularization term and the cost matrix for OT + + The algorithm used for solving the problem is the Sinkhorn-Knopp matrix scaling algorithm as proposed in [3]_ + + Parameters + ---------- + A : ndarray, shape (dim, n_hists) + n_hists training distributions a_i of size dim + M : ndarray, shape (dim, dim) + loss matrix for OT + reg : float + Regularization term > 0 + tau : float + thershold for max value in u or v for log scaling + weights : ndarray, shape (n_hists,) + Weights of each histogram a_i on the simplex (barycentric coodinates) + numItermax : int, optional + Max number of iterations + stopThr : float, optional + Stop threshol on error (>0) + verbose : bool, optional + Print information along iterations + log : bool, optional + record log if True + + + Returns + ------- + a : (dim,) ndarray + Wasserstein barycenter + log : dict + log dictionary return only if log==True in parameters + + + References + ---------- + + .. [3] Benamou, J. D., Carlier, G., Cuturi, M., Nenna, L., & Peyré, G. (2015). Iterative Bregman projections for regularized transportation problems. SIAM Journal on Scientific Computing, 37(2), A1111-A1138. + + """ + + dim, n_hists = A.shape + if weights is None: + weights = np.ones(n_hists) / n_hists + else: + assert(len(weights) == A.shape[1]) + + if log: + log = {'err': []} + + u = np.ones((dim, n_hists)) / dim + v = np.ones((dim, n_hists)) / dim + + # print(reg) + # Next 3 lines equivalent to K= np.exp(-M/reg), but faster to compute + K = np.empty(M.shape, dtype=M.dtype) + np.divide(M, -reg, out=K) + np.exp(K, out=K) + + cpt = 0 + err = 1. + alpha = np.zeros(dim) + beta = np.zeros(dim) + q = np.ones(dim) / dim + while (err > stopThr and cpt < numItermax): + qprev = q + Kv = K.dot(v) + u = A / (Kv + 1e-16) + Ktu = K.T.dot(u) + q = geometricBar(weights, Ktu) + Q = q[:, None] + v = Q / (Ktu + 1e-16) + absorbing = False + if (u > tau).any() or (v > tau).any(): + absorbing = True + print("YEAH absorbing") + alpha = alpha + reg * np.log(np.max(u, 1)) + beta = beta + reg * np.log(np.max(v, 1)) + K = np.exp((alpha[:, None] + beta[None, :] - + M) / reg) + v = np.ones_like(v) + Kv = K.dot(v) + if (np.any(Ktu == 0.) + or np.any(np.isnan(u)) or np.any(np.isnan(v)) + or np.any(np.isinf(u)) or np.any(np.isinf(v))): + # we have reached the machine precision + # come back to previous solution and quit loop + warnings.warn('Numerical errors at iteration %s' % cpt) + q = qprev + break + if (cpt % 10 == 0 and not absorbing) or cpt == 0: + # we can speed up the process by checking for the error only all + # the 10th iterations + err = abs(u * Kv - A).max() + if log: + log['err'].append(err) + if verbose: + if cpt % 50 == 0: + print( + '{:5s}|{:12s}'.format('It.', 'Err') + '\n' + '-' * 19) + print('{:5d}|{:8e}|'.format(cpt, err)) + + cpt += 1 + if err > stopThr: + warnings.warn("Stabilized Unbalanced Sinkhorn did not converge." + + "Try a larger entropy `reg`" + + "Or a larger absorption threshold `tau`.") + if log: + log['niter'] = cpt + log['logu'] = np.log(u + 1e-16) + log['logv'] = np.log(v + 1e-16) + return q, log + else: + return q + + def convolutional_barycenter2d(A, reg, weights=None, numItermax=10000, stopThr=1e-9, stabThr=1e-30, verbose=False, log=False): r"""Compute the entropic regularized wasserstein barycenter of distributions A where A is a collection of 2D images. @@ -1101,16 +1283,16 @@ def convolutional_barycenter2d(A, reg, weights=None, numItermax=10000, stopThr=1 Parameters ---------- - A : ndarray, shape (n, w, h) - n distributions (2D images) of size w x h + A : ndarray, shape (n_hists, width, height) + n distributions (2D images) of size width x height reg : float Regularization term >0 - weights : ndarray, shape (n,) + weights : ndarray, shape (n_hists,) Weights of each image on the simplex (barycentric coodinates) numItermax : int, optional Max number of iterations stopThr : float, optional - Stop threshol on error (>0) + Stop threshol on error (> 0) stabThr : float, optional Stabilization threshold to avoid numerical precision issue verbose : bool, optional @@ -1120,7 +1302,7 @@ def convolutional_barycenter2d(A, reg, weights=None, numItermax=10000, stopThr=1 Returns ------- - a : ndarray, shape (w, h) + a : ndarray, shape (width, height) 2D Wasserstein barycenter log : dict log dictionary return only if log==True in parameters @@ -1214,15 +1396,15 @@ def unmix(a, D, M, M0, h0, reg, reg0, alpha, numItermax=1000, Parameters ---------- - a : ndarray, shape (d) + a : ndarray, shape (n_observed) observed distribution - D : ndarray, shape (d, n) + D : ndarray, shape (dim, dim) dictionary matrix - M : ndarray, shape (d, d) + M : ndarray, shape (dim, dim) loss matrix - M0 : ndarray, shape (n, n) + M0 : ndarray, shape (n_observed, n_observed) loss matrix - h0 : ndarray, shape (n,) + h0 : ndarray, shape (dim,) prior on h reg : float Regularization term >0 (Wasserstein data fitting) @@ -1242,7 +1424,7 @@ def unmix(a, D, M, M0, h0, reg, reg0, alpha, numItermax=1000, Returns ------- - a : ndarray, shape (d,) + a : ndarray, shape (dim,) Wasserstein barycenter log : dict log dictionary return only if log==True in parameters @@ -1315,22 +1497,22 @@ def empirical_sinkhorn(X_s, X_t, reg, a=None, b=None, metric='sqeuclidean', numI \gamma\geq 0 where : - - :math:`M` is the (ns,nt) metric cost matrix + - :math:`M` is the (dim_a, n_b) metric cost matrix - :math:`\Omega` is the entropic regularization term :math:`\Omega(\gamma)=\sum_{i,j} \gamma_{i,j}\log(\gamma_{i,j})` - :math:`a` and :math:`b` are source and target weights (sum to 1) Parameters ---------- - X_s : ndarray, shape (ns, d) + X_s : ndarray, shape (dim_a, d) samples in the source domain - X_t : ndarray, shape (nt, d) + X_t : ndarray, shape (dim_b, d) samples in the target domain reg : float Regularization term >0 - a : ndarray, shape (ns,) + a : ndarray, shape (dim_a,) samples weights in the source domain - b : ndarray, shape (nt,) + b : ndarray, shape (dim_b,) samples weights in the target domain numItermax : int, optional Max number of iterations @@ -1344,7 +1526,7 @@ def empirical_sinkhorn(X_s, X_t, reg, a=None, b=None, metric='sqeuclidean', numI Returns ------- - gamma : ndarray, shape (ns, nt) + gamma : ndarray, shape (dim_a, n_b) Regularized optimal transportation matrix for the given parameters log : dict log dictionary return only if log==True in parameters @@ -1352,11 +1534,11 @@ def empirical_sinkhorn(X_s, X_t, reg, a=None, b=None, metric='sqeuclidean', numI Examples -------- - >>> n_s = 2 - >>> n_t = 2 + >>> n_a = 2 + >>> n_b = 2 >>> reg = 0.1 - >>> X_s = np.reshape(np.arange(n_s), (n_s, 1)) - >>> X_t = np.reshape(np.arange(0, n_t), (n_t, 1)) + >>> X_s = np.reshape(np.arange(n_a), (dim_a, 1)) + >>> X_t = np.reshape(np.arange(0, n_b), (dim_b, 1)) >>> empirical_sinkhorn(X_s, X_t, reg, verbose=False) # doctest: +NORMALIZE_WHITESPACE array([[4.99977301e-01, 2.26989344e-05], [2.26989344e-05, 4.99977301e-01]]) @@ -1405,22 +1587,22 @@ def empirical_sinkhorn2(X_s, X_t, reg, a=None, b=None, metric='sqeuclidean', num \gamma\geq 0 where : - - :math:`M` is the (ns,nt) metric cost matrix + - :math:`M` is the (dim_a, n_b) metric cost matrix - :math:`\Omega` is the entropic regularization term :math:`\Omega(\gamma)=\sum_{i,j} \gamma_{i,j}\log(\gamma_{i,j})` - :math:`a` and :math:`b` are source and target weights (sum to 1) Parameters ---------- - X_s : ndarray, shape (ns, d) + X_s : ndarray, shape (n_samples_a, dim) samples in the source domain - X_t : ndarray, shape (nt, d) + X_t : ndarray, shape (n_samples_b, d) samples in the target domain reg : float Regularization term >0 - a : ndarray, shape (ns,) + a : ndarray, shape (n_samples_a,) samples weights in the source domain - b : ndarray, shape (nt,) + b : ndarray, shape (n_samples_b,) samples weights in the target domain numItermax : int, optional Max number of iterations @@ -1434,7 +1616,7 @@ def empirical_sinkhorn2(X_s, X_t, reg, a=None, b=None, metric='sqeuclidean', num Returns ------- - gamma : ndarray, shape (ns, nt) + gamma : ndarray, shape (n_samples_a, n_samples_b) Regularized optimal transportation matrix for the given parameters log : dict log dictionary return only if log==True in parameters @@ -1442,11 +1624,11 @@ def empirical_sinkhorn2(X_s, X_t, reg, a=None, b=None, metric='sqeuclidean', num Examples -------- - >>> n_s = 2 - >>> n_t = 2 + >>> n_a = 2 + >>> n_b = 2 >>> reg = 0.1 - >>> X_s = np.reshape(np.arange(n_s), (n_s, 1)) - >>> X_t = np.reshape(np.arange(0, n_t), (n_t, 1)) + >>> X_s = np.reshape(np.arange(n_samples_a), (n_samples_a, 1)) + >>> X_t = np.reshape(np.arange(0, n_samples_b), (n_samples_b, 1)) >>> empirical_sinkhorn2(X_s, X_t, reg, verbose=False) array([4.53978687e-05]) @@ -1513,22 +1695,22 @@ def empirical_sinkhorn_divergence(X_s, X_t, reg, a=None, b=None, metric='sqeucli \gamma_b\geq 0 where : - - :math:`M` (resp. :math:`M_a, M_b`) is the (ns,nt) metric cost matrix (resp (ns, ns) and (nt, nt)) + - :math:`M` (resp. :math:`M_a, M_b`) is the (dim_a, n_b) metric cost matrix (resp (dim_a, ns) and (dim_b, nt)) - :math:`\Omega` is the entropic regularization term :math:`\Omega(\gamma)=\sum_{i,j} \gamma_{i,j}\log(\gamma_{i,j})` - :math:`a` and :math:`b` are source and target weights (sum to 1) Parameters ---------- - X_s : ndarray, shape (ns, d) + X_s : ndarray, shape (n_samples_a, dim) samples in the source domain - X_t : ndarray, shape (nt, d) + X_t : ndarray, shape (n_samples_b, dim) samples in the target domain reg : float Regularization term >0 - a : ndarray, shape (ns,) + a : ndarray, shape (n_samples_a,) samples weights in the source domain - b : ndarray, shape (nt,) + b : ndarray, shape (n_samples_b,) samples weights in the target domain numItermax : int, optional Max number of iterations @@ -1541,18 +1723,18 @@ def empirical_sinkhorn_divergence(X_s, X_t, reg, a=None, b=None, metric='sqeucli Returns ------- - gamma : ndarray, shape (ns, nt) + gamma : ndarray, shape (n_samples_a, n_samples_b) Regularized optimal transportation matrix for the given parameters log : dict log dictionary return only if log==True in parameters Examples -------- - >>> n_s = 2 - >>> n_t = 4 + >>> n_a = 2 + >>> n_b = 4 >>> reg = 0.1 - >>> X_s = np.reshape(np.arange(n_s), (n_s, 1)) - >>> X_t = np.reshape(np.arange(0, n_t), (n_t, 1)) + >>> X_s = np.reshape(np.arange(n_samples_a), (n_samples_a, 1)) + >>> X_t = np.reshape(np.arange(0, n_samples_b), (n_samples_b, 1)) >>> empirical_sinkhorn_divergence(X_s, X_t, reg) # doctest: +ELLIPSIS array([1.499...]) diff --git a/ot/unbalanced.py b/ot/unbalanced.py index 0f0692e..3f71d28 100644 --- a/ot/unbalanced.py +++ b/ot/unbalanced.py @@ -9,51 +9,56 @@ Regularized Unbalanced OT from __future__ import division import warnings import numpy as np +from scipy.special import logsumexp + # from .utils import unif, dist -def sinkhorn_unbalanced(a, b, M, reg, alpha, method='sinkhorn', numItermax=1000, - stopThr=1e-9, verbose=False, log=False, **kwargs): +def sinkhorn_unbalanced(a, b, M, reg, reg_m, method='sinkhorn', numItermax=1000, + stopThr=1e-6, verbose=False, log=False, **kwargs): r""" - Solve the unbalanced entropic regularization optimal transport problem and return the loss + Solve the unbalanced entropic regularization optimal transport problem + and return the OT plan The function solves the following optimization problem: .. math:: - W = \min_\gamma <\gamma,M>_F + reg\cdot\Omega(\gamma) + \\alpha KL(\gamma 1, a) + \\alpha KL(\gamma^T 1, b) + W = \min_\gamma <\gamma,M>_F + reg\cdot\Omega(\gamma) + reg_m KL(\gamma 1, a) + reg_m KL(\gamma^T 1, b) s.t. \gamma\geq 0 where : - - M is the (ns, nt) metric cost matrix - - :math:`\Omega` is the entropic regularization term :math:`\Omega(\gamma)=\sum_{i,j} \gamma_{i,j}\log(\gamma_{i,j})` - - a and b are source and target weights + - M is the (dim_a, dim_b) metric cost matrix + - :math:`\Omega` is the entropic regularization + term :math:`\Omega(\gamma)=\sum_{i,j} \gamma_{i,j}\log(\gamma_{i,j})` + - a and b are source and target unbalanced distributions - KL is the Kullback-Leibler divergence - The algorithm used for solving the problem is the generalized Sinkhorn-Knopp matrix scaling algorithm as proposed in [10, 23]_ + The algorithm used for solving the problem is the generalized + Sinkhorn-Knopp matrix scaling algorithm as proposed in [10, 23]_ Parameters ---------- - a : np.ndarray (ns,) - samples weights in the source domain - b : np.ndarray (nt,) or np.ndarray (nt,n_hists) - samples in the target domain, compute sinkhorn with multiple targets - and fixed M if b is a matrix (return OT loss + dual variables in log) - M : np.ndarray (ns, nt) + a : np.ndarray (dim_a,) + Unnormalized histogram of dimension dim_a + b : np.ndarray (dim_b,) or np.ndarray (dim_b, n_hists) + One or multiple unnormalized histograms of dimension dim_b + If many, compute all the OT distances (a, b_i) + M : np.ndarray (dim_a, dim_b) loss matrix reg : float Entropy regularization term > 0 - alpha : float + reg_m: float Marginal relaxation term > 0 method : str method used for the solver either 'sinkhorn', 'sinkhorn_stabilized' or - 'sinkhorn_epsilon_scaling', see those function for specific parameters + 'sinkhorn_reg_scaling', see those function for specific parameters numItermax : int, optional Max number of iterations stopThr : float, optional - Stop threshol on error (> 0) + Stop threshol on error (>0) verbose : bool, optional Print information along iterations log : bool, optional @@ -62,10 +67,16 @@ def sinkhorn_unbalanced(a, b, M, reg, alpha, method='sinkhorn', numItermax=1000, Returns ------- - W : (nt) ndarray or float - Optimal transportation matrix for the given parameters - log : dict - log dictionary return only if log==True in parameters + if n_hists == 1: + gamma : (dim_a x dim_b) ndarray + Optimal transportation matrix for the given parameters + log : dict + log dictionary returned only if `log` is `True` + else: + ot_distance : (n_hists,) ndarray + the OT distance between `a` and each of the histograms `b_i` + log : dict + log dictionary returned only if `log` is `True` Examples -------- @@ -82,83 +93,96 @@ def sinkhorn_unbalanced(a, b, M, reg, alpha, method='sinkhorn', numItermax=1000, References ---------- - .. [2] M. Cuturi, Sinkhorn Distances : Lightspeed Computation of Optimal Transport, Advances in Neural Information Processing Systems (NIPS) 26, 2013 + .. [2] M. Cuturi, Sinkhorn Distances : Lightspeed Computation of Optimal + Transport, Advances in Neural Information Processing Systems + (NIPS) 26, 2013 - .. [9] Schmitzer, B. (2016). Stabilized Sparse Scaling Algorithms for Entropy Regularized Transport Problems. arXiv preprint arXiv:1610.06519. + .. [9] Schmitzer, B. (2016). Stabilized Sparse Scaling Algorithms for + Entropy Regularized Transport Problems. arXiv preprint arXiv:1610.06519. - .. [10] Chizat, L., Peyré, G., Schmitzer, B., & Vialard, F. X. (2016). Scaling algorithms for unbalanced transport problems. arXiv preprint arXiv:1607.05816. + .. [10] Chizat, L., Peyré, G., Schmitzer, B., & Vialard, F. X. (2016). + Scaling algorithms for unbalanced transport problems. arXiv preprint + arXiv:1607.05816. - .. [25] Frogner C., Zhang C., Mobahi H., Araya-Polo M., Poggio T. : Learning with a Wasserstein Loss, Advances in Neural Information Processing Systems (NIPS) 2015 + .. [25] Frogner C., Zhang C., Mobahi H., Araya-Polo M., Poggio T. : + Learning with a Wasserstein Loss, Advances in Neural Information + Processing Systems (NIPS) 2015 See Also -------- ot.unbalanced.sinkhorn_knopp_unbalanced : Unbalanced Classic Sinkhorn [10] - ot.unbalanced.sinkhorn_stabilized_unbalanced: Unbalanced Stabilized sinkhorn [9][10] - ot.unbalanced.sinkhorn_epsilon_scaling_unbalanced: Unbalanced Sinkhorn with epslilon scaling [9][10] + ot.unbalanced.sinkhorn_stabilized_unbalanced: + Unbalanced Stabilized sinkhorn [9][10] + ot.unbalanced.sinkhorn_reg_scaling_unbalanced: + Unbalanced Sinkhorn with epslilon scaling [9][10] """ if method.lower() == 'sinkhorn': - def sink(): - return sinkhorn_knopp_unbalanced(a, b, M, reg, alpha, - numItermax=numItermax, - stopThr=stopThr, verbose=verbose, - log=log, **kwargs) - - elif method.lower() in ['sinkhorn_stabilized', 'sinkhorn_epsilon_scaling']: + return _sinkhorn_knopp_unbalanced(a, b, M, reg, reg_m, + numItermax=numItermax, + stopThr=stopThr, verbose=verbose, + log=log, **kwargs) + + elif method.lower() == 'sinkhorn_stabilized': + return _sinkhorn_stabilized_unbalanced(a, b, M, reg, reg_m, + numItermax=numItermax, + stopThr=stopThr, + verbose=verbose, + log=log, **kwargs) + elif method.lower() in ['sinkhorn_reg_scaling']: warnings.warn('Method not implemented yet. Using classic Sinkhorn Knopp') - - def sink(): - return sinkhorn_knopp_unbalanced(a, b, M, reg, alpha, - numItermax=numItermax, - stopThr=stopThr, verbose=verbose, - log=log, **kwargs) + return _sinkhorn_knopp_unbalanced(a, b, M, reg, reg_m, + numItermax=numItermax, + stopThr=stopThr, verbose=verbose, + log=log, **kwargs) else: - raise ValueError('Unknown method. Using classic Sinkhorn Knopp') - - return sink() + raise ValueError("Unknown method '%s'." % method) -def sinkhorn_unbalanced2(a, b, M, reg, alpha, method='sinkhorn', - numItermax=1000, stopThr=1e-9, verbose=False, +def sinkhorn_unbalanced2(a, b, M, reg, reg_m, method='sinkhorn', + numItermax=1000, stopThr=1e-6, verbose=False, log=False, **kwargs): r""" - Solve the entropic regularization unbalanced optimal transport problem and return the loss + Solve the entropic regularization unbalanced optimal transport problem and + return the loss The function solves the following optimization problem: .. math:: - W = \min_\gamma <\gamma,M>_F + reg\cdot\Omega(\gamma) + \\alpha KL(\gamma 1, a) + \\alpha KL(\gamma^T 1, b) + W = \min_\gamma <\gamma,M>_F + reg\cdot\Omega(\gamma) + reg_m KL(\gamma 1, a) + reg_m KL(\gamma^T 1, b) s.t. \gamma\geq 0 where : - - M is the (ns, nt) metric cost matrix - - :math:`\Omega` is the entropic regularization term :math:`\Omega(\gamma)=\sum_{i,j} \gamma_{i,j}\log(\gamma_{i,j})` - - a and b are source and target weights + - M is the (dim_a, dim_b) metric cost matrix + - :math:`\Omega` is the entropic regularization term + :math:`\Omega(\gamma)=\sum_{i,j} \gamma_{i,j}\log(\gamma_{i,j})` + - a and b are source and target unbalanced distributions - KL is the Kullback-Leibler divergence - The algorithm used for solving the problem is the generalized Sinkhorn-Knopp matrix scaling algorithm as proposed in [10, 23]_ + The algorithm used for solving the problem is the generalized + Sinkhorn-Knopp matrix scaling algorithm as proposed in [10, 23]_ Parameters ---------- - a : np.ndarray (ns,) - samples weights in the source domain - b : np.ndarray (nt,) or np.ndarray (nt, n_hists) - samples in the target domain, compute sinkhorn with multiple targets - and fixed M if b is a matrix (return OT loss + dual variables in log) - M : np.ndarray (ns,nt) + a : np.ndarray (dim_a,) + Unnormalized histogram of dimension dim_a + b : np.ndarray (dim_b,) or np.ndarray (dim_b, n_hists) + One or multiple unnormalized histograms of dimension dim_b + If many, compute all the OT distances (a, b_i) + M : np.ndarray (dim_a, dim_b) loss matrix reg : float Entropy regularization term > 0 - alpha : float + reg_m: float Marginal relaxation term > 0 method : str method used for the solver either 'sinkhorn', 'sinkhorn_stabilized' or - 'sinkhorn_epsilon_scaling', see those function for specific parameters + 'sinkhorn_reg_scaling', see those function for specific parameters numItermax : int, optional Max number of iterations stopThr : float, optional @@ -171,10 +195,10 @@ def sinkhorn_unbalanced2(a, b, M, reg, alpha, method='sinkhorn', Returns ------- - W : (nt) ndarray or float - Optimal transportation matrix for the given parameters + ot_distance : (n_hists,) ndarray + the OT distance between `a` and each of the histograms `b_i` log : dict - log dictionary return only if log==True in parameters + log dictionary returned only if `log` is `True` Examples -------- @@ -191,64 +215,70 @@ def sinkhorn_unbalanced2(a, b, M, reg, alpha, method='sinkhorn', References ---------- - .. [2] M. Cuturi, Sinkhorn Distances : Lightspeed Computation of Optimal Transport, Advances in Neural Information Processing Systems (NIPS) 26, 2013 + .. [2] M. Cuturi, Sinkhorn Distances : Lightspeed Computation of Optimal + Transport, Advances in Neural Information Processing Systems + (NIPS) 26, 2013 - .. [9] Schmitzer, B. (2016). Stabilized Sparse Scaling Algorithms for Entropy Regularized Transport Problems. arXiv preprint arXiv:1610.06519. + .. [9] Schmitzer, B. (2016). Stabilized Sparse Scaling Algorithms for + Entropy Regularized Transport Problems. arXiv preprint arXiv:1610.06519. - .. [10] Chizat, L., Peyré, G., Schmitzer, B., & Vialard, F. X. (2016). Scaling algorithms for unbalanced transport problems. arXiv preprint arXiv:1607.05816. + .. [10] Chizat, L., Peyré, G., Schmitzer, B., & Vialard, F. X. (2016). + Scaling algorithms for unbalanced transport problems. arXiv preprint + arXiv:1607.05816. - .. [25] Frogner C., Zhang C., Mobahi H., Araya-Polo M., Poggio T. : Learning with a Wasserstein Loss, Advances in Neural Information Processing Systems (NIPS) 2015 + .. [25] Frogner C., Zhang C., Mobahi H., Araya-Polo M., Poggio T. : + Learning with a Wasserstein Loss, Advances in Neural Information + Processing Systems (NIPS) 2015 See Also -------- ot.unbalanced.sinkhorn_knopp : Unbalanced Classic Sinkhorn [10] ot.unbalanced.sinkhorn_stabilized: Unbalanced Stabilized sinkhorn [9][10] - ot.unbalanced.sinkhorn_epsilon_scaling: Unbalanced Sinkhorn with epslilon scaling [9][10] + ot.unbalanced.sinkhorn_reg_scaling: Unbalanced Sinkhorn with epslilon scaling [9][10] """ - - if method.lower() == 'sinkhorn': - def sink(): - return sinkhorn_knopp_unbalanced(a, b, M, reg, alpha, - numItermax=numItermax, - stopThr=stopThr, verbose=verbose, - log=log, **kwargs) - - elif method.lower() in ['sinkhorn_stabilized', 'sinkhorn_epsilon_scaling']: - warnings.warn('Method not implemented yet. Using classic Sinkhorn Knopp') - - def sink(): - return sinkhorn_knopp_unbalanced(a, b, M, reg, alpha, - numItermax=numItermax, - stopThr=stopThr, verbose=verbose, - log=log, **kwargs) - else: - raise ValueError('Unknown method. Using classic Sinkhorn Knopp') - b = np.asarray(b, dtype=np.float64) if len(b.shape) < 2: b = b[:, None] - - return sink() + if method.lower() == 'sinkhorn': + return _sinkhorn_knopp_unbalanced(a, b, M, reg, reg_m, + numItermax=numItermax, + stopThr=stopThr, verbose=verbose, + log=log, **kwargs) + + elif method.lower() == 'sinkhorn_stabilized': + return _sinkhorn_stabilized_unbalanced(a, b, M, reg, reg_m, + numItermax=numItermax, + stopThr=stopThr, + verbose=verbose, + log=log, **kwargs) + elif method.lower() in ['sinkhorn_reg_scaling']: + warnings.warn('Method not implemented yet. Using classic Sinkhorn Knopp') + return _sinkhorn_knopp_unbalanced(a, b, M, reg, reg_m, + numItermax=numItermax, + stopThr=stopThr, verbose=verbose, + log=log, **kwargs) + else: + raise ValueError('Unknown method %s.' % method) -def sinkhorn_knopp_unbalanced(a, b, M, reg, alpha, numItermax=1000, - stopThr=1e-9, verbose=False, log=False, **kwargs): +def _sinkhorn_knopp_unbalanced(a, b, M, reg, reg_m, numItermax=1000, + stopThr=1e-6, verbose=False, log=False, **kwargs): r""" Solve the entropic regularization unbalanced optimal transport problem and return the loss The function solves the following optimization problem: .. math:: - W = \min_\gamma <\gamma,M>_F + reg\cdot\Omega(\gamma) + \\alpha KL(\gamma 1, a) + \\alpha KL(\gamma^T 1, b) + W = \min_\gamma <\gamma,M>_F + reg\cdot\Omega(\gamma) + \reg_m KL(\gamma 1, a) + \reg_m KL(\gamma^T 1, b) s.t. \gamma\geq 0 where : - - M is the (ns, nt) metric cost matrix + - M is the (dim_a, dim_b) metric cost matrix - :math:`\Omega` is the entropic regularization term :math:`\Omega(\gamma)=\sum_{i,j} \gamma_{i,j}\log(\gamma_{i,j})` - - a and b are source and target weights + - a and b are source and target unbalanced distributions - KL is the Kullback-Leibler divergence The algorithm used for solving the problem is the generalized Sinkhorn-Knopp matrix scaling algorithm as proposed in [10, 23]_ @@ -256,16 +286,16 @@ def sinkhorn_knopp_unbalanced(a, b, M, reg, alpha, numItermax=1000, Parameters ---------- - a : np.ndarray (ns,) - samples weights in the source domain - b : np.ndarray (nt,) or np.ndarray (nt, n_hists) - samples in the target domain, compute sinkhorn with multiple targets - and fixed M if b is a matrix (return OT loss + dual variables in log) - M : np.ndarray (ns,nt) + a : np.ndarray (dim_a,) + Unnormalized histogram of dimension dim_a + b : np.ndarray (dim_b,) or np.ndarray (dim_b, n_hists) + One or multiple unnormalized histograms of dimension dim_b + If many, compute all the OT distances (a, b_i) + M : np.ndarray (dim_a, dim_b) loss matrix reg : float Entropy regularization term > 0 - alpha : float + reg_m: float Marginal relaxation term > 0 numItermax : int, optional Max number of iterations @@ -279,11 +309,16 @@ def sinkhorn_knopp_unbalanced(a, b, M, reg, alpha, numItermax=1000, Returns ------- - gamma : (ns x nt) ndarray - Optimal transportation matrix for the given parameters - log : dict - log dictionary return only if log==True in parameters - + if n_hists == 1: + gamma : (dim_a x dim_b) ndarray + Optimal transportation matrix for the given parameters + log : dict + log dictionary returned only if `log` is `True` + else: + ot_distance : (n_hists,) ndarray + the OT distance between `a` and each of the histograms `b_i` + log : dict + log dictionary returned only if `log` is `True` Examples -------- @@ -291,16 +326,20 @@ def sinkhorn_knopp_unbalanced(a, b, M, reg, alpha, numItermax=1000, >>> a=[.5, .5] >>> b=[.5, .5] >>> M=[[0., 1.],[1., 0.]] - >>> ot.unbalanced.sinkhorn_knopp_unbalanced(a, b, M, 1., 1.) + >>> ot.unbalanced._sinkhorn_knopp_unbalanced(a, b, M, 1., 1.) array([[0.51122823, 0.18807035], [0.18807035, 0.51122823]]) References ---------- - .. [10] Chizat, L., Peyré, G., Schmitzer, B., & Vialard, F. X. (2016). Scaling algorithms for unbalanced transport problems. arXiv preprint arXiv:1607.05816. + .. [10] Chizat, L., Peyré, G., Schmitzer, B., & Vialard, F. X. (2016). + Scaling algorithms for unbalanced transport problems. arXiv preprint + arXiv:1607.05816. - .. [25] Frogner C., Zhang C., Mobahi H., Araya-Polo M., Poggio T. : Learning with a Wasserstein Loss, Advances in Neural Information Processing Systems (NIPS) 2015 + .. [25] Frogner C., Zhang C., Mobahi H., Araya-Polo M., Poggio T. : + Learning with a Wasserstein Loss, Advances in Neural Information + Processing Systems (NIPS) 2015 See Also -------- @@ -313,12 +352,12 @@ def sinkhorn_knopp_unbalanced(a, b, M, reg, alpha, numItermax=1000, b = np.asarray(b, dtype=np.float64) M = np.asarray(M, dtype=np.float64) - n_a, n_b = M.shape + dim_a, dim_b = M.shape if len(a) == 0: - a = np.ones(n_a, dtype=np.float64) / n_a + a = np.ones(dim_a, dtype=np.float64) / dim_a if len(b) == 0: - b = np.ones(n_b, dtype=np.float64) / n_b + b = np.ones(dim_b, dtype=np.float64) / dim_b if len(b.shape) > 1: n_hists = b.shape[1] @@ -331,21 +370,19 @@ def sinkhorn_knopp_unbalanced(a, b, M, reg, alpha, numItermax=1000, # we assume that no distances are null except those of the diagonal of # distances if n_hists: - u = np.ones((n_a, 1)) / n_a - v = np.ones((n_b, n_hists)) / n_b - a = a.reshape(n_a, 1) + u = np.ones((dim_a, 1)) / dim_a + v = np.ones((dim_b, n_hists)) / dim_b + a = a.reshape(dim_a, 1) else: - u = np.ones(n_a) / n_a - v = np.ones(n_b) / n_b + u = np.ones(dim_a) / dim_a + v = np.ones(dim_b) / dim_b - # print(reg) # Next 3 lines equivalent to K= np.exp(-M/reg), but faster to compute K = np.empty(M.shape, dtype=M.dtype) np.divide(M, -reg, out=K) np.exp(K, out=K) - # print(np.min(K)) - fi = alpha / (alpha + reg) + fi = reg_m / (reg_m + reg) cpt = 0 err = 1. @@ -371,8 +408,9 @@ def sinkhorn_knopp_unbalanced(a, b, M, reg, alpha, numItermax=1000, if cpt % 10 == 0: # we can speed up the process by checking for the error only all # the 10th iterations - err = np.sum((u - uprev)**2) / np.sum((u)**2) + \ - np.sum((v - vprev)**2) / np.sum((v)**2) + err_u = abs(u - uprev).max() / max(abs(u).max(), abs(uprev).max(), 1.) + err_v = abs(v - vprev).max() / max(abs(v).max(), abs(vprev).max(), 1.) + err = 0.5 * (err_u + err_v) if log: log['err'].append(err) if verbose: @@ -383,8 +421,8 @@ def sinkhorn_knopp_unbalanced(a, b, M, reg, alpha, numItermax=1000, cpt += 1 if log: - log['u'] = u - log['v'] = v + log['logu'] = np.log(u + 1e-16) + log['logv'] = np.log(v + 1e-16) if n_hists: # return only loss res = np.einsum('ik,ij,jk,ij->k', u, K, v, M) @@ -401,9 +439,224 @@ def sinkhorn_knopp_unbalanced(a, b, M, reg, alpha, numItermax=1000, return u[:, None] * K * v[None, :] -def barycenter_unbalanced(A, M, reg, alpha, weights=None, numItermax=1000, - stopThr=1e-4, verbose=False, log=False): - r"""Compute the entropic regularized unbalanced wasserstein barycenter of distributions A +def _sinkhorn_stabilized_unbalanced(a, b, M, reg, reg_m, tau=1e5, numItermax=1000, + stopThr=1e-6, verbose=False, log=False, + **kwargs): + r""" + Solve the entropic regularization unbalanced optimal transport + problem and return the loss + + The function solves the following optimization problem using log-domain + stabilization as proposed in [10]: + + .. math:: + W = \min_\gamma <\gamma,M>_F + reg\cdot\Omega(\gamma) + reg_m KL(\gamma 1, a) + reg_m KL(\gamma^T 1, b) + + s.t. + \gamma\geq 0 + where : + + - M is the (dim_a, dim_b) metric cost matrix + - :math:`\Omega` is the entropic regularization + term :math:`\Omega(\gamma)=\sum_{i,j} \gamma_{i,j}\log(\gamma_{i,j})` + - a and b are source and target unbalanced distributions + - KL is the Kullback-Leibler divergence + + The algorithm used for solving the problem is the generalized + Sinkhorn-Knopp matrix scaling algorithm as proposed in [10, 23]_ + + + Parameters + ---------- + a : np.ndarray (dim_a,) + Unnormalized histogram of dimension dim_a + b : np.ndarray (dim_b,) or np.ndarray (dim_b, n_hists) + One or multiple unnormalized histograms of dimension dim_b + If many, compute all the OT distances (a, b_i) + M : np.ndarray (dim_a, dim_b) + loss matrix + reg : float + Entropy regularization term > 0 + reg_m: float + Marginal relaxation term > 0 + tau : float + thershold for max value in u or v for log scaling + numItermax : int, optional + Max number of iterations + stopThr : float, optional + Stop threshol on error (>0) + verbose : bool, optional + Print information along iterations + log : bool, optional + record log if True + + + Returns + ------- + if n_hists == 1: + gamma : (dim_a x dim_b) ndarray + Optimal transportation matrix for the given parameters + log : dict + log dictionary returned only if `log` is `True` + else: + ot_distance : (n_hists,) ndarray + the OT distance between `a` and each of the histograms `b_i` + log : dict + log dictionary returned only if `log` is `True` + Examples + -------- + + >>> import ot + >>> a=[.5, .5] + >>> b=[.5, .5] + >>> M=[[0., 1.],[1., 0.]] + >>> ot.unbalanced._sinkhorn_stabilized_unbalanced(a, b, M, 1., 1.) + array([[0.51122823, 0.18807035], + [0.18807035, 0.51122823]]) + + References + ---------- + + .. [10] Chizat, L., Peyré, G., Schmitzer, B., & Vialard, F. X. (2016). + Scaling algorithms for unbalanced transport problems. arXiv preprint arXiv:1607.05816. + + .. [25] Frogner C., Zhang C., Mobahi H., Araya-Polo M., Poggio T. : + Learning with a Wasserstein Loss, Advances in Neural Information + Processing Systems (NIPS) 2015 + + See Also + -------- + ot.lp.emd : Unregularized OT + ot.optim.cg : General regularized OT + + """ + + a = np.asarray(a, dtype=np.float64) + b = np.asarray(b, dtype=np.float64) + M = np.asarray(M, dtype=np.float64) + + dim_a, dim_b = M.shape + + if len(a) == 0: + a = np.ones(dim_a, dtype=np.float64) / dim_a + if len(b) == 0: + b = np.ones(dim_b, dtype=np.float64) / dim_b + + if len(b.shape) > 1: + n_hists = b.shape[1] + else: + n_hists = 0 + + if log: + log = {'err': []} + + # we assume that no distances are null except those of the diagonal of + # distances + if n_hists: + u = np.ones((dim_a, n_hists)) / dim_a + v = np.ones((dim_b, n_hists)) / dim_b + a = a.reshape(dim_a, 1) + else: + u = np.ones(dim_a) / dim_a + v = np.ones(dim_b) / dim_b + + # print(reg) + # Next 3 lines equivalent to K= np.exp(-M/reg), but faster to compute + K = np.empty(M.shape, dtype=M.dtype) + np.divide(M, -reg, out=K) + np.exp(K, out=K) + + fi = reg_m / (reg_m + reg) + + cpt = 0 + err = 1. + alpha = np.zeros(dim_a) + beta = np.zeros(dim_b) + while (err > stopThr and cpt < numItermax): + uprev = u + vprev = v + + Kv = K.dot(v) + f_alpha = np.exp(- alpha / (reg + reg_m)) + f_beta = np.exp(- beta / (reg + reg_m)) + + if n_hists: + f_alpha = f_alpha[:, None] + f_beta = f_beta[:, None] + u = ((a / (Kv + 1e-16)) ** fi) * f_alpha + Ktu = K.T.dot(u) + v = ((b / (Ktu + 1e-16)) ** fi) * f_beta + absorbing = False + if (u > tau).any() or (v > tau).any(): + absorbing = True + if n_hists: + alpha = alpha + reg * np.log(np.max(u, 1)) + beta = beta + reg * np.log(np.max(v, 1)) + else: + alpha = alpha + reg * np.log(np.max(u)) + beta = beta + reg * np.log(np.max(v)) + K = np.exp((alpha[:, None] + beta[None, :] - + M) / reg) + v = np.ones_like(v) + Kv = K.dot(v) + + if (np.any(Ktu == 0.) + or np.any(np.isnan(u)) or np.any(np.isnan(v)) + or np.any(np.isinf(u)) or np.any(np.isinf(v))): + # we have reached the machine precision + # come back to previous solution and quit loop + warnings.warn('Numerical errors at iteration %s' % cpt) + u = uprev + v = vprev + break + if (cpt % 10 == 0 and not absorbing) or cpt == 0: + # we can speed up the process by checking for the error only all + # the 10th iterations + err = abs(u - uprev).max() / max(abs(u).max(), abs(uprev).max(), + 1.) + if log: + log['err'].append(err) + if verbose: + if cpt % 200 == 0: + print( + '{:5s}|{:12s}'.format('It.', 'Err') + '\n' + '-' * 19) + print('{:5d}|{:8e}|'.format(cpt, err)) + cpt = cpt + 1 + + if err > stopThr: + warnings.warn("Stabilized Unbalanced Sinkhorn did not converge." + + "Try a larger entropy `reg` or a lower mass `reg_m`." + + "Or a larger absorption threshold `tau`.") + if n_hists: + logu = alpha[:, None] / reg + np.log(u) + logv = beta[:, None] / reg + np.log(v) + else: + logu = alpha / reg + np.log(u) + logv = beta / reg + np.log(v) + if log: + log['logu'] = logu + log['logv'] = logv + if n_hists: # return only loss + res = logsumexp(np.log(M + 1e-100)[:, :, None] + logu[:, None, :] + + logv[None, :, :] - M[:, :, None] / reg, axis=(0, 1)) + res = np.exp(res) + if log: + return res, log + else: + return res + + else: # return OT matrix + ot_matrix = np.exp(logu[:, None] + logv[None, :] - M / reg) + if log: + return ot_matrix, log + else: + return ot_matrix + + +def _barycenter_unbalanced_stabilized(A, M, reg, reg_m, weights=None, tau=1e3, + numItermax=1000, stopThr=1e-6, + verbose=False, log=False): + r"""Compute the entropic unbalanced wasserstein barycenter of A with stabilization. The function solves the following optimization problem: @@ -412,28 +665,184 @@ def barycenter_unbalanced(A, M, reg, alpha, weights=None, numItermax=1000, where : - - :math:`Wu_{reg}(\cdot,\cdot)` is the unbalanced entropic regularized Wasserstein distance (see ot.unbalanced.sinkhorn_unbalanced) - - :math:`\mathbf{a}_i` are training distributions in the columns of matrix :math:`\mathbf{A}` - - reg and :math:`\mathbf{M}` are respectively the regularization term and the cost matrix for OT - - alpha is the marginal relaxation hyperparameter - The algorithm used for solving the problem is the generalized Sinkhorn-Knopp matrix scaling algorithm as proposed in [10]_ + - :math:`Wu_{reg}(\cdot,\cdot)` is the unbalanced entropic regularized + Wasserstein distance (see ot.unbalanced.sinkhorn_unbalanced) + - :math:`\mathbf{a}_i` are training distributions in the columns of + matrix :math:`\mathbf{A}` + - reg and :math:`\mathbf{M}` are respectively the regularization term and + the cost matrix for OT + - reg_mis the marginal relaxation hyperparameter + The algorithm used for solving the problem is the generalized + Sinkhorn-Knopp matrix scaling algorithm as proposed in [10]_ Parameters ---------- - A : np.ndarray (d,n) - n training distributions a_i of size d - M : np.ndarray (d,d) - loss matrix for OT + A : np.ndarray (dim, n_hists) + `n_hists` training distributions a_i of dimension dim + M : np.ndarray (dim, dim) + ground metric matrix for OT. reg : float Entropy regularization term > 0 - alpha : float + reg_m : float Marginal relaxation term > 0 - weights : np.ndarray (n,) - Weights of each histogram a_i on the simplex (barycentric coodinates) + tau : float + Stabilization threshold for log domain absorption. + weights : np.ndarray (n_hists,) optional + Weight of each distribution (barycentric coodinates) + If None, uniform weights are used. numItermax : int, optional Max number of iterations stopThr : float, optional - Stop threshol on error (>0) + Stop threshol on error (> 0) + verbose : bool, optional + Print information along iterations + log : bool, optional + record log if True + + + Returns + ------- + a : (dim,) ndarray + Unbalanced Wasserstein barycenter + log : dict + log dictionary return only if log==True in parameters + + + References + ---------- + + .. [3] Benamou, J. D., Carlier, G., Cuturi, M., Nenna, L., & Peyré, + G. (2015). Iterative Bregman projections for regularized transportation + problems. SIAM Journal on Scientific Computing, 37(2), A1111-A1138. + .. [10] Chizat, L., Peyré, G., Schmitzer, B., & Vialard, F. X. (2016). + Scaling algorithms for unbalanced transport problems. arXiv preprint + arXiv:1607.05816. + + + """ + dim, n_hists = A.shape + if weights is None: + weights = np.ones(n_hists) / n_hists + else: + assert(len(weights) == A.shape[1]) + + if log: + log = {'err': []} + + fi = reg_m / (reg_m + reg) + + u = np.ones((dim, n_hists)) / dim + v = np.ones((dim, n_hists)) / dim + + # print(reg) + # Next 3 lines equivalent to K= np.exp(-M/reg), but faster to compute + K = np.empty(M.shape, dtype=M.dtype) + np.divide(M, -reg, out=K) + np.exp(K, out=K) + + fi = reg_m / (reg_m + reg) + + cpt = 0 + err = 1. + alpha = np.zeros(dim) + beta = np.zeros(dim) + q = np.ones(dim) / dim + while (err > stopThr and cpt < numItermax): + qprev = q + Kv = K.dot(v) + f_alpha = np.exp(- alpha / (reg + reg_m)) + f_beta = np.exp(- beta / (reg + reg_m)) + f_alpha = f_alpha[:, None] + f_beta = f_beta[:, None] + u = ((A / (Kv + 1e-16)) ** fi) * f_alpha + Ktu = K.T.dot(u) + q = (Ktu ** (1 - fi)) * f_beta + q = q.dot(weights) ** (1 / (1 - fi)) + Q = q[:, None] + v = ((Q / (Ktu + 1e-16)) ** fi) * f_beta + absorbing = False + if (u > tau).any() or (v > tau).any(): + absorbing = True + alpha = alpha + reg * np.log(np.max(u, 1)) + beta = beta + reg * np.log(np.max(v, 1)) + K = np.exp((alpha[:, None] + beta[None, :] - + M) / reg) + v = np.ones_like(v) + Kv = K.dot(v) + if (np.any(Ktu == 0.) + or np.any(np.isnan(u)) or np.any(np.isnan(v)) + or np.any(np.isinf(u)) or np.any(np.isinf(v))): + # we have reached the machine precision + # come back to previous solution and quit loop + warnings.warn('Numerical errors at iteration %s' % cpt) + q = qprev + break + if (cpt % 10 == 0 and not absorbing) or cpt == 0: + # we can speed up the process by checking for the error only all + # the 10th iterations + err = abs(q - qprev).max() / max(abs(q).max(), + abs(qprev).max(), 1.) + if log: + log['err'].append(err) + if verbose: + if cpt % 50 == 0: + print( + '{:5s}|{:12s}'.format('It.', 'Err') + '\n' + '-' * 19) + print('{:5d}|{:8e}|'.format(cpt, err)) + + cpt += 1 + if err > stopThr: + warnings.warn("Stabilized Unbalanced Sinkhorn did not converge." + + "Try a larger entropy `reg` or a lower mass `reg_m`." + + "Or a larger absorption threshold `tau`.") + if log: + log['niter'] = cpt + log['logu'] = np.log(u + 1e-16) + log['logv'] = np.log(v + 1e-16) + return q, log + else: + return q + + +def _barycenter_unbalanced(A, M, reg, reg_m, weights=None, + numItermax=1000, stopThr=1e-6, + verbose=False, log=False): + r"""Compute the entropic unbalanced wasserstein barycenter of A. + + The function solves the following optimization problem with a + + .. math:: + \mathbf{a} = arg\min_\mathbf{a} \sum_i Wu_{reg}(\mathbf{a},\mathbf{a}_i) + + where : + + - :math:`Wu_{reg}(\cdot,\cdot)` is the unbalanced entropic regularized + Wasserstein distance (see ot.unbalanced.sinkhorn_unbalanced) + - :math:`\mathbf{a}_i` are training distributions in the columns of matrix + :math:`\mathbf{A}` + - reg and :math:`\mathbf{M}` are respectively the regularization term and + the cost matrix for OT + - reg_mis the marginal relaxation hyperparameter + The algorithm used for solving the problem is the generalized + Sinkhorn-Knopp matrix scaling algorithm as proposed in [10]_ + + Parameters + ---------- + A : np.ndarray (dim, n_hists) + `n_hists` training distributions a_i of dimension dim + M : np.ndarray (dim, dim) + ground metric matrix for OT. + reg : float + Entropy regularization term > 0 + reg_m: float + Marginal relaxation term > 0 + weights : np.ndarray (n_hists,) optional + Weight of each distribution (barycentric coodinates) + If None, uniform weights are used. + numItermax : int, optional + Max number of iterations + stopThr : float, optional + Stop threshol on error (> 0) verbose : bool, optional Print information along iterations log : bool, optional @@ -442,7 +851,7 @@ def barycenter_unbalanced(A, M, reg, alpha, weights=None, numItermax=1000, Returns ------- - a : (d,) ndarray + a : (dim,) ndarray Unbalanced Wasserstein barycenter log : dict log dictionary return only if log==True in parameters @@ -451,12 +860,16 @@ def barycenter_unbalanced(A, M, reg, alpha, weights=None, numItermax=1000, References ---------- - .. [3] Benamou, J. D., Carlier, G., Cuturi, M., Nenna, L., & Peyré, G. (2015). Iterative Bregman projections for regularized transportation problems. SIAM Journal on Scientific Computing, 37(2), A1111-A1138. - .. [10] Chizat, L., Peyré, G., Schmitzer, B., & Vialard, F. X. (2016). Scaling algorithms for unbalanced transport problems. arXiv preprint arXiv:1607.05816. + .. [3] Benamou, J. D., Carlier, G., Cuturi, M., Nenna, L., & Peyré, G. + (2015). Iterative Bregman projections for regularized transportation + problems. SIAM Journal on Scientific Computing, 37(2), A1111-A1138. + .. [10] Chizat, L., Peyré, G., Schmitzer, B., & Vialard, F. X. (2016). + Scaling algorithms for unbalanced transport problems. arXiv preprin + arXiv:1607.05816. """ - p, n_hists = A.shape + dim, n_hists = A.shape if weights is None: weights = np.ones(n_hists) / n_hists else: @@ -467,10 +880,10 @@ def barycenter_unbalanced(A, M, reg, alpha, weights=None, numItermax=1000, K = np.exp(- M / reg) - fi = alpha / (alpha + reg) + fi = reg_m / (reg_m + reg) - v = np.ones((p, n_hists)) / p - u = np.ones((p, 1)) / p + v = np.ones((dim, n_hists)) / dim + u = np.ones((dim, 1)) / dim cpt = 0 err = 1. @@ -499,8 +912,11 @@ def barycenter_unbalanced(A, M, reg, alpha, weights=None, numItermax=1000, if cpt % 10 == 0: # we can speed up the process by checking for the error only all # the 10th iterations - err = np.sum((u - uprev) ** 2) / np.sum((u) ** 2) + \ - np.sum((v - vprev) ** 2) / np.sum((v) ** 2) + err_u = abs(u - uprev).max() + err_u /= max(abs(u).max(), abs(uprev).max(), 1.) + err_v = abs(v - vprev).max() + err_v /= max(abs(v).max(), abs(vprev).max(), 1.) + err = 0.5 * (err_u + err_v) if log: log['err'].append(err) if verbose: @@ -512,8 +928,95 @@ def barycenter_unbalanced(A, M, reg, alpha, weights=None, numItermax=1000, cpt += 1 if log: log['niter'] = cpt - log['u'] = u - log['v'] = v + log['logu'] = np.log(u + 1e-16) + log['logv'] = np.log(v + 1e-16) return q, log else: return q + + +def barycenter_unbalanced(A, M, reg, reg_m, method="sinkhorn", weights=None, + numItermax=1000, stopThr=1e-6, + verbose=False, log=False, **kwargs): + r"""Compute the entropic unbalanced wasserstein barycenter of A. + + The function solves the following optimization problem with a + + .. math:: + \mathbf{a} = arg\min_\mathbf{a} \sum_i Wu_{reg}(\mathbf{a},\mathbf{a}_i) + + where : + + - :math:`Wu_{reg}(\cdot,\cdot)` is the unbalanced entropic regularized + Wasserstein distance (see ot.unbalanced.sinkhorn_unbalanced) + - :math:`\mathbf{a}_i` are training distributions in the columns of matrix + :math:`\mathbf{A}` + - reg and :math:`\mathbf{M}` are respectively the regularization term and + the cost matrix for OT + - reg_mis the marginal relaxation hyperparameter + The algorithm used for solving the problem is the generalized + Sinkhorn-Knopp matrix scaling algorithm as proposed in [10]_ + + Parameters + ---------- + A : np.ndarray (dim, n_hists) + `n_hists` training distributions a_i of dimension dim + M : np.ndarray (dim, dim) + ground metric matrix for OT. + reg : float + Entropy regularization term > 0 + reg_m: float + Marginal relaxation term > 0 + weights : np.ndarray (n_hists,) optional + Weight of each distribution (barycentric coodinates) + If None, uniform weights are used. + numItermax : int, optional + Max number of iterations + stopThr : float, optional + Stop threshol on error (> 0) + verbose : bool, optional + Print information along iterations + log : bool, optional + record log if True + + + Returns + ------- + a : (dim,) ndarray + Unbalanced Wasserstein barycenter + log : dict + log dictionary return only if log==True in parameters + + + References + ---------- + + .. [3] Benamou, J. D., Carlier, G., Cuturi, M., Nenna, L., & Peyré, G. + (2015). Iterative Bregman projections for regularized transportation + problems. SIAM Journal on Scientific Computing, 37(2), A1111-A1138. + .. [10] Chizat, L., Peyré, G., Schmitzer, B., & Vialard, F. X. (2016). + Scaling algorithms for unbalanced transport problems. arXiv preprin + arXiv:1607.05816. + + """ + + if method.lower() == 'sinkhorn': + return _barycenter_unbalanced(A, M, reg, reg_m, + numItermax=numItermax, + stopThr=stopThr, verbose=verbose, + log=log, **kwargs) + + elif method.lower() == 'sinkhorn_stabilized': + return _barycenter_unbalanced_stabilized(A, M, reg, reg_m, + numItermax=numItermax, + stopThr=stopThr, + verbose=verbose, + log=log, **kwargs) + elif method.lower() in ['sinkhorn_reg_scaling']: + warnings.warn('Method not implemented yet. Using classic Sinkhorn Knopp') + return _barycenter_unbalanced(A, M, reg, reg_m, + numItermax=numItermax, + stopThr=stopThr, verbose=verbose, + log=log, **kwargs) + else: + raise ValueError("Unknown method '%s'." % method) diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..e69de29 diff --git a/test/test_bregman.py b/test/test_bregman.py index 83ebba8..f70df10 100644 --- a/test/test_bregman.py +++ b/test/test_bregman.py @@ -7,6 +7,7 @@ import numpy as np import ot +import pytest def test_sinkhorn(): @@ -71,13 +72,11 @@ def test_sinkhorn_variants(): Gs = ot.sinkhorn(u, u, M, 1, method='sinkhorn_stabilized', stopThr=1e-10) Ges = ot.sinkhorn( u, u, M, 1, method='sinkhorn_epsilon_scaling', stopThr=1e-10) - Gerr = ot.sinkhorn(u, u, M, 1, method='do_not_exists', stopThr=1e-10) G_green = ot.sinkhorn(u, u, M, 1, method='greenkhorn', stopThr=1e-10) # check values np.testing.assert_allclose(G0, Gs, atol=1e-05) np.testing.assert_allclose(G0, Ges, atol=1e-05) - np.testing.assert_allclose(G0, Gerr) np.testing.assert_allclose(G0, G_green, atol=1e-5) print(G0, G_green) @@ -96,18 +95,17 @@ def test_sinkhorn_variants_log(): Gs, logs = ot.sinkhorn(u, u, M, 1, method='sinkhorn_stabilized', stopThr=1e-10, log=True) Ges, loges = ot.sinkhorn( u, u, M, 1, method='sinkhorn_epsilon_scaling', stopThr=1e-10, log=True) - Gerr, logerr = ot.sinkhorn(u, u, M, 1, method='do_not_exists', stopThr=1e-10, log=True) G_green, loggreen = ot.sinkhorn(u, u, M, 1, method='greenkhorn', stopThr=1e-10, log=True) # check values np.testing.assert_allclose(G0, Gs, atol=1e-05) np.testing.assert_allclose(G0, Ges, atol=1e-05) - np.testing.assert_allclose(G0, Gerr) np.testing.assert_allclose(G0, G_green, atol=1e-5) print(G0, G_green) -def test_bary(): +@pytest.mark.parametrize("method", ["sinkhorn", "sinkhorn_stabilized"]) +def test_barycenter(method): n_bins = 100 # nb bins @@ -126,14 +124,42 @@ def test_bary(): weights = np.array([1 - alpha, alpha]) # wasserstein - reg = 1e-3 - bary_wass = ot.bregman.barycenter(A, M, reg, weights) + reg = 1e-2 + bary_wass = ot.bregman.barycenter(A, M, reg, weights, method=method) np.testing.assert_allclose(1, np.sum(bary_wass)) ot.bregman.barycenter(A, M, reg, log=True, verbose=True) +def test_barycenter_stabilization(): + + n_bins = 100 # nb bins + + # Gaussian distributions + a1 = ot.datasets.make_1D_gauss(n_bins, m=30, s=10) # m= mean, s= std + a2 = ot.datasets.make_1D_gauss(n_bins, m=40, s=10) + + # creating matrix A containing all distributions + A = np.vstack((a1, a2)).T + + # loss matrix + normalization + M = ot.utils.dist0(n_bins) + M /= M.max() + + alpha = 0.5 # 0<=alpha<=1 + weights = np.array([1 - alpha, alpha]) + + # wasserstein + reg = 1e-2 + bar_stable = ot.bregman.barycenter(A, M, reg, weights, + method="sinkhorn_stabilized", + stopThr=1e-8) + bar = ot.bregman.barycenter(A, M, reg, weights, method="sinkhorn", + stopThr=1e-8) + np.testing.assert_allclose(bar, bar_stable) + + def test_wasserstein_bary_2d(): size = 100 # size of a square image @@ -279,3 +305,35 @@ def test_stabilized_vs_sinkhorn_multidim(): method="sinkhorn", log=True) np.testing.assert_allclose(G, G2) + + +def test_implemented_methods(): + IMPLEMENTED_METHODS = ['sinkhorn', 'sinkhorn_stabilized'] + ONLY_1D_methods = ['greenkhorn', 'sinkhorn_epsilon_scaling'] + NOT_VALID_TOKENS = ['foo'] + # test generalized sinkhorn for unbalanced OT barycenter + n = 3 + rng = np.random.RandomState(42) + + x = rng.randn(n, 2) + a = ot.utils.unif(n) + + # make dists unbalanced + b = ot.utils.unif(n) + A = rng.rand(n, 2) + M = ot.dist(x, x) + epsilon = 1. + + for method in IMPLEMENTED_METHODS: + ot.bregman.sinkhorn(a, b, M, epsilon, method=method) + ot.bregman.sinkhorn2(a, b, M, epsilon, method=method) + ot.bregman.barycenter(A, M, reg=epsilon, method=method) + with pytest.raises(ValueError): + for method in set(NOT_VALID_TOKENS): + ot.bregman.sinkhorn(a, b, M, epsilon, method=method) + ot.bregman.sinkhorn2(a, b, M, epsilon, method=method) + ot.bregman.barycenter(A, M, reg=epsilon, method=method) + for method in ONLY_1D_methods: + ot.bregman.sinkhorn(a, b, M, epsilon, method=method) + with pytest.raises(ValueError): + ot.bregman.sinkhorn2(a, b, M, epsilon, method=method) diff --git a/test/test_unbalanced.py b/test/test_unbalanced.py index 1395fe1..ca1efba 100644 --- a/test/test_unbalanced.py +++ b/test/test_unbalanced.py @@ -7,9 +7,12 @@ import numpy as np import ot import pytest +from ot.unbalanced import barycenter_unbalanced +from scipy.special import logsumexp -@pytest.mark.parametrize("method", ["sinkhorn"]) + +@pytest.mark.parametrize("method", ["sinkhorn", "sinkhorn_stabilized"]) def test_unbalanced_convergence(method): # test generalized sinkhorn for unbalanced OT n = 100 @@ -23,29 +26,35 @@ def test_unbalanced_convergence(method): M = ot.dist(x, x) epsilon = 1. - alpha = 1. - K = np.exp(- M / epsilon) + reg_m = 1. - G, log = ot.unbalanced.sinkhorn_unbalanced(a, b, M, reg=epsilon, alpha=alpha, - stopThr=1e-10, method=method, + G, log = ot.unbalanced.sinkhorn_unbalanced(a, b, M, reg=epsilon, + reg_m=reg_m, + method=method, log=True) - loss = ot.unbalanced.sinkhorn_unbalanced2(a, b, M, epsilon, alpha, + loss = ot.unbalanced.sinkhorn_unbalanced2(a, b, M, epsilon, reg_m, method=method) # check fixed point equations - fi = alpha / (alpha + epsilon) - v_final = (b / K.T.dot(log["u"])) ** fi - u_final = (a / K.dot(log["v"])) ** fi + # in log-domain + fi = reg_m / (reg_m + epsilon) + logb = np.log(b + 1e-16) + loga = np.log(a + 1e-16) + logKtu = logsumexp(log["logu"][None, :] - M.T / epsilon, axis=1) + logKv = logsumexp(log["logv"][None, :] - M / epsilon, axis=1) + + v_final = fi * (logb - logKtu) + u_final = fi * (loga - logKv) np.testing.assert_allclose( - u_final, log["u"], atol=1e-05) + u_final, log["logu"], atol=1e-05) np.testing.assert_allclose( - v_final, log["v"], atol=1e-05) + v_final, log["logv"], atol=1e-05) # check if sinkhorn_unbalanced2 returns the correct loss np.testing.assert_allclose((G * M).sum(), loss, atol=1e-5) -@pytest.mark.parametrize("method", ["sinkhorn"]) +@pytest.mark.parametrize("method", ["sinkhorn", "sinkhorn_stabilized"]) def test_unbalanced_multiple_inputs(method): # test generalized sinkhorn for unbalanced OT n = 100 @@ -59,28 +68,59 @@ def test_unbalanced_multiple_inputs(method): M = ot.dist(x, x) epsilon = 1. - alpha = 1. - K = np.exp(- M / epsilon) + reg_m = 1. loss, log = ot.unbalanced.sinkhorn_unbalanced(a, b, M, reg=epsilon, - alpha=alpha, - stopThr=1e-10, method=method, + reg_m=reg_m, + method=method, log=True) # check fixed point equations - fi = alpha / (alpha + epsilon) - v_final = (b / K.T.dot(log["u"])) ** fi - - u_final = (a[:, None] / K.dot(log["v"])) ** fi + # in log-domain + fi = reg_m / (reg_m + epsilon) + logb = np.log(b + 1e-16) + loga = np.log(a + 1e-16)[:, None] + logKtu = logsumexp(log["logu"][:, None, :] - M[:, :, None] / epsilon, + axis=0) + logKv = logsumexp(log["logv"][None, :] - M[:, :, None] / epsilon, axis=1) + v_final = fi * (logb - logKtu) + u_final = fi * (loga - logKv) np.testing.assert_allclose( - u_final, log["u"], atol=1e-05) + u_final, log["logu"], atol=1e-05) np.testing.assert_allclose( - v_final, log["v"], atol=1e-05) + v_final, log["logv"], atol=1e-05) assert len(loss) == b.shape[1] -def test_unbalanced_barycenter(): +def test_stabilized_vs_sinkhorn(): + # test if stable version matches sinkhorn + n = 100 + + # Gaussian distributions + a = ot.datasets.make_1D_gauss(n, m=20, s=5) # m= mean, s= std + b1 = ot.datasets.make_1D_gauss(n, m=60, s=8) + b2 = ot.datasets.make_1D_gauss(n, m=30, s=4) + + # creating matrix A containing all distributions + b = np.vstack((b1, b2)).T + + M = ot.utils.dist0(n) + M /= np.median(M) + epsilon = 0.1 + reg_m = 1. + G, log = ot.unbalanced.sinkhorn_unbalanced2(a, b, M, reg=epsilon, + method="sinkhorn_stabilized", + reg_m=reg_m, + log=True) + G2, log2 = ot.unbalanced.sinkhorn_unbalanced2(a, b, M, epsilon, reg_m, + method="sinkhorn", log=True) + + np.testing.assert_allclose(G, G2, atol=1e-5) + + +@pytest.mark.parametrize("method", ["sinkhorn", "sinkhorn_stabilized"]) +def test_unbalanced_barycenter(method): # test generalized sinkhorn for unbalanced OT barycenter n = 100 rng = np.random.RandomState(42) @@ -92,27 +132,56 @@ def test_unbalanced_barycenter(): A = A * np.array([1, 2])[None, :] M = ot.dist(x, x) epsilon = 1. - alpha = 1. - K = np.exp(- M / epsilon) + reg_m = 1. - q, log = ot.unbalanced.barycenter_unbalanced(A, M, reg=epsilon, alpha=alpha, - stopThr=1e-10, - log=True) + q, log = barycenter_unbalanced(A, M, reg=epsilon, reg_m=reg_m, + method=method, log=True) # check fixed point equations - fi = alpha / (alpha + epsilon) - v_final = (q[:, None] / K.T.dot(log["u"])) ** fi - u_final = (A / K.dot(log["v"])) ** fi + fi = reg_m / (reg_m + epsilon) + logA = np.log(A + 1e-16) + logq = np.log(q + 1e-16)[:, None] + logKtu = logsumexp(log["logu"][:, None, :] - M[:, :, None] / epsilon, + axis=0) + logKv = logsumexp(log["logv"][None, :] - M[:, :, None] / epsilon, axis=1) + v_final = fi * (logq - logKtu) + u_final = fi * (logA - logKv) np.testing.assert_allclose( - u_final, log["u"], atol=1e-05) + u_final, log["logu"], atol=1e-05) np.testing.assert_allclose( - v_final, log["v"], atol=1e-05) + v_final, log["logv"], atol=1e-05) + + +def test_barycenter_stabilized_vs_sinkhorn(): + # test generalized sinkhorn for unbalanced OT barycenter + n = 100 + rng = np.random.RandomState(42) + + x = rng.randn(n, 2) + A = rng.rand(n, 2) + + # make dists unbalanced + A = A * np.array([1, 4])[None, :] + M = ot.dist(x, x) + epsilon = 0.5 + reg_m = 10 + + qstable, log = barycenter_unbalanced(A, M, reg=epsilon, + reg_m=reg_m, log=True, + tau=100, + method="sinkhorn_stabilized", + ) + q, log = barycenter_unbalanced(A, M, reg=epsilon, reg_m=reg_m, + method="sinkhorn", + log=True) + + np.testing.assert_allclose( + q, qstable, atol=1e-05) def test_implemented_methods(): - IMPLEMENTED_METHODS = ['sinkhorn'] - TO_BE_IMPLEMENTED_METHODS = ['sinkhorn_stabilized', - 'sinkhorn_epsilon_scaling'] + IMPLEMENTED_METHODS = ['sinkhorn', 'sinkhorn_stabilized'] + TO_BE_IMPLEMENTED_METHODS = ['sinkhorn_reg_scaling'] NOT_VALID_TOKENS = ['foo'] # test generalized sinkhorn for unbalanced OT barycenter n = 3 @@ -123,24 +192,30 @@ def test_implemented_methods(): # make dists unbalanced b = ot.utils.unif(n) * 1.5 - + A = rng.rand(n, 2) M = ot.dist(x, x) epsilon = 1. - alpha = 1. + reg_m = 1. for method in IMPLEMENTED_METHODS: - ot.unbalanced.sinkhorn_unbalanced(a, b, M, epsilon, alpha, + ot.unbalanced.sinkhorn_unbalanced(a, b, M, epsilon, reg_m, method=method) - ot.unbalanced.sinkhorn_unbalanced2(a, b, M, epsilon, alpha, + ot.unbalanced.sinkhorn_unbalanced2(a, b, M, epsilon, reg_m, method=method) + barycenter_unbalanced(A, M, reg=epsilon, reg_m=reg_m, + method=method) with pytest.warns(UserWarning, match='not implemented'): for method in set(TO_BE_IMPLEMENTED_METHODS): - ot.unbalanced.sinkhorn_unbalanced(a, b, M, epsilon, alpha, + ot.unbalanced.sinkhorn_unbalanced(a, b, M, epsilon, reg_m, method=method) - ot.unbalanced.sinkhorn_unbalanced2(a, b, M, epsilon, alpha, + ot.unbalanced.sinkhorn_unbalanced2(a, b, M, epsilon, reg_m, method=method) + barycenter_unbalanced(A, M, reg=epsilon, reg_m=reg_m, + method=method) with pytest.raises(ValueError): for method in set(NOT_VALID_TOKENS): - ot.unbalanced.sinkhorn_unbalanced(a, b, M, epsilon, alpha, + ot.unbalanced.sinkhorn_unbalanced(a, b, M, epsilon, reg_m, method=method) - ot.unbalanced.sinkhorn_unbalanced2(a, b, M, epsilon, alpha, + ot.unbalanced.sinkhorn_unbalanced2(a, b, M, epsilon, reg_m, method=method) + barycenter_unbalanced(A, M, reg=epsilon, reg_m=reg_m, + method=method) -- cgit v1.2.3 From f4e8523c92a96d061040e3f25037e129d67a2d94 Mon Sep 17 00:00:00 2001 From: Hicham Janati Date: Wed, 28 Aug 2019 15:47:52 +0200 Subject: fix empirical sinkhorn doc-example --- ot/bregman.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'ot/bregman.py') diff --git a/ot/bregman.py b/ot/bregman.py index 2f27d58..76698c2 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -1534,11 +1534,11 @@ def empirical_sinkhorn(X_s, X_t, reg, a=None, b=None, metric='sqeuclidean', numI Examples -------- - >>> n_a = 2 - >>> n_b = 2 + >>> n_samples_a = 2 + >>> n_samples_b = 2 >>> reg = 0.1 - >>> X_s = np.reshape(np.arange(n_a), (dim_a, 1)) - >>> X_t = np.reshape(np.arange(0, n_b), (dim_b, 1)) + >>> X_s = np.reshape(np.arange(n_samples_a), (n_samples_a, 1)) + >>> X_t = np.reshape(np.arange(0, n_samples_b), (n_samples_b, 1)) >>> empirical_sinkhorn(X_s, X_t, reg, verbose=False) # doctest: +NORMALIZE_WHITESPACE array([[4.99977301e-01, 2.26989344e-05], [2.26989344e-05, 4.99977301e-01]]) @@ -1624,8 +1624,8 @@ def empirical_sinkhorn2(X_s, X_t, reg, a=None, b=None, metric='sqeuclidean', num Examples -------- - >>> n_a = 2 - >>> n_b = 2 + >>> n_samples_a = 2 + >>> n_samples_b = 2 >>> reg = 0.1 >>> X_s = np.reshape(np.arange(n_samples_a), (n_samples_a, 1)) >>> X_t = np.reshape(np.arange(0, n_samples_b), (n_samples_b, 1)) @@ -1730,8 +1730,8 @@ def empirical_sinkhorn_divergence(X_s, X_t, reg, a=None, b=None, metric='sqeucli Examples -------- - >>> n_a = 2 - >>> n_b = 4 + >>> n_samples_a = 2 + >>> n_samples_b = 4 >>> reg = 0.1 >>> X_s = np.reshape(np.arange(n_samples_a), (n_samples_a, 1)) >>> X_t = np.reshape(np.arange(0, n_samples_b), (n_samples_b, 1)) -- cgit v1.2.3 From c7269d3fc72c679711699a9df7b5670b0dd176b0 Mon Sep 17 00:00:00 2001 From: Hicham Janati Date: Tue, 3 Sep 2019 17:26:22 +0200 Subject: style + make funcs public --- ot/bregman.py | 180 +++++++++++++++++++++++++++++++--------------------------- 1 file changed, 95 insertions(+), 85 deletions(-) (limited to 'ot/bregman.py') diff --git a/ot/bregman.py b/ot/bregman.py index 76698c2..02aeb6d 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -33,7 +33,7 @@ def sinkhorn(a, b, M, reg, method='sinkhorn', numItermax=1000, \gamma\geq 0 where : - - M is the (dim_a, n_b) metric cost matrix + - M is the (dim_a, dim_b) metric cost matrix - :math:`\Omega` is the entropic regularization term :math:`\Omega(\gamma)=\sum_{i,j} \gamma_{i,j}\log(\gamma_{i,j})` - a and b are source and target weights (sum to 1) @@ -47,7 +47,7 @@ def sinkhorn(a, b, M, reg, method='sinkhorn', numItermax=1000, b : ndarray, shape (dim_b,) or ndarray, shape (dim_b, n_hists) samples in the target domain, compute sinkhorn with multiple targets and fixed M if b is a matrix (return OT loss + dual variables in log) - M : ndarray, shape (dim_a, n_b) + M : ndarray, shape (dim_a, dim_b) loss matrix reg : float Regularization term >0 @@ -66,7 +66,7 @@ def sinkhorn(a, b, M, reg, method='sinkhorn', numItermax=1000, Returns ------- - gamma : ndarray, shape (dim_a, n_b) + gamma : ndarray, shape (dim_a, dim_b) Optimal transportation matrix for the given parameters log : dict log dictionary return only if log==True in parameters @@ -105,21 +105,21 @@ def sinkhorn(a, b, M, reg, method='sinkhorn', numItermax=1000, """ if method.lower() == 'sinkhorn': - return _sinkhorn_knopp(a, b, M, reg, numItermax=numItermax, - stopThr=stopThr, verbose=verbose, log=log, - **kwargs) + return sinkhorn_knopp(a, b, M, reg, numItermax=numItermax, + stopThr=stopThr, verbose=verbose, log=log, + **kwargs) elif method.lower() == 'greenkhorn': - return _greenkhorn(a, b, M, reg, numItermax=numItermax, - stopThr=stopThr, verbose=verbose, log=log) + return greenkhorn(a, b, M, reg, numItermax=numItermax, + stopThr=stopThr, verbose=verbose, log=log) elif method.lower() == 'sinkhorn_stabilized': - return _sinkhorn_stabilized(a, b, M, reg, numItermax=numItermax, - stopThr=stopThr, verbose=verbose, - log=log, **kwargs) + return sinkhorn_stabilized(a, b, M, reg, numItermax=numItermax, + stopThr=stopThr, verbose=verbose, + log=log, **kwargs) elif method.lower() == 'sinkhorn_epsilon_scaling': - return _sinkhorn_epsilon_scaling(a, b, M, reg, - numItermax=numItermax, - stopThr=stopThr, verbose=verbose, - log=log, **kwargs) + return sinkhorn_epsilon_scaling(a, b, M, reg, + numItermax=numItermax, + stopThr=stopThr, verbose=verbose, + log=log, **kwargs) else: raise ValueError("Unknown method '%s'." % method) @@ -141,7 +141,7 @@ def sinkhorn2(a, b, M, reg, method='sinkhorn', numItermax=1000, \gamma\geq 0 where : - - M is the (dim_a, n_b) metric cost matrix + - M is the (dim_a, dim_b) metric cost matrix - :math:`\Omega` is the entropic regularization term :math:`\Omega(\gamma)=\sum_{i,j} \gamma_{i,j}\log(\gamma_{i,j})` - a and b are source and target weights (sum to 1) @@ -155,7 +155,7 @@ def sinkhorn2(a, b, M, reg, method='sinkhorn', numItermax=1000, b : ndarray, shape (dim_b,) or ndarray, shape (dim_b, n_hists) samples in the target domain, compute sinkhorn with multiple targets and fixed M if b is a matrix (return OT loss + dual variables in log) - M : ndarray, shape (dim_a, n_b) + M : ndarray, shape (dim_a, dim_b) loss matrix reg : float Regularization term >0 @@ -173,8 +173,8 @@ def sinkhorn2(a, b, M, reg, method='sinkhorn', numItermax=1000, Returns ------- - W : (nt) ndarray or float - Optimal transportation matrix for the given parameters + W : (n_hists) ndarray or float + Optimal transportation loss for the given parameters log : dict log dictionary return only if log==True in parameters @@ -217,21 +217,23 @@ def sinkhorn2(a, b, M, reg, method='sinkhorn', numItermax=1000, if len(b.shape) < 2: b = b[:, None] if method.lower() == 'sinkhorn': - return _sinkhorn_knopp(a, b, M, reg, numItermax=numItermax, - stopThr=stopThr, verbose=verbose, log=log, **kwargs) + return sinkhorn_knopp(a, b, M, reg, numItermax=numItermax, + stopThr=stopThr, verbose=verbose, log=log, + **kwargs) elif method.lower() == 'sinkhorn_stabilized': - return _sinkhorn_stabilized(a, b, M, reg, numItermax=numItermax, - stopThr=stopThr, verbose=verbose, log=log, **kwargs) + return sinkhorn_stabilized(a, b, M, reg, numItermax=numItermax, + stopThr=stopThr, verbose=verbose, log=log, + **kwargs) elif method.lower() == 'sinkhorn_epsilon_scaling': - return _sinkhorn_epsilon_scaling(a, b, M, reg, numItermax=numItermax, - stopThr=stopThr, verbose=verbose, - log=log, **kwargs) + return sinkhorn_epsilon_scaling(a, b, M, reg, numItermax=numItermax, + stopThr=stopThr, verbose=verbose, + log=log, **kwargs) else: raise ValueError("Unknown method '%s'." % method) -def _sinkhorn_knopp(a, b, M, reg, numItermax=1000, - stopThr=1e-9, verbose=False, log=False, **kwargs): +def sinkhorn_knopp(a, b, M, reg, numItermax=1000, + stopThr=1e-9, verbose=False, log=False, **kwargs): r""" Solve the entropic regularization optimal transport problem and return the OT matrix @@ -247,7 +249,7 @@ def _sinkhorn_knopp(a, b, M, reg, numItermax=1000, \gamma\geq 0 where : - - M is the (dim_a, n_b) metric cost matrix + - M is the (dim_a, dim_b) metric cost matrix - :math:`\Omega` is the entropic regularization term :math:`\Omega(\gamma)=\sum_{i,j} \gamma_{i,j}\log(\gamma_{i,j})` - a and b are source and target weights (sum to 1) @@ -261,7 +263,7 @@ def _sinkhorn_knopp(a, b, M, reg, numItermax=1000, b : ndarray, shape (dim_b,) or ndarray, shape (dim_b, n_hists) samples in the target domain, compute sinkhorn with multiple targets and fixed M if b is a matrix (return OT loss + dual variables in log) - M : ndarray, shape (dim_a, n_b) + M : ndarray, shape (dim_a, dim_b) loss matrix reg : float Regularization term >0 @@ -276,7 +278,7 @@ def _sinkhorn_knopp(a, b, M, reg, numItermax=1000, Returns ------- - gamma : ndarray, shape (dim_a, n_b) + gamma : ndarray, shape (dim_a, dim_b) Optimal transportation matrix for the given parameters log : dict log dictionary return only if log==True in parameters @@ -403,7 +405,8 @@ def _sinkhorn_knopp(a, b, M, reg, numItermax=1000, return u.reshape((-1, 1)) * K * v.reshape((1, -1)) -def _greenkhorn(a, b, M, reg, numItermax=10000, stopThr=1e-9, verbose=False, log=False): +def greenkhorn(a, b, M, reg, numItermax=10000, stopThr=1e-9, verbose=False, + log=False): r""" Solve the entropic regularization optimal transport problem and return the OT matrix @@ -427,7 +430,7 @@ def _greenkhorn(a, b, M, reg, numItermax=10000, stopThr=1e-9, verbose=False, log \gamma\geq 0 where : - - M is the (dim_a, n_b) metric cost matrix + - M is the (dim_a, dim_b) metric cost matrix - :math:`\Omega` is the entropic regularization term :math:`\Omega(\gamma)=\sum_{i,j} \gamma_{i,j}\log(\gamma_{i,j})` - a and b are source and target weights (sum to 1) @@ -440,7 +443,7 @@ def _greenkhorn(a, b, M, reg, numItermax=10000, stopThr=1e-9, verbose=False, log b : ndarray, shape (dim_b,) or ndarray, shape (dim_b, n_hists) samples in the target domain, compute sinkhorn with multiple targets and fixed M if b is a matrix (return OT loss + dual variables in log) - M : ndarray, shape (dim_a, n_b) + M : ndarray, shape (dim_a, dim_b) loss matrix reg : float Regularization term >0 @@ -453,7 +456,7 @@ def _greenkhorn(a, b, M, reg, numItermax=10000, stopThr=1e-9, verbose=False, log Returns ------- - gamma : ndarray, shape (dim_a, n_b) + gamma : ndarray, shape (dim_a, dim_b) Optimal transportation matrix for the given parameters log : dict log dictionary return only if log==True in parameters @@ -465,7 +468,7 @@ def _greenkhorn(a, b, M, reg, numItermax=10000, stopThr=1e-9, verbose=False, log >>> a=[.5, .5] >>> b=[.5, .5] >>> M=[[0., 1.], [1., 0.]] - >>> ot.bregman._greenkhorn(a, b, M, 1) + >>> ot.bregman.greenkhorn(a, b, M, 1) array([[0.36552929, 0.13447071], [0.13447071, 0.36552929]]) @@ -555,9 +558,9 @@ def _greenkhorn(a, b, M, reg, numItermax=10000, stopThr=1e-9, verbose=False, log return G -def _sinkhorn_stabilized(a, b, M, reg, numItermax=1000, tau=1e3, stopThr=1e-9, - warmstart=None, verbose=False, print_period=20, - log=False, **kwargs): +def sinkhorn_stabilized(a, b, M, reg, numItermax=1000, tau=1e3, stopThr=1e-9, + warmstart=None, verbose=False, print_period=20, + log=False, **kwargs): r""" Solve the entropic regularization OT problem with log stabilization @@ -573,7 +576,7 @@ def _sinkhorn_stabilized(a, b, M, reg, numItermax=1000, tau=1e3, stopThr=1e-9, \gamma\geq 0 where : - - M is the (dim_a, n_b) metric cost matrix + - M is the (dim_a, dim_b) metric cost matrix - :math:`\Omega` is the entropic regularization term :math:`\Omega(\gamma)=\sum_{i,j} \gamma_{i,j}\log(\gamma_{i,j})` - a and b are source and target weights (sum to 1) @@ -588,7 +591,7 @@ def _sinkhorn_stabilized(a, b, M, reg, numItermax=1000, tau=1e3, stopThr=1e-9, samples weights in the source domain b : ndarray, shape (dim_b,) samples in the target domain - M : ndarray, shape (dim_a, n_b) + M : ndarray, shape (dim_a, dim_b) loss matrix reg : float Regularization term >0 @@ -607,7 +610,7 @@ def _sinkhorn_stabilized(a, b, M, reg, numItermax=1000, tau=1e3, stopThr=1e-9, Returns ------- - gamma : ndarray, shape (dim_a, n_b) + gamma : ndarray, shape (dim_a, dim_b) Optimal transportation matrix for the given parameters log : dict log dictionary return only if log==True in parameters @@ -619,7 +622,7 @@ def _sinkhorn_stabilized(a, b, M, reg, numItermax=1000, tau=1e3, stopThr=1e-9, >>> a=[.5,.5] >>> b=[.5,.5] >>> M=[[0.,1.],[1.,0.]] - >>> ot.bregman._sinkhorn_stabilized(a, b, M, 1) + >>> ot.bregman.sinkhorn_stabilized(a, b, M, 1) array([[0.36552929, 0.13447071], [0.13447071, 0.36552929]]) @@ -658,8 +661,8 @@ def _sinkhorn_stabilized(a, b, M, reg, numItermax=1000, tau=1e3, stopThr=1e-9, n_hists = 0 # init data - na = len(a) - nb = len(b) + dim_a = len(a) + dim_b = len(b) cpt = 0 if log: @@ -668,24 +671,25 @@ def _sinkhorn_stabilized(a, b, M, reg, numItermax=1000, tau=1e3, stopThr=1e-9, # we assume that no distances are null except those of the diagonal of # distances if warmstart is None: - alpha, beta = np.zeros(na), np.zeros(nb) + alpha, beta = np.zeros(dim_a), np.zeros(dim_b) else: alpha, beta = warmstart if n_hists: - u, v = np.ones((na, n_hists)) / na, np.ones((nb, n_hists)) / nb + u = np.ones((dim_a, n_hists)) / dim_a + v = np.ones((dim_b, n_hists)) / dim_b else: - u, v = np.ones(na) / na, np.ones(nb) / nb + u, v = np.ones(dim_a) / dim_a, np.ones(dim_b) / dim_b def get_K(alpha, beta): """log space computation""" - return np.exp(-(M - alpha.reshape((na, 1)) - - beta.reshape((1, nb))) / reg) + return np.exp(-(M - alpha.reshape((dim_a, 1)) + - beta.reshape((1, dim_b))) / reg) def get_Gamma(alpha, beta, u, v): """log space gamma computation""" - return np.exp(-(M - alpha.reshape((na, 1)) - beta.reshape((1, nb))) - / reg + np.log(u.reshape((na, 1))) + np.log(v.reshape((1, nb)))) + return np.exp(-(M - alpha.reshape((dim_a, 1)) - beta.reshape((1, dim_b))) + / reg + np.log(u.reshape((dim_a, 1))) + np.log(v.reshape((1, dim_b)))) # print(np.min(K)) @@ -711,9 +715,9 @@ def _sinkhorn_stabilized(a, b, M, reg, numItermax=1000, tau=1e3, stopThr=1e-9, else: alpha, beta = alpha + reg * np.log(u), beta + reg * np.log(v) if n_hists: - u, v = np.ones((na, n_hists)) / na, np.ones((nb, n_hists)) / nb + u, v = np.ones((dim_a, n_hists)) / dim_a, np.ones((dim_b, n_hists)) / dim_b else: - u, v = np.ones(na) / na, np.ones(nb) / nb + u, v = np.ones(dim_a) / dim_a, np.ones(dim_b) / dim_b K = get_K(alpha, beta) if cpt % print_period == 0: @@ -782,10 +786,10 @@ def _sinkhorn_stabilized(a, b, M, reg, numItermax=1000, tau=1e3, stopThr=1e-9, return get_Gamma(alpha, beta, u, v) -def _sinkhorn_epsilon_scaling(a, b, M, reg, numItermax=100, epsilon0=1e4, - numInnerItermax=100, tau=1e3, stopThr=1e-9, - warmstart=None, verbose=False, print_period=10, - log=False, **kwargs): +def sinkhorn_epsilon_scaling(a, b, M, reg, numItermax=100, epsilon0=1e4, + numInnerItermax=100, tau=1e3, stopThr=1e-9, + warmstart=None, verbose=False, print_period=10, + log=False, **kwargs): r""" Solve the entropic regularization optimal transport problem with log stabilization and epsilon scaling. @@ -802,7 +806,7 @@ def _sinkhorn_epsilon_scaling(a, b, M, reg, numItermax=100, epsilon0=1e4, \gamma\geq 0 where : - - M is the (dim_a, n_b) metric cost matrix + - M is the (dim_a, dim_b) metric cost matrix - :math:`\Omega` is the entropic regularization term :math:`\Omega(\gamma)=\sum_{i,j} \gamma_{i,j}\log(\gamma_{i,j})` - a and b are source and target weights (sum to 1) @@ -817,7 +821,7 @@ def _sinkhorn_epsilon_scaling(a, b, M, reg, numItermax=100, epsilon0=1e4, samples weights in the source domain b : ndarray, shape (dim_b,) samples in the target domain - M : ndarray, shape (dim_a, n_b) + M : ndarray, shape (dim_a, dim_b) loss matrix reg : float Regularization term >0 @@ -840,7 +844,7 @@ def _sinkhorn_epsilon_scaling(a, b, M, reg, numItermax=100, epsilon0=1e4, Returns ------- - gamma : ndarray, shape (dim_a, n_b) + gamma : ndarray, shape (dim_a, dim_b) Optimal transportation matrix for the given parameters log : dict log dictionary return only if log==True in parameters @@ -852,7 +856,7 @@ def _sinkhorn_epsilon_scaling(a, b, M, reg, numItermax=100, epsilon0=1e4, >>> a=[.5, .5] >>> b=[.5, .5] >>> M=[[0., 1.], [1., 0.]] - >>> ot.bregman._sinkhorn_epsilon_scaling(a, b, M, 1) + >>> ot.bregman.sinkhorn_epsilon_scaling(a, b, M, 1) array([[0.36552929, 0.13447071], [0.13447071, 0.36552929]]) @@ -915,8 +919,10 @@ def _sinkhorn_epsilon_scaling(a, b, M, reg, numItermax=100, epsilon0=1e4, regi = get_reg(cpt) - G, logi = _sinkhorn_stabilized(a, b, M, regi, numItermax=numInnerItermax, stopThr=1e-9, warmstart=( - alpha, beta), verbose=False, print_period=20, tau=tau, log=True) + G, logi = sinkhorn_stabilized(a, b, M, regi, + numItermax=numInnerItermax, stopThr=1e-9, + warmstart=(alpha, beta), verbose=False, + print_period=20, tau=tau, log=True) alpha = logi['alpha'] beta = logi['beta'] @@ -1029,19 +1035,19 @@ def barycenter(A, M, reg, weights=None, method="sinkhorn", numItermax=10000, """ if method.lower() == 'sinkhorn': - return _barycenter(A, M, reg, numItermax=numItermax, - stopThr=stopThr, verbose=verbose, log=log, - **kwargs) + return barycenter_sinkhorn(A, M, reg, numItermax=numItermax, + stopThr=stopThr, verbose=verbose, log=log, + **kwargs) elif method.lower() == 'sinkhorn_stabilized': - return _barycenter_stabilized(A, M, reg, numItermax=numItermax, - stopThr=stopThr, verbose=verbose, - log=log, **kwargs) + return barycenter_stabilized(A, M, reg, numItermax=numItermax, + stopThr=stopThr, verbose=verbose, + log=log, **kwargs) else: raise ValueError("Unknown method '%s'." % method) -def _barycenter(A, M, reg, weights=None, numItermax=1000, - stopThr=1e-4, verbose=False, log=False): +def barycenter_sinkhorn(A, M, reg, weights=None, numItermax=1000, + stopThr=1e-4, verbose=False, log=False): r"""Compute the entropic regularized wasserstein barycenter of distributions A The function solves the following optimization problem: @@ -1134,8 +1140,8 @@ def _barycenter(A, M, reg, weights=None, numItermax=1000, return geometricBar(weights, UKv) -def _barycenter_stabilized(A, M, reg, tau=1e10, weights=None, numItermax=1000, - stopThr=1e-4, verbose=False, log=False): +def barycenter_stabilized(A, M, reg, tau=1e10, weights=None, numItermax=1000, + stopThr=1e-4, verbose=False, log=False): r"""Compute the entropic regularized wasserstein barycenter of distributions A with stabilization. @@ -1264,7 +1270,9 @@ def _barycenter_stabilized(A, M, reg, tau=1e10, weights=None, numItermax=1000, return q -def convolutional_barycenter2d(A, reg, weights=None, numItermax=10000, stopThr=1e-9, stabThr=1e-30, verbose=False, log=False): +def convolutional_barycenter2d(A, reg, weights=None, numItermax=10000, + stopThr=1e-9, stabThr=1e-30, verbose=False, + log=False): r"""Compute the entropic regularized wasserstein barycenter of distributions A where A is a collection of 2D images. @@ -1480,7 +1488,9 @@ def unmix(a, D, M, M0, h0, reg, reg0, alpha, numItermax=1000, return np.sum(K0, axis=1) -def empirical_sinkhorn(X_s, X_t, reg, a=None, b=None, metric='sqeuclidean', numIterMax=10000, stopThr=1e-9, verbose=False, log=False, **kwargs): +def empirical_sinkhorn(X_s, X_t, reg, a=None, b=None, metric='sqeuclidean', + numIterMax=10000, stopThr=1e-9, verbose=False, + log=False, **kwargs): r''' Solve the entropic regularization optimal transport problem and return the OT matrix from empirical data @@ -1497,22 +1507,22 @@ def empirical_sinkhorn(X_s, X_t, reg, a=None, b=None, metric='sqeuclidean', numI \gamma\geq 0 where : - - :math:`M` is the (dim_a, n_b) metric cost matrix + - :math:`M` is the (n_samples_a, n_samples_b) metric cost matrix - :math:`\Omega` is the entropic regularization term :math:`\Omega(\gamma)=\sum_{i,j} \gamma_{i,j}\log(\gamma_{i,j})` - :math:`a` and :math:`b` are source and target weights (sum to 1) Parameters ---------- - X_s : ndarray, shape (dim_a, d) + X_s : ndarray, shape (n_samples_a, dim) samples in the source domain - X_t : ndarray, shape (dim_b, d) + X_t : ndarray, shape (n_samples_b, dim) samples in the target domain reg : float Regularization term >0 - a : ndarray, shape (dim_a,) + a : ndarray, shape (n_samples_a,) samples weights in the source domain - b : ndarray, shape (dim_b,) + b : ndarray, shape (n_samples_b,) samples weights in the target domain numItermax : int, optional Max number of iterations @@ -1526,7 +1536,7 @@ def empirical_sinkhorn(X_s, X_t, reg, a=None, b=None, metric='sqeuclidean', numI Returns ------- - gamma : ndarray, shape (dim_a, n_b) + gamma : ndarray, shape (n_samples_a, n_samples_b) Regularized optimal transportation matrix for the given parameters log : dict log dictionary return only if log==True in parameters @@ -1587,7 +1597,7 @@ def empirical_sinkhorn2(X_s, X_t, reg, a=None, b=None, metric='sqeuclidean', num \gamma\geq 0 where : - - :math:`M` is the (dim_a, n_b) metric cost matrix + - :math:`M` is the (n_samples_a, n_samples_b) metric cost matrix - :math:`\Omega` is the entropic regularization term :math:`\Omega(\gamma)=\sum_{i,j} \gamma_{i,j}\log(\gamma_{i,j})` - :math:`a` and :math:`b` are source and target weights (sum to 1) @@ -1596,7 +1606,7 @@ def empirical_sinkhorn2(X_s, X_t, reg, a=None, b=None, metric='sqeuclidean', num ---------- X_s : ndarray, shape (n_samples_a, dim) samples in the source domain - X_t : ndarray, shape (n_samples_b, d) + X_t : ndarray, shape (n_samples_b, dim) samples in the target domain reg : float Regularization term >0 @@ -1695,7 +1705,7 @@ def empirical_sinkhorn_divergence(X_s, X_t, reg, a=None, b=None, metric='sqeucli \gamma_b\geq 0 where : - - :math:`M` (resp. :math:`M_a, M_b`) is the (dim_a, n_b) metric cost matrix (resp (dim_a, ns) and (dim_b, nt)) + - :math:`M` (resp. :math:`M_a, M_b`) is the (n_samples_a, n_samples_b) metric cost matrix (resp (n_samples_a, n_samples_a) and (n_samples_b, n_samples_b)) - :math:`\Omega` is the entropic regularization term :math:`\Omega(\gamma)=\sum_{i,j} \gamma_{i,j}\log(\gamma_{i,j})` - :math:`a` and :math:`b` are source and target weights (sum to 1) -- cgit v1.2.3 From e0c935a865a57bc4603144b27f1b58cbfba87760 Mon Sep 17 00:00:00 2001 From: Hicham Janati Date: Wed, 4 Sep 2019 10:28:04 +0200 Subject: improve doc --- docs/source/quickstart.rst | 10 ++++------ ot/__init__.py | 3 ++- ot/bregman.py | 40 ++++++++++++++++++++++------------------ 3 files changed, 28 insertions(+), 25 deletions(-) (limited to 'ot/bregman.py') diff --git a/docs/source/quickstart.rst b/docs/source/quickstart.rst index 9729664..978eaff 100644 --- a/docs/source/quickstart.rst +++ b/docs/source/quickstart.rst @@ -577,10 +577,10 @@ Unbalanced optimal transport Unbalanced OT is a relaxation of the entropy regularized OT problem where the violation of the constraint on the marginals is added to the objective of the optimization -problem. The unbalanced OT metric between two histograms a and b is defined as [25]_ [10]_: +problem. The unbalanced OT metric between two unbalanced histograms a and b is defined as [25]_ [10]_: .. math:: - W_u(a, b) = \min_\gamma \quad \sum_{i,j}\gamma_{i,j}M_{i,j} + reg\cdot\Omega(\gamma) + \alpha KL(\gamma 1, a) + \alpha KL(\gamma^T 1, b) + W_u(a, b) = \min_\gamma \quad \sum_{i,j}\gamma_{i,j}M_{i,j} + reg\cdot\Omega(\gamma) + reg_m KL(\gamma 1, a) + reg_m KL(\gamma^T 1, b) s.t. \quad \gamma\geq 0 @@ -593,13 +593,11 @@ in :any:`ot.unbalanced`. Computing the optimal transport plan or the transport cost is similar to the balanced case. The Sinkhorn-Knopp algorithm is implemented in :any:`ot.sinkhorn_unbalanced` and :any:`ot.sinkhorn_unbalanced2` that return respectively the OT matrix and the value of the -linear term. Note that the regularization parameter :math:`\alpha` in the -equation above is given to those functions with the parameter :code:`reg_m`. - +linear term. .. note:: The main function to solve entropic regularized UOT is :any:`ot.sinkhorn_unbalanced`. - This function is a wrapper and the parameter :code:`method` help you select + This function is a wrapper and the parameter :code:`method` helps you select the actual algorithm used to solve the problem: + :code:`method='sinkhorn'` calls :any:`ot.unbalanced.sinkhorn_knopp_unbalanced` diff --git a/ot/__init__.py b/ot/__init__.py index 99f288e..df0ef8a 100644 --- a/ot/__init__.py +++ b/ot/__init__.py @@ -77,4 +77,5 @@ __all__ = ["emd", "emd2", 'emd_1d', "sinkhorn", "sinkhorn2", "utils", 'datasets' 'bregman', 'lp', 'tic', 'toc', 'toq', 'gromov', 'emd_1d', 'emd2_1d', 'wasserstein_1d', 'dist', 'unif', 'barycenter', 'sinkhorn_lpl1_mm', 'da', 'optim', - 'sinkhorn_unbalanced', "barycenter_unbalanced"] + 'sinkhorn_unbalanced', 'barycenter_unbalanced', + 'sinkhorn_unbalanced2'] diff --git a/ot/bregman.py b/ot/bregman.py index 02aeb6d..2cd832b 100644 --- a/ot/bregman.py +++ b/ot/bregman.py @@ -35,7 +35,7 @@ def sinkhorn(a, b, M, reg, method='sinkhorn', numItermax=1000, - M is the (dim_a, dim_b) metric cost matrix - :math:`\Omega` is the entropic regularization term :math:`\Omega(\gamma)=\sum_{i,j} \gamma_{i,j}\log(\gamma_{i,j})` - - a and b are source and target weights (sum to 1) + - a and b are source and target weights (histograms, both sum to 1) The algorithm used for solving the problem is the Sinkhorn-Knopp matrix scaling algorithm as proposed in [2]_ @@ -143,7 +143,7 @@ def sinkhorn2(a, b, M, reg, method='sinkhorn', numItermax=1000, - M is the (dim_a, dim_b) metric cost matrix - :math:`\Omega` is the entropic regularization term :math:`\Omega(\gamma)=\sum_{i,j} \gamma_{i,j}\log(\gamma_{i,j})` - - a and b are source and target weights (sum to 1) + - a and b are source and target weights (histograms, both sum to 1) The algorithm used for solving the problem is the Sinkhorn-Knopp matrix scaling algorithm as proposed in [2]_ @@ -251,7 +251,7 @@ def sinkhorn_knopp(a, b, M, reg, numItermax=1000, - M is the (dim_a, dim_b) metric cost matrix - :math:`\Omega` is the entropic regularization term :math:`\Omega(\gamma)=\sum_{i,j} \gamma_{i,j}\log(\gamma_{i,j})` - - a and b are source and target weights (sum to 1) + - a and b are source and target weights (histograms, both sum to 1) The algorithm used for solving the problem is the Sinkhorn-Knopp matrix scaling algorithm as proposed in [2]_ @@ -432,7 +432,7 @@ def greenkhorn(a, b, M, reg, numItermax=10000, stopThr=1e-9, verbose=False, - M is the (dim_a, dim_b) metric cost matrix - :math:`\Omega` is the entropic regularization term :math:`\Omega(\gamma)=\sum_{i,j} \gamma_{i,j}\log(\gamma_{i,j})` - - a and b are source and target weights (sum to 1) + - a and b are source and target weights (histograms, both sum to 1) @@ -578,7 +578,8 @@ def sinkhorn_stabilized(a, b, M, reg, numItermax=1000, tau=1e3, stopThr=1e-9, - M is the (dim_a, dim_b) metric cost matrix - :math:`\Omega` is the entropic regularization term :math:`\Omega(\gamma)=\sum_{i,j} \gamma_{i,j}\log(\gamma_{i,j})` - - a and b are source and target weights (sum to 1) + - a and b are source and target weights (histograms, both sum to 1) + The algorithm used for solving the problem is the Sinkhorn-Knopp matrix scaling algorithm as proposed in [2]_ but with the log stabilization @@ -808,7 +809,8 @@ def sinkhorn_epsilon_scaling(a, b, M, reg, numItermax=100, epsilon0=1e4, - M is the (dim_a, dim_b) metric cost matrix - :math:`\Omega` is the entropic regularization term :math:`\Omega(\gamma)=\sum_{i,j} \gamma_{i,j}\log(\gamma_{i,j})` - - a and b are source and target weights (sum to 1) + - a and b are source and target weights (histograms, both sum to 1) + The algorithm used for solving the problem is the Sinkhorn-Knopp matrix scaling algorithm as proposed in [2]_ but with the log stabilization @@ -1229,7 +1231,6 @@ def barycenter_stabilized(A, M, reg, tau=1e10, weights=None, numItermax=1000, absorbing = False if (u > tau).any() or (v > tau).any(): absorbing = True - print("YEAH absorbing") alpha = alpha + reg * np.log(np.max(u, 1)) beta = beta + reg * np.log(np.max(v, 1)) K = np.exp((alpha[:, None] + beta[None, :] - @@ -1394,9 +1395,12 @@ def unmix(a, D, M, M0, h0, reg, reg0, alpha, numItermax=1000, where : - :math:`W_{M,reg}(\cdot,\cdot)` is the entropic regularized Wasserstein distance with M loss matrix (see ot.bregman.sinkhorn) - - :math:`\mathbf{a}` is an observed distribution, :math:`\mathbf{h}_0` is aprior on unmixing - - reg and :math:`\mathbf{M}` are respectively the regularization term and the cost matrix for OT data fitting - - reg0 and :math:`\mathbf{M0}` are respectively the regularization term and the cost matrix for regularization + - :math: `\mathbf{D}` is a dictionary of `n_atoms` atoms of dimension `dim_a`, its expected shape is `(dim_a, n_atoms)` + - :math:`\mathbf{h}` is the estimated unmixing of dimension `n_atoms` + - :math:`\mathbf{a}` is an observed distribution of dimension `dim_a` + - :math:`\mathbf{h}_0` is a prior on `h` of dimension `dim_prior` + - reg and :math:`\mathbf{M}` are respectively the regularization term and the cost matrix (dim_a, dim_a) for OT data fitting + - reg0 and :math:`\mathbf{M0}` are respectively the regularization term and the cost matrix (dim_prior, n_atoms) regularization - :math:`\\alpha`weight data fitting and regularization The optimization problem is solved suing the algorithm described in [4] @@ -1404,16 +1408,16 @@ def unmix(a, D, M, M0, h0, reg, reg0, alpha, numItermax=1000, Parameters ---------- - a : ndarray, shape (n_observed) - observed distribution - D : ndarray, shape (dim, dim) + a : ndarray, shape (dim_a) + observed distribution (histogram, sums to 1) + D : ndarray, shape (dim_a, n_atoms) dictionary matrix - M : ndarray, shape (dim, dim) + M : ndarray, shape (dim_a, dim_a) loss matrix - M0 : ndarray, shape (n_observed, n_observed) + M0 : ndarray, shape (n_atoms, dim_prior) loss matrix - h0 : ndarray, shape (dim,) - prior on h + h0 : ndarray, shape (n_atoms,) + prior on the estimated unmixing h reg : float Regularization term >0 (Wasserstein data fitting) reg0 : float @@ -1432,7 +1436,7 @@ def unmix(a, D, M, M0, h0, reg, reg0, alpha, numItermax=1000, Returns ------- - a : ndarray, shape (dim,) + h : ndarray, shape (n_atoms,) Wasserstein barycenter log : dict log dictionary return only if log==True in parameters -- cgit v1.2.3