# -*- coding: utf-8 -*- """ Multi-lib backend for POT The goal is to write backend-agnostic code. Whether you're using Numpy, PyTorch, Jax, Cupy, or Tensorflow, POT code should work nonetheless. To achieve that, POT provides backend classes which implements functions in their respective backend imitating Numpy API. As a convention, we use nx instead of np to refer to the backend. Examples -------- >>> from ot.utils import list_to_array >>> from ot.backend import get_backend >>> def f(a, b): # the function does not know which backend to use ... a, b = list_to_array(a, b) # if a list in given, make it an array ... nx = get_backend(a, b) # infer the backend from the arguments ... c = nx.dot(a, b) # now use the backend to do any calculation ... return c .. warning:: Tensorflow only works with the Numpy API. To activate it, please run the following: .. code-block:: from tensorflow.python.ops.numpy_ops import np_config np_config.enable_numpy_behavior() Performance -------- - CPU: Intel(R) Xeon(R) Gold 6248 CPU @ 2.50GHz - GPU: Tesla V100-SXM2-32GB - Date of the benchmark: December 8th, 2021 - Commit of benchmark: PR #316, https://github.com/PythonOT/POT/pull/316 .. raw:: html
Sinkhorn Knopp - Averaged on 100 runs
Bitsize32 bits
DeviceCPUGPU
Sample sizeNumpyPytorchTensorflowCupyJaxPytorchTensorflow
500.00080.00220.01510.00950.01930.00510.0293
1000.00050.00130.00970.00570.01150.00290.0173
5000.00090.00160.01100.00580.01150.00290.0166
10000.00210.00210.01450.00560.01180.00290.0168
20000.00690.00430.02780.00590.01180.00300.0165
50000.07070.03140.13950.00740.01250.00350.0198
 
Bitsize64 bits
DeviceCPUGPU
Sample sizeNumpyPytorchTensorflowCupyJaxPytorchTensorflow
500.00080.00200.01540.00930.01910.00510.0328
1000.00050.00130.00940.00560.01140.00290.0169
5000.00130.00170.01200.00590.01160.00290.0168
10000.00340.00270.01770.00580.01180.00290.0167
20000.01460.00750.04360.00590.01200.00290.0165
50000.14670.05680.24680.00770.01460.00450.0204
""" # Author: Remi Flamary # Nicolas Courty # # License: MIT License import numpy as np import scipy import scipy.linalg import scipy.special as special from scipy.sparse import issparse, coo_matrix, csr_matrix import warnings import time try: import torch torch_type = torch.Tensor except ImportError: torch = False torch_type = float try: import jax import jax.numpy as jnp import jax.scipy.special as jspecial from jax.lib import xla_bridge jax_type = jax.numpy.ndarray except ImportError: jax = False jax_type = float try: import cupy as cp import cupyx cp_type = cp.ndarray except ImportError: cp = False cp_type = float try: import tensorflow as tf import tensorflow.experimental.numpy as tnp tf_type = tf.Tensor except ImportError: tf = False tf_type = float str_type_error = "All array should be from the same type/backend. Current types are : {}" def get_backend_list(): """Returns the list of available backends""" lst = [NumpyBackend(), ] if torch: lst.append(TorchBackend()) if jax: lst.append(JaxBackend()) if cp: # pragma: no cover lst.append(CupyBackend()) if tf: lst.append(TensorflowBackend()) return lst def get_backend(*args): """Returns the proper backend for a list of input arrays Also raises TypeError if all arrays are not from the same backend """ # check that some arrays given if not len(args) > 0: raise ValueError(" The function takes at least one parameter") # check all same type if not len(set(type(a) for a in args)) == 1: raise ValueError(str_type_error.format([type(a) for a in args])) if isinstance(args[0], np.ndarray): return NumpyBackend() elif isinstance(args[0], torch_type): return TorchBackend() elif isinstance(args[0], jax_type): return JaxBackend() elif isinstance(args[0], cp_type): # pragma: no cover return CupyBackend() elif isinstance(args[0], tf_type): return TensorflowBackend() else: raise ValueError("Unknown type of non implemented backend.") def to_numpy(*args): """Returns numpy arrays from any compatible backend""" if len(args) == 1: return get_backend(args[0]).to_numpy(args[0]) else: return [get_backend(a).to_numpy(a) for a in args] class Backend(): """ Backend abstract class. Implementations: :py:class:`JaxBackend`, :py:class:`NumpyBackend`, :py:class:`TorchBackend`, :py:class:`CupyBackend`, :py:class:`TensorflowBackend` - The `__name__` class attribute refers to the name of the backend. - The `__type__` class attribute refers to the data structure used by the backend. """ __name__ = None __type__ = None __type_list__ = None rng_ = None def __str__(self): return self.__name__ # convert batch of tensors to numpy def to_numpy(self, *arrays): """Returns the numpy version of tensors""" if len(arrays) == 1: return self._to_numpy(arrays[0]) else: return [self._to_numpy(array) for array in arrays] # convert a tensor to numpy def _to_numpy(self, a): """Returns the numpy version of a tensor""" raise NotImplementedError() # convert batch of arrays from numpy def from_numpy(self, *arrays, type_as=None): """Creates tensors cloning a numpy array, with the given precision (defaulting to input's precision) and the given device (in case of GPUs)""" if len(arrays) == 1: return self._from_numpy(arrays[0], type_as=type_as) else: return [self._from_numpy(array, type_as=type_as) for array in arrays] # convert an array from numpy def _from_numpy(self, a, type_as=None): """Creates a tensor cloning a numpy array, with the given precision (defaulting to input's precision) and the given device (in case of GPUs)""" raise NotImplementedError() def set_gradients(self, val, inputs, grads): """Define the gradients for the value val wrt the inputs """ raise NotImplementedError() def zeros(self, shape, type_as=None): r""" Creates a tensor full of zeros. This function follows the api from :any:`numpy.zeros` See: https://numpy.org/doc/stable/reference/generated/numpy.zeros.html """ raise NotImplementedError() def ones(self, shape, type_as=None): r""" Creates a tensor full of ones. This function follows the api from :any:`numpy.ones` See: https://numpy.org/doc/stable/reference/generated/numpy.ones.html """ raise NotImplementedError() def arange(self, stop, start=0, step=1, type_as=None): r""" Returns evenly spaced values within a given interval. This function follows the api from :any:`numpy.arange` See: https://numpy.org/doc/stable/reference/generated/numpy.arange.html """ raise NotImplementedError() def full(self, shape, fill_value, type_as=None): r""" Creates a tensor with given shape, filled with given value. This function follows the api from :any:`numpy.full` See: https://numpy.org/doc/stable/reference/generated/numpy.full.html """ raise NotImplementedError() def eye(self, N, M=None, type_as=None): r""" Creates the identity matrix of given size. This function follows the api from :any:`numpy.eye` See: https://numpy.org/doc/stable/reference/generated/numpy.eye.html """ raise NotImplementedError() def sum(self, a, axis=None, keepdims=False): r""" Sums tensor elements over given dimensions. This function follows the api from :any:`numpy.sum` See: https://numpy.org/doc/stable/reference/generated/numpy.sum.html """ raise NotImplementedError() def cumsum(self, a, axis=None): r""" Returns the cumulative sum of tensor elements over given dimensions. This function follows the api from :any:`numpy.cumsum` See: https://numpy.org/doc/stable/reference/generated/numpy.cumsum.html """ raise NotImplementedError() def max(self, a, axis=None, keepdims=False): r""" Returns the maximum of an array or maximum along given dimensions. This function follows the api from :any:`numpy.amax` See: https://numpy.org/doc/stable/reference/generated/numpy.amax.html """ raise NotImplementedError() def min(self, a, axis=None, keepdims=False): r""" Returns the maximum of an array or maximum along given dimensions. This function follows the api from :any:`numpy.amin` See: https://numpy.org/doc/stable/reference/generated/numpy.amin.html """ raise NotImplementedError() def maximum(self, a, b): r""" Returns element-wise maximum of array elements. This function follows the api from :any:`numpy.maximum` See: https://numpy.org/doc/stable/reference/generated/numpy.maximum.html """ raise NotImplementedError() def minimum(self, a, b): r""" Returns element-wise minimum of array elements. This function follows the api from :any:`numpy.minimum` See: https://numpy.org/doc/stable/reference/generated/numpy.minimum.html """ raise NotImplementedError() def dot(self, a, b): r""" Returns the dot product of two tensors. This function follows the api from :any:`numpy.dot` See: https://numpy.org/doc/stable/reference/generated/numpy.dot.html """ raise NotImplementedError() def abs(self, a): r""" Computes the absolute value element-wise. This function follows the api from :any:`numpy.absolute` See: https://numpy.org/doc/stable/reference/generated/numpy.absolute.html """ raise NotImplementedError() def exp(self, a): r""" Computes the exponential value element-wise. This function follows the api from :any:`numpy.exp` See: https://numpy.org/doc/stable/reference/generated/numpy.exp.html """ raise NotImplementedError() def log(self, a): r""" Computes the natural logarithm, element-wise. This function follows the api from :any:`numpy.log` See: https://numpy.org/doc/stable/reference/generated/numpy.log.html """ raise NotImplementedError() def sqrt(self, a): r""" Returns the non-ngeative square root of a tensor, element-wise. This function follows the api from :any:`numpy.sqrt` See: https://numpy.org/doc/stable/reference/generated/numpy.sqrt.html """ raise NotImplementedError() def power(self, a, exponents): r""" First tensor elements raised to powers from second tensor, element-wise. This function follows the api from :any:`numpy.power` See: https://numpy.org/doc/stable/reference/generated/numpy.power.html """ raise NotImplementedError() def norm(self, a): r""" Computes the matrix frobenius norm. This function follows the api from :any:`numpy.linalg.norm` See: https://numpy.org/doc/stable/reference/generated/numpy.linalg.norm.html """ raise NotImplementedError() def any(self, a): r""" Tests whether any tensor element along given dimensions evaluates to True. This function follows the api from :any:`numpy.any` See: https://numpy.org/doc/stable/reference/generated/numpy.any.html """ raise NotImplementedError() def isnan(self, a): r""" Tests element-wise for NaN and returns result as a boolean tensor. This function follows the api from :any:`numpy.isnan` See: https://numpy.org/doc/stable/reference/generated/numpy.isnan.html """ raise NotImplementedError() def isinf(self, a): r""" Tests element-wise for positive or negative infinity and returns result as a boolean tensor. This function follows the api from :any:`numpy.isinf` See: https://numpy.org/doc/stable/reference/generated/numpy.isinf.html """ raise NotImplementedError() def einsum(self, subscripts, *operands): r""" Evaluates the Einstein summation convention on the operands. This function follows the api from :any:`numpy.einsum` See: https://numpy.org/doc/stable/reference/generated/numpy.einsum.html """ raise NotImplementedError() def sort(self, a, axis=-1): r""" Returns a sorted copy of a tensor. This function follows the api from :any:`numpy.sort` See: https://numpy.org/doc/stable/reference/generated/numpy.sort.html """ raise NotImplementedError() def argsort(self, a, axis=None): r""" Returns the indices that would sort a tensor. This function follows the api from :any:`numpy.argsort` See: https://numpy.org/doc/stable/reference/generated/numpy.argsort.html """ raise NotImplementedError() def searchsorted(self, a, v, side='left'): r""" Finds indices where elements should be inserted to maintain order in given tensor. This function follows the api from :any:`numpy.searchsorted` See: https://numpy.org/doc/stable/reference/generated/numpy.searchsorted.html """ raise NotImplementedError() def flip(self, a, axis=None): r""" Reverses the order of elements in a tensor along given dimensions. This function follows the api from :any:`numpy.flip` See: https://numpy.org/doc/stable/reference/generated/numpy.flip.html """ raise NotImplementedError() def clip(self, a, a_min, a_max): """ Limits the values in a tensor. This function follows the api from :any:`numpy.clip` See: https://numpy.org/doc/stable/reference/generated/numpy.clip.html """ raise NotImplementedError() def repeat(self, a, repeats, axis=None): r""" Repeats elements of a tensor. This function follows the api from :any:`numpy.repeat` See: https://numpy.org/doc/stable/reference/generated/numpy.repeat.html """ raise NotImplementedError() def take_along_axis(self, arr, indices, axis): r""" Gathers elements of a tensor along given dimensions. This function follows the api from :any:`numpy.take_along_axis` See: https://numpy.org/doc/stable/reference/generated/numpy.take_along_axis.html """ raise NotImplementedError() def concatenate(self, arrays, axis=0): r""" Joins a sequence of tensors along an existing dimension. This function follows the api from :any:`numpy.concatenate` See: https://numpy.org/doc/stable/reference/generated/numpy.concatenate.html """ raise NotImplementedError() def zero_pad(self, a, pad_width, value=0): r""" Pads a tensor with a given value (0 by default). This function follows the api from :any:`numpy.pad` See: https://numpy.org/doc/stable/reference/generated/numpy.pad.html """ raise NotImplementedError() def argmax(self, a, axis=None): r""" Returns the indices of the maximum values of a tensor along given dimensions. This function follows the api from :any:`numpy.argmax` See: https://numpy.org/doc/stable/reference/generated/numpy.argmax.html """ raise NotImplementedError() def argmin(self, a, axis=None): r""" Returns the indices of the minimum values of a tensor along given dimensions. This function follows the api from :any:`numpy.argmin` See: https://numpy.org/doc/stable/reference/generated/numpy.argmin.html """ raise NotImplementedError() def mean(self, a, axis=None): r""" Computes the arithmetic mean of a tensor along given dimensions. This function follows the api from :any:`numpy.mean` See: https://numpy.org/doc/stable/reference/generated/numpy.mean.html """ raise NotImplementedError() def std(self, a, axis=None): r""" Computes the standard deviation of a tensor along given dimensions. This function follows the api from :any:`numpy.std` See: https://numpy.org/doc/stable/reference/generated/numpy.std.html """ raise NotImplementedError() def linspace(self, start, stop, num): r""" Returns a specified number of evenly spaced values over a given interval. This function follows the api from :any:`numpy.linspace` See: https://numpy.org/doc/stable/reference/generated/numpy.linspace.html """ raise NotImplementedError() def meshgrid(self, a, b): r""" Returns coordinate matrices from coordinate vectors (Numpy convention). This function follows the api from :any:`numpy.meshgrid` See: https://numpy.org/doc/stable/reference/generated/numpy.meshgrid.html """ raise NotImplementedError() def diag(self, a, k=0): r""" Extracts or constructs a diagonal tensor. This function follows the api from :any:`numpy.diag` See: https://numpy.org/doc/stable/reference/generated/numpy.diag.html """ raise NotImplementedError() def unique(self, a): r""" Finds unique elements of given tensor. This function follows the api from :any:`numpy.unique` See: https://numpy.org/doc/stable/reference/generated/numpy.unique.html """ raise NotImplementedError() def logsumexp(self, a, axis=None): r""" Computes the log of the sum of exponentials of input elements. This function follows the api from :any:`scipy.special.logsumexp` See: https://docs.scipy.org/doc/scipy/reference/generated/scipy.special.logsumexp.html """ raise NotImplementedError() def stack(self, arrays, axis=0): r""" Joins a sequence of tensors along a new dimension. This function follows the api from :any:`numpy.stack` See: https://numpy.org/doc/stable/reference/generated/numpy.stack.html """ raise NotImplementedError() def outer(self, a, b): r""" Computes the outer product between two vectors. This function follows the api from :any:`numpy.outer` See: https://numpy.org/doc/stable/reference/generated/numpy.outer.html """ raise NotImplementedError() def reshape(self, a, shape): r""" Gives a new shape to a tensor without changing its data. This function follows the api from :any:`numpy.reshape` See: https://numpy.org/doc/stable/reference/generated/numpy.reshape.html """ raise NotImplementedError() def seed(self, seed=None): r""" Sets the seed for the random generator. This function follows the api from :any:`numpy.random.seed` See: https://numpy.org/doc/stable/reference/generated/numpy.random.seed.html """ raise NotImplementedError() def rand(self, *size, type_as=None): r""" Generate uniform random numbers. This function follows the api from :any:`numpy.random.rand` See: https://numpy.org/doc/stable/reference/generated/numpy.random.rand.html """ raise NotImplementedError() def randn(self, *size, type_as=None): r""" Generate normal Gaussian random numbers. This function follows the api from :any:`numpy.random.rand` See: https://numpy.org/doc/stable/reference/generated/numpy.random.rand.html """ raise NotImplementedError() def coo_matrix(self, data, rows, cols, shape=None, type_as=None): r""" Creates a sparse tensor in COOrdinate format. This function follows the api from :any:`scipy.sparse.coo_matrix` See: https://docs.scipy.org/doc/scipy/reference/generated/scipy.sparse.coo_matrix.html """ raise NotImplementedError() def issparse(self, a): r""" Checks whether or not the input tensor is a sparse tensor. This function follows the api from :any:`scipy.sparse.issparse` See: https://docs.scipy.org/doc/scipy/reference/generated/scipy.sparse.issparse.html """ raise NotImplementedError() def tocsr(self, a): r""" Converts this matrix to Compressed Sparse Row format. This function follows the api from :any:`scipy.sparse.coo_matrix.tocsr` See: https://docs.scipy.org/doc/scipy/reference/generated/scipy.sparse.coo_matrix.tocsr.html """ raise NotImplementedError() def eliminate_zeros(self, a, threshold=0.): r""" Removes entries smaller than the given threshold from the sparse tensor. This function follows the api from :any:`scipy.sparse.csr_matrix.eliminate_zeros` See: https://docs.scipy.org/doc/scipy-0.14.0/reference/generated/scipy.sparse.csr_matrix.eliminate_zeros.html """ raise NotImplementedError() def todense(self, a): r""" Converts a sparse tensor to a dense tensor. This function follows the api from :any:`scipy.sparse.csr_matrix.toarray` See: https://docs.scipy.org/doc/scipy/reference/generated/scipy.sparse.csr_matrix.toarray.html """ raise NotImplementedError() def where(self, condition, x, y): r""" Returns elements chosen from x or y depending on condition. This function follows the api from :any:`numpy.where` See: https://numpy.org/doc/stable/reference/generated/numpy.where.html """ raise NotImplementedError() def copy(self, a): r""" Returns a copy of the given tensor. This function follows the api from :any:`numpy.copy` See: https://numpy.org/doc/stable/reference/generated/numpy.copy.html """ raise NotImplementedError() def allclose(self, a, b, rtol=1e-05, atol=1e-08, equal_nan=False): r""" Returns True if two arrays are element-wise equal within a tolerance. This function follows the api from :any:`numpy.allclose` See: https://numpy.org/doc/stable/reference/generated/numpy.allclose.html """ raise NotImplementedError() def dtype_device(self, a): r""" Returns the dtype and the device of the given tensor. """ raise NotImplementedError() def assert_same_dtype_device(self, a, b): r""" Checks whether or not the two given inputs have the same dtype as well as the same device """ raise NotImplementedError() def squeeze(self, a, axis=None): r""" Remove axes of length one from a. This function follows the api from :any:`numpy.squeeze`. See: https://numpy.org/doc/stable/reference/generated/numpy.squeeze.html """ raise NotImplementedError() def bitsize(self, type_as): r""" Gives the number of bits used by the data type of the given tensor. """ raise NotImplementedError() def device_type(self, type_as): r""" Returns CPU or GPU depending on the device where the given tensor is located. """ raise NotImplementedError() def _bench(self, callable, *args, n_runs=1, warmup_runs=1): r""" Executes a benchmark of the given callable with the given arguments. """ raise NotImplementedError() def solve(self, a, b): r""" Solves a linear matrix equation, or system of linear scalar equations. This function follows the api from :any:`numpy.linalg.solve`. See: https://numpy.org/doc/stable/reference/generated/numpy.linalg.solve.html """ raise NotImplementedError() def trace(self, a): r""" Returns the sum along diagonals of the array. This function follows the api from :any:`numpy.trace`. See: https://numpy.org/doc/stable/reference/generated/numpy.trace.html """ raise NotImplementedError() def inv(self, a): r""" Computes the inverse of a matrix. This function follows the api from :any:`scipy.linalg.inv`. See: https://docs.scipy.org/doc/scipy/reference/generated/scipy.linalg.inv.html """ raise NotImplementedError() def sqrtm(self, a): r""" Computes the matrix square root. Requires input to be definite positive. This function follows the api from :any:`scipy.linalg.sqrtm`. See: https://docs.scipy.org/doc/scipy/reference/generated/scipy.linalg.sqrtm.html """ raise NotImplementedError() def kl_div(self, p, q, eps=1e-16): r""" Computes the Kullback-Leibler divergence. This function follows the api from :any:`scipy.stats.entropy`. Parameter eps is used to avoid numerical errors and is added in the log. .. math:: KL(p,q) = \sum_i p(i) \log (\frac{p(i)}{q(i)}+\epsilon) See: https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.entropy.html """ raise NotImplementedError() def isfinite(self, a): r""" Tests element-wise for finiteness (not infinity and not Not a Number). This function follows the api from :any:`numpy.isfinite`. See: https://numpy.org/doc/stable/reference/generated/numpy.isfinite.html """ raise NotImplementedError() def array_equal(self, a, b): r""" True if two arrays have the same shape and elements, False otherwise. This function follows the api from :any:`numpy.array_equal`. See: https://numpy.org/doc/stable/reference/generated/numpy.array_equal.html """ raise NotImplementedError() def is_floating_point(self, a): r""" Returns whether or not the input consists of floats """ raise NotImplementedError() def tile(self, a, reps): r""" Construct an array by repeating a the number of times given by reps See: https://numpy.org/doc/stable/reference/generated/numpy.tile.html """ raise NotImplementedError() def floor(self, a): r""" Return the floor of the input element-wise See: https://numpy.org/doc/stable/reference/generated/numpy.floor.html """ raise NotImplementedError() def prod(self, a, axis=None): r""" Return the product of all elements. See: https://numpy.org/doc/stable/reference/generated/numpy.prod.html """ raise NotImplementedError() def sort2(self, a, axis=None): r""" Return the sorted array and the indices to sort the array See: https://pytorch.org/docs/stable/generated/torch.sort.html """ raise NotImplementedError() def qr(self, a): r""" Return the QR factorization See: https://numpy.org/doc/stable/reference/generated/numpy.linalg.qr.html """ raise NotImplementedError() def atan2(self, a, b): r""" Element wise arctangent See: https://numpy.org/doc/stable/reference/generated/numpy.arctan2.html """ raise NotImplementedError() def transpose(self, a, axes=None): r""" Returns a tensor that is a transposed version of a. The given dimensions dim0 and dim1 are swapped. See: https://numpy.org/doc/stable/reference/generated/numpy.transpose.html """ raise NotImplementedError() class NumpyBackend(Backend): """ NumPy implementation of the backend - `__name__` is "numpy" - `__type__` is np.ndarray """ __name__ = 'numpy' __type__ = np.ndarray __type_list__ = [np.array(1, dtype=np.float32), np.array(1, dtype=np.float64)] rng_ = np.random.RandomState() def _to_numpy(self, a): return a def _from_numpy(self, a, type_as=None): if type_as is None: return a elif isinstance(a, float): return a else: return a.astype(type_as.dtype) def set_gradients(self, val, inputs, grads): # No gradients for numpy return val def zeros(self, shape, type_as=None): if type_as is None: return np.zeros(shape) else: return np.zeros(shape, dtype=type_as.dtype) def ones(self, shape, type_as=None): if type_as is None: return np.ones(shape) else: return np.ones(shape, dtype=type_as.dtype) def arange(self, stop, start=0, step=1, type_as=None): return np.arange(start, stop, step) def full(self, shape, fill_value, type_as=None): if type_as is None: return np.full(shape, fill_value) else: return np.full(shape, fill_value, dtype=type_as.dtype) def eye(self, N, M=None, type_as=None): if type_as is None: return np.eye(N, M) else: return np.eye(N, M, dtype=type_as.dtype) def sum(self, a, axis=None, keepdims=False): return np.sum(a, axis, keepdims=keepdims) def cumsum(self, a, axis=None): return np.cumsum(a, axis) def max(self, a, axis=None, keepdims=False): return np.max(a, axis, keepdims=keepdims) def min(self, a, axis=None, keepdims=False): return np.min(a, axis, keepdims=keepdims) def maximum(self, a, b): return np.maximum(a, b) def minimum(self, a, b): return np.minimum(a, b) def dot(self, a, b): return np.dot(a, b) def abs(self, a): return np.abs(a) def exp(self, a): return np.exp(a) def log(self, a): return np.log(a) def sqrt(self, a): return np.sqrt(a) def power(self, a, exponents): return np.power(a, exponents) def norm(self, a): return np.sqrt(np.sum(np.square(a))) def any(self, a): return np.any(a) def isnan(self, a): return np.isnan(a) def isinf(self, a): return np.isinf(a) def einsum(self, subscripts, *operands): return np.einsum(subscripts, *operands) def sort(self, a, axis=-1): return np.sort(a, axis) def argsort(self, a, axis=-1): return np.argsort(a, axis) def searchsorted(self, a, v, side='left'): if a.ndim == 1: return np.searchsorted(a, v, side) else: # this is a not very efficient way to make numpy # searchsorted work on 2d arrays ret = np.empty(v.shape, dtype=int) for i in range(a.shape[0]): ret[i, :] = np.searchsorted(a[i, :], v[i, :], side) return ret def flip(self, a, axis=None): return np.flip(a, axis) def outer(self, a, b): return np.outer(a, b) def clip(self, a, a_min, a_max): return np.clip(a, a_min, a_max) def repeat(self, a, repeats, axis=None): return np.repeat(a, repeats, axis) def take_along_axis(self, arr, indices, axis): return np.take_along_axis(arr, indices, axis) def concatenate(self, arrays, axis=0): return np.concatenate(arrays, axis) def zero_pad(self, a, pad_width, value=0): return np.pad(a, pad_width, constant_values=value) def argmax(self, a, axis=None): return np.argmax(a, axis=axis) def argmin(self, a, axis=None): return np.argmin(a, axis=axis) def mean(self, a, axis=None): return np.mean(a, axis=axis) def std(self, a, axis=None): return np.std(a, axis=axis) def linspace(self, start, stop, num): return np.linspace(start, stop, num) def meshgrid(self, a, b): return np.meshgrid(a, b) def diag(self, a, k=0): return np.diag(a, k) def unique(self, a): return np.unique(a) def logsumexp(self, a, axis=None): return special.logsumexp(a, axis=axis) def stack(self, arrays, axis=0): return np.stack(arrays, axis) def reshape(self, a, shape): return np.reshape(a, shape) def seed(self, seed=None): if seed is not None: self.rng_.seed(seed) def rand(self, *size, type_as=None): return self.rng_.rand(*size) def randn(self, *size, type_as=None): return self.rng_.randn(*size) def coo_matrix(self, data, rows, cols, shape=None, type_as=None): if type_as is None: return coo_matrix((data, (rows, cols)), shape=shape) else: return coo_matrix((data, (rows, cols)), shape=shape, dtype=type_as.dtype) def issparse(self, a): return issparse(a) def tocsr(self, a): if self.issparse(a): return a.tocsr() else: return csr_matrix(a) def eliminate_zeros(self, a, threshold=0.): if threshold > 0: if self.issparse(a): a.data[self.abs(a.data) <= threshold] = 0 else: a[self.abs(a) <= threshold] = 0 if self.issparse(a): a.eliminate_zeros() return a def todense(self, a): if self.issparse(a): return a.toarray() else: return a def where(self, condition, x=None, y=None): if x is None and y is None: return np.where(condition) else: return np.where(condition, x, y) def copy(self, a): return a.copy() def allclose(self, a, b, rtol=1e-05, atol=1e-08, equal_nan=False): return np.allclose(a, b, rtol=rtol, atol=atol, equal_nan=equal_nan) def dtype_device(self, a): if hasattr(a, "dtype"): return a.dtype, "cpu" else: return type(a), "cpu" def assert_same_dtype_device(self, a, b): # numpy has implicit type conversion so we automatically validate the test pass def squeeze(self, a, axis=None): return np.squeeze(a, axis=axis) def bitsize(self, type_as): return type_as.itemsize * 8 def device_type(self, type_as): return "CPU" def _bench(self, callable, *args, n_runs=1, warmup_runs=1): results = dict() for type_as in self.__type_list__: inputs = [self.from_numpy(arg, type_as=type_as) for arg in args] for _ in range(warmup_runs): callable(*inputs) t0 = time.perf_counter() for _ in range(n_runs): callable(*inputs) t1 = time.perf_counter() key = ("Numpy", self.device_type(type_as), self.bitsize(type_as)) results[key] = (t1 - t0) / n_runs return results def solve(self, a, b): return np.linalg.solve(a, b) def trace(self, a): return np.trace(a) def inv(self, a): return scipy.linalg.inv(a) def sqrtm(self, a): return scipy.linalg.sqrtm(a) def kl_div(self, p, q, eps=1e-16): return np.sum(p * np.log(p / q + eps)) def isfinite(self, a): return np.isfinite(a) def array_equal(self, a, b): return np.array_equal(a, b) def is_floating_point(self, a): return a.dtype.kind == "f" def tile(self, a, reps): return np.tile(a, reps) def floor(self, a): return np.floor(a) def prod(self, a, axis=0): return np.prod(a, axis=axis) def sort2(self, a, axis=-1): return self.sort(a, axis), self.argsort(a, axis) def qr(self, a): np_version = tuple([int(k) for k in np.__version__.split(".")]) if np_version < (1, 22, 0): M, N = a.shape[-2], a.shape[-1] K = min(M, N) if len(a.shape) >= 3: n = a.shape[0] qs, rs = np.zeros((n, M, K)), np.zeros((n, K, N)) for i in range(a.shape[0]): qs[i], rs[i] = np.linalg.qr(a[i]) else: return np.linalg.qr(a) return qs, rs return np.linalg.qr(a) def atan2(self, a, b): return np.arctan2(a, b) def transpose(self, a, axes=None): return np.transpose(a, axes) class JaxBackend(Backend): """ JAX implementation of the backend - `__name__` is "jax" - `__type__` is jax.numpy.ndarray """ __name__ = 'jax' __type__ = jax_type __type_list__ = None rng_ = None def __init__(self): self.rng_ = jax.random.PRNGKey(42) self.__type_list__ = [] # available_devices = jax.devices("cpu") available_devices = [] if xla_bridge.get_backend().platform == "gpu": available_devices += jax.devices("gpu") for d in available_devices: self.__type_list__ += [ jax.device_put(jnp.array(1, dtype=jnp.float32), d), jax.device_put(jnp.array(1, dtype=jnp.float64), d) ] def _to_numpy(self, a): return np.array(a) def _change_device(self, a, type_as): return jax.device_put(a, type_as.device_buffer.device()) def _from_numpy(self, a, type_as=None): if isinstance(a, float): a = np.array(a) if type_as is None: return jnp.array(a) else: return self._change_device(jnp.array(a).astype(type_as.dtype), type_as) def set_gradients(self, val, inputs, grads): from jax.flatten_util import ravel_pytree val, = jax.lax.stop_gradient((val,)) ravelled_inputs, _ = ravel_pytree(inputs) ravelled_grads, _ = ravel_pytree(grads) aux = jnp.sum(ravelled_inputs * ravelled_grads) / 2 aux = aux - jax.lax.stop_gradient(aux) val, = jax.tree_map(lambda z: z + aux, (val,)) return val def zeros(self, shape, type_as=None): if type_as is None: return jnp.zeros(shape) else: return self._change_device(jnp.zeros(shape, dtype=type_as.dtype), type_as) def ones(self, shape, type_as=None): if type_as is None: return jnp.ones(shape) else: return self._change_device(jnp.ones(shape, dtype=type_as.dtype), type_as) def arange(self, stop, start=0, step=1, type_as=None): return jnp.arange(start, stop, step) def full(self, shape, fill_value, type_as=None): if type_as is None: return jnp.full(shape, fill_value) else: return self._change_device(jnp.full(shape, fill_value, dtype=type_as.dtype), type_as) def eye(self, N, M=None, type_as=None): if type_as is None: return jnp.eye(N, M) else: return self._change_device(jnp.eye(N, M, dtype=type_as.dtype), type_as) def sum(self, a, axis=None, keepdims=False): return jnp.sum(a, axis, keepdims=keepdims) def cumsum(self, a, axis=None): return jnp.cumsum(a, axis) def max(self, a, axis=None, keepdims=False): return jnp.max(a, axis, keepdims=keepdims) def min(self, a, axis=None, keepdims=False): return jnp.min(a, axis, keepdims=keepdims) def maximum(self, a, b): return jnp.maximum(a, b) def minimum(self, a, b): return jnp.minimum(a, b) def dot(self, a, b): return jnp.dot(a, b) def abs(self, a): return jnp.abs(a) def exp(self, a): return jnp.exp(a) def log(self, a): return jnp.log(a) def sqrt(self, a): return jnp.sqrt(a) def power(self, a, exponents): return jnp.power(a, exponents) def norm(self, a): return jnp.sqrt(jnp.sum(jnp.square(a))) def any(self, a): return jnp.any(a) def isnan(self, a): return jnp.isnan(a) def isinf(self, a): return jnp.isinf(a) def einsum(self, subscripts, *operands): return jnp.einsum(subscripts, *operands) def sort(self, a, axis=-1): return jnp.sort(a, axis) def argsort(self, a, axis=-1): return jnp.argsort(a, axis) def searchsorted(self, a, v, side='left'): if a.ndim == 1: return jnp.searchsorted(a, v, side) else: # this is a not very efficient way to make jax numpy # searchsorted work on 2d arrays return jnp.array([jnp.searchsorted(a[i, :], v[i, :], side) for i in range(a.shape[0])]) def flip(self, a, axis=None): return jnp.flip(a, axis) def outer(self, a, b): return jnp.outer(a, b) def clip(self, a, a_min, a_max): return jnp.clip(a, a_min, a_max) def repeat(self, a, repeats, axis=None): return jnp.repeat(a, repeats, axis) def take_along_axis(self, arr, indices, axis): return jnp.take_along_axis(arr, indices, axis) def concatenate(self, arrays, axis=0): return jnp.concatenate(arrays, axis) def zero_pad(self, a, pad_width, value=0): return jnp.pad(a, pad_width, constant_values=value) def argmax(self, a, axis=None): return jnp.argmax(a, axis=axis) def argmin(self, a, axis=None): return jnp.argmin(a, axis=axis) def mean(self, a, axis=None): return jnp.mean(a, axis=axis) def std(self, a, axis=None): return jnp.std(a, axis=axis) def linspace(self, start, stop, num): return jnp.linspace(start, stop, num) def meshgrid(self, a, b): return jnp.meshgrid(a, b) def diag(self, a, k=0): return jnp.diag(a, k) def unique(self, a): return jnp.unique(a) def logsumexp(self, a, axis=None): return jspecial.logsumexp(a, axis=axis) def stack(self, arrays, axis=0): return jnp.stack(arrays, axis) def reshape(self, a, shape): return jnp.reshape(a, shape) def seed(self, seed=None): if seed is not None: self.rng_ = jax.random.PRNGKey(seed) def rand(self, *size, type_as=None): self.rng_, subkey = jax.random.split(self.rng_) if type_as is not None: return jax.random.uniform(subkey, shape=size, dtype=type_as.dtype) else: return jax.random.uniform(subkey, shape=size) def randn(self, *size, type_as=None): self.rng_, subkey = jax.random.split(self.rng_) if type_as is not None: return jax.random.normal(subkey, shape=size, dtype=type_as.dtype) else: return jax.random.normal(subkey, shape=size) def coo_matrix(self, data, rows, cols, shape=None, type_as=None): # Currently, JAX does not support sparse matrices data = self.to_numpy(data) rows = self.to_numpy(rows) cols = self.to_numpy(cols) nx = NumpyBackend() coo_matrix = nx.coo_matrix(data, rows, cols, shape=shape, type_as=type_as) matrix = nx.todense(coo_matrix) return self.from_numpy(matrix) def issparse(self, a): # Currently, JAX does not support sparse matrices return False def tocsr(self, a): # Currently, JAX does not support sparse matrices return a def eliminate_zeros(self, a, threshold=0.): # Currently, JAX does not support sparse matrices if threshold > 0: return self.where( self.abs(a) <= threshold, self.zeros((1,), type_as=a), a ) return a def todense(self, a): # Currently, JAX does not support sparse matrices return a def where(self, condition, x=None, y=None): if x is None and y is None: return jnp.where(condition) else: return jnp.where(condition, x, y) def copy(self, a): # No need to copy, JAX arrays are immutable return a def allclose(self, a, b, rtol=1e-05, atol=1e-08, equal_nan=False): return jnp.allclose(a, b, rtol=rtol, atol=atol, equal_nan=equal_nan) def dtype_device(self, a): return a.dtype, a.device_buffer.device() def assert_same_dtype_device(self, a, b): a_dtype, a_device = self.dtype_device(a) b_dtype, b_device = self.dtype_device(b) assert a_dtype == b_dtype, "Dtype discrepancy" assert a_device == b_device, f"Device discrepancy. First input is on {str(a_device)}, whereas second input is on {str(b_device)}" def squeeze(self, a, axis=None): return jnp.squeeze(a, axis=axis) def bitsize(self, type_as): return type_as.dtype.itemsize * 8 def device_type(self, type_as): return self.dtype_device(type_as)[1].platform.upper() def _bench(self, callable, *args, n_runs=1, warmup_runs=1): results = dict() for type_as in self.__type_list__: inputs = [self.from_numpy(arg, type_as=type_as) for arg in args] for _ in range(warmup_runs): a = callable(*inputs) a.block_until_ready() t0 = time.perf_counter() for _ in range(n_runs): a = callable(*inputs) a.block_until_ready() t1 = time.perf_counter() key = ("Jax", self.device_type(type_as), self.bitsize(type_as)) results[key] = (t1 - t0) / n_runs return results def solve(self, a, b): return jnp.linalg.solve(a, b) def trace(self, a): return jnp.trace(a) def inv(self, a): return jnp.linalg.inv(a) def sqrtm(self, a): L, V = jnp.linalg.eigh(a) return (V * jnp.sqrt(L)[None, :]) @ V.T def kl_div(self, p, q, eps=1e-16): return jnp.sum(p * jnp.log(p / q + eps)) def isfinite(self, a): return jnp.isfinite(a) def array_equal(self, a, b): return jnp.array_equal(a, b) def is_floating_point(self, a): return a.dtype.kind == "f" def tile(self, a, reps): return jnp.tile(a, reps) def floor(self, a): return jnp.floor(a) def prod(self, a, axis=0): return jnp.prod(a, axis=axis) def sort2(self, a, axis=-1): return self.sort(a, axis), self.argsort(a, axis) def qr(self, a): return jnp.linalg.qr(a) def atan2(self, a, b): return jnp.arctan2(a, b) def transpose(self, a, axes=None): return jnp.transpose(a, axes) class TorchBackend(Backend): """ PyTorch implementation of the backend - `__name__` is "torch" - `__type__` is torch.Tensor """ __name__ = 'torch' __type__ = torch_type __type_list__ = None rng_ = None def __init__(self): self.rng_ = torch.Generator("cpu") self.rng_.seed() self.__type_list__ = [torch.tensor(1, dtype=torch.float32), torch.tensor(1, dtype=torch.float64)] if torch.cuda.is_available(): self.rng_cuda_ = torch.Generator("cuda") self.rng_cuda_.seed() self.__type_list__.append(torch.tensor(1, dtype=torch.float32, device='cuda')) self.__type_list__.append(torch.tensor(1, dtype=torch.float64, device='cuda')) else: self.rng_cuda_ = torch.Generator("cpu") from torch.autograd import Function # define a function that takes inputs val and grads # ad returns a val tensor with proper gradients class ValFunction(Function): @staticmethod def forward(ctx, val, grads, *inputs): ctx.grads = grads return val @staticmethod def backward(ctx, grad_output): # the gradients are grad return (None, None) + tuple(g * grad_output for g in ctx.grads) self.ValFunction = ValFunction def _to_numpy(self, a): return a.cpu().detach().numpy() def _from_numpy(self, a, type_as=None): if isinstance(a, float): a = np.array(a) if type_as is None: return torch.from_numpy(a) else: return torch.as_tensor(a, dtype=type_as.dtype, device=type_as.device) def set_gradients(self, val, inputs, grads): Func = self.ValFunction res = Func.apply(val, grads, *inputs) return res def zeros(self, shape, type_as=None): if isinstance(shape, int): shape = (shape,) if type_as is None: return torch.zeros(shape) else: return torch.zeros(shape, dtype=type_as.dtype, device=type_as.device) def ones(self, shape, type_as=None): if isinstance(shape, int): shape = (shape,) if type_as is None: return torch.ones(shape) else: return torch.ones(shape, dtype=type_as.dtype, device=type_as.device) def arange(self, stop, start=0, step=1, type_as=None): if type_as is None: return torch.arange(start, stop, step) else: return torch.arange(start, stop, step, device=type_as.device) def full(self, shape, fill_value, type_as=None): if isinstance(shape, int): shape = (shape,) if type_as is None: return torch.full(shape, fill_value) else: return torch.full(shape, fill_value, dtype=type_as.dtype, device=type_as.device) def eye(self, N, M=None, type_as=None): if M is None: M = N if type_as is None: return torch.eye(N, m=M) else: return torch.eye(N, m=M, dtype=type_as.dtype, device=type_as.device) def sum(self, a, axis=None, keepdims=False): if axis is None: return torch.sum(a) else: return torch.sum(a, axis, keepdim=keepdims) def cumsum(self, a, axis=None): if axis is None: return torch.cumsum(a.flatten(), 0) else: return torch.cumsum(a, axis) def max(self, a, axis=None, keepdims=False): if axis is None: return torch.max(a) else: return torch.max(a, axis, keepdim=keepdims)[0] def min(self, a, axis=None, keepdims=False): if axis is None: return torch.min(a) else: return torch.min(a, axis, keepdim=keepdims)[0] def maximum(self, a, b): if isinstance(a, int) or isinstance(a, float): a = torch.tensor([float(a)], dtype=b.dtype, device=b.device) if isinstance(b, int) or isinstance(b, float): b = torch.tensor([float(b)], dtype=a.dtype, device=a.device) if hasattr(torch, "maximum"): return torch.maximum(a, b) else: return torch.max(torch.stack(torch.broadcast_tensors(a, b)), axis=0)[0] def minimum(self, a, b): if isinstance(a, int) or isinstance(a, float): a = torch.tensor([float(a)], dtype=b.dtype, device=b.device) if isinstance(b, int) or isinstance(b, float): b = torch.tensor([float(b)], dtype=a.dtype, device=a.device) if hasattr(torch, "minimum"): return torch.minimum(a, b) else: return torch.min(torch.stack(torch.broadcast_tensors(a, b)), axis=0)[0] def dot(self, a, b): return torch.matmul(a, b) def abs(self, a): return torch.abs(a) def exp(self, a): return torch.exp(a) def log(self, a): return torch.log(a) def sqrt(self, a): return torch.sqrt(a) def power(self, a, exponents): return torch.pow(a, exponents) def norm(self, a): return torch.sqrt(torch.sum(torch.square(a))) def any(self, a): return torch.any(a) def isnan(self, a): return torch.isnan(a) def isinf(self, a): return torch.isinf(a) def einsum(self, subscripts, *operands): return torch.einsum(subscripts, *operands) def sort(self, a, axis=-1): sorted0, indices = torch.sort(a, dim=axis) return sorted0 def argsort(self, a, axis=-1): sorted, indices = torch.sort(a, dim=axis) return indices def searchsorted(self, a, v, side='left'): right = (side != 'left') return torch.searchsorted(a, v, right=right) def flip(self, a, axis=None): if axis is None: return torch.flip(a, tuple(i for i in range(len(a.shape)))) if isinstance(axis, int): return torch.flip(a, (axis,)) else: return torch.flip(a, dims=axis) def outer(self, a, b): return torch.outer(a, b) def clip(self, a, a_min, a_max): return torch.clamp(a, a_min, a_max) def repeat(self, a, repeats, axis=None): return torch.repeat_interleave(a, repeats, dim=axis) def take_along_axis(self, arr, indices, axis): return torch.gather(arr, axis, indices) def concatenate(self, arrays, axis=0): return torch.cat(arrays, dim=axis) def zero_pad(self, a, pad_width, value=0): from torch.nn.functional import pad # pad_width is an array of ndim tuples indicating how many 0 before and after # we need to add. We first need to make it compliant with torch syntax, that # starts with the last dim, then second last, etc. how_pad = tuple(element for tupl in pad_width[::-1] for element in tupl) return pad(a, how_pad, value=value) def argmax(self, a, axis=None): return torch.argmax(a, dim=axis) def argmin(self, a, axis=None): return torch.argmin(a, dim=axis) def mean(self, a, axis=None): if axis is not None: return torch.mean(a, dim=axis) else: return torch.mean(a) def std(self, a, axis=None): if axis is not None: return torch.std(a, dim=axis, unbiased=False) else: return torch.std(a, unbiased=False) def linspace(self, start, stop, num): return torch.linspace(start, stop, num, dtype=torch.float64) def meshgrid(self, a, b): try: return torch.meshgrid(a, b, indexing="xy") except TypeError: X, Y = torch.meshgrid(a, b) return X.T, Y.T def diag(self, a, k=0): return torch.diag(a, diagonal=k) def unique(self, a): return torch.unique(a) def logsumexp(self, a, axis=None): if axis is not None: return torch.logsumexp(a, dim=axis) else: return torch.logsumexp(a, dim=tuple(range(len(a.shape)))) def stack(self, arrays, axis=0): return torch.stack(arrays, dim=axis) def reshape(self, a, shape): return torch.reshape(a, shape) def seed(self, seed=None): if isinstance(seed, int): self.rng_.manual_seed(seed) self.rng_cuda_.manual_seed(seed) elif isinstance(seed, torch.Generator): if self.device_type(seed) == "GPU": self.rng_cuda_ = seed else: self.rng_ = seed else: raise ValueError("Non compatible seed : {}".format(seed)) def rand(self, *size, type_as=None): if type_as is not None: generator = self.rng_cuda_ if self.device_type(type_as) == "GPU" else self.rng_ return torch.rand(size=size, generator=generator, dtype=type_as.dtype, device=type_as.device) else: return torch.rand(size=size, generator=self.rng_) def randn(self, *size, type_as=None): if type_as is not None: generator = self.rng_cuda_ if self.device_type(type_as) == "GPU" else self.rng_ return torch.randn(size=size, dtype=type_as.dtype, generator=generator, device=type_as.device) else: return torch.randn(size=size, generator=self.rng_) def coo_matrix(self, data, rows, cols, shape=None, type_as=None): if type_as is None: return torch.sparse_coo_tensor(torch.stack([rows, cols]), data, size=shape) else: return torch.sparse_coo_tensor( torch.stack([rows, cols]), data, size=shape, dtype=type_as.dtype, device=type_as.device ) def issparse(self, a): return getattr(a, "is_sparse", False) or getattr(a, "is_sparse_csr", False) def tocsr(self, a): # Versions older than 1.9 do not support CSR tensors. PyTorch 1.9 and 1.10 offer a very limited support return self.todense(a) def eliminate_zeros(self, a, threshold=0.): if self.issparse(a): if threshold > 0: mask = self.abs(a) <= threshold mask = ~mask mask = mask.nonzero() else: mask = a._values().nonzero() nv = a._values().index_select(0, mask.view(-1)) ni = a._indices().index_select(1, mask.view(-1)) return self.coo_matrix(nv, ni[0], ni[1], shape=a.shape, type_as=a) else: if threshold > 0: a[self.abs(a) <= threshold] = 0 return a def todense(self, a): if self.issparse(a): return a.to_dense() else: return a def where(self, condition, x=None, y=None): if x is None and y is None: return torch.where(condition) else: return torch.where(condition, x, y) def copy(self, a): return torch.clone(a) def allclose(self, a, b, rtol=1e-05, atol=1e-08, equal_nan=False): return torch.allclose(a, b, rtol=rtol, atol=atol, equal_nan=equal_nan) def dtype_device(self, a): return a.dtype, a.device def assert_same_dtype_device(self, a, b): a_dtype, a_device = self.dtype_device(a) b_dtype, b_device = self.dtype_device(b) assert a_dtype == b_dtype, "Dtype discrepancy" assert a_device == b_device, f"Device discrepancy. First input is on {str(a_device)}, whereas second input is on {str(b_device)}" def squeeze(self, a, axis=None): if axis is None: return torch.squeeze(a) else: return torch.squeeze(a, dim=axis) def bitsize(self, type_as): return torch.finfo(type_as.dtype).bits def device_type(self, type_as): return type_as.device.type.replace("cuda", "gpu").upper() def _bench(self, callable, *args, n_runs=1, warmup_runs=1): results = dict() for type_as in self.__type_list__: inputs = [self.from_numpy(arg, type_as=type_as) for arg in args] for _ in range(warmup_runs): callable(*inputs) if self.device_type(type_as) == "GPU": # pragma: no cover torch.cuda.synchronize() start = torch.cuda.Event(enable_timing=True) end = torch.cuda.Event(enable_timing=True) start.record() else: start = time.perf_counter() for _ in range(n_runs): callable(*inputs) if self.device_type(type_as) == "GPU": # pragma: no cover end.record() torch.cuda.synchronize() duration = start.elapsed_time(end) / 1000. else: end = time.perf_counter() duration = end - start key = ("Pytorch", self.device_type(type_as), self.bitsize(type_as)) results[key] = duration / n_runs if torch.cuda.is_available(): torch.cuda.empty_cache() return results def solve(self, a, b): return torch.linalg.solve(a, b) def trace(self, a): return torch.trace(a) def inv(self, a): return torch.linalg.inv(a) def sqrtm(self, a): L, V = torch.linalg.eigh(a) return (V * torch.sqrt(L)[None, :]) @ V.T def kl_div(self, p, q, eps=1e-16): return torch.sum(p * torch.log(p / q + eps)) def isfinite(self, a): return torch.isfinite(a) def array_equal(self, a, b): return torch.equal(a, b) def is_floating_point(self, a): return a.dtype.is_floating_point def tile(self, a, reps): return a.repeat(reps) def floor(self, a): return torch.floor(a) def prod(self, a, axis=0): return torch.prod(a, dim=axis) def sort2(self, a, axis=-1): return torch.sort(a, axis) def qr(self, a): return torch.linalg.qr(a) def atan2(self, a, b): return torch.atan2(a, b) def transpose(self, a, axes=None): if axes is None: axes = tuple(range(a.ndim)[::-1]) return a.permute(axes) class CupyBackend(Backend): # pragma: no cover """ CuPy implementation of the backend - `__name__` is "cupy" - `__type__` is cp.ndarray """ __name__ = 'cupy' __type__ = cp_type __type_list__ = None rng_ = None def __init__(self): self.rng_ = cp.random.RandomState() self.__type_list__ = [ cp.array(1, dtype=cp.float32), cp.array(1, dtype=cp.float64) ] def _to_numpy(self, a): return cp.asnumpy(a) def _from_numpy(self, a, type_as=None): if isinstance(a, float): a = np.array(a) if type_as is None: return cp.asarray(a) else: with cp.cuda.Device(type_as.device): return cp.asarray(a, dtype=type_as.dtype) def set_gradients(self, val, inputs, grads): # No gradients for cupy return val def zeros(self, shape, type_as=None): if isinstance(shape, (list, tuple)): shape = tuple(int(i) for i in shape) if type_as is None: return cp.zeros(shape) else: with cp.cuda.Device(type_as.device): return cp.zeros(shape, dtype=type_as.dtype) def ones(self, shape, type_as=None): if isinstance(shape, (list, tuple)): shape = tuple(int(i) for i in shape) if type_as is None: return cp.ones(shape) else: with cp.cuda.Device(type_as.device): return cp.ones(shape, dtype=type_as.dtype) def arange(self, stop, start=0, step=1, type_as=None): return cp.arange(start, stop, step) def full(self, shape, fill_value, type_as=None): if isinstance(shape, (list, tuple)): shape = tuple(int(i) for i in shape) if type_as is None: return cp.full(shape, fill_value) else: with cp.cuda.Device(type_as.device): return cp.full(shape, fill_value, dtype=type_as.dtype) def eye(self, N, M=None, type_as=None): if type_as is None: return cp.eye(N, M) else: with cp.cuda.Device(type_as.device): return cp.eye(N, M, dtype=type_as.dtype) def sum(self, a, axis=None, keepdims=False): return cp.sum(a, axis, keepdims=keepdims) def cumsum(self, a, axis=None): return cp.cumsum(a, axis) def max(self, a, axis=None, keepdims=False): return cp.max(a, axis, keepdims=keepdims) def min(self, a, axis=None, keepdims=False): return cp.min(a, axis, keepdims=keepdims) def maximum(self, a, b): return cp.maximum(a, b) def minimum(self, a, b): return cp.minimum(a, b) def abs(self, a): return cp.abs(a) def exp(self, a): return cp.exp(a) def log(self, a): return cp.log(a) def sqrt(self, a): return cp.sqrt(a) def power(self, a, exponents): return cp.power(a, exponents) def dot(self, a, b): return cp.dot(a, b) def norm(self, a): return cp.sqrt(cp.sum(cp.square(a))) def any(self, a): return cp.any(a) def isnan(self, a): return cp.isnan(a) def isinf(self, a): return cp.isinf(a) def einsum(self, subscripts, *operands): return cp.einsum(subscripts, *operands) def sort(self, a, axis=-1): return cp.sort(a, axis) def argsort(self, a, axis=-1): return cp.argsort(a, axis) def searchsorted(self, a, v, side='left'): if a.ndim == 1: return cp.searchsorted(a, v, side) else: # this is a not very efficient way to make numpy # searchsorted work on 2d arrays ret = cp.empty(v.shape, dtype=int) for i in range(a.shape[0]): ret[i, :] = cp.searchsorted(a[i, :], v[i, :], side) return ret def flip(self, a, axis=None): return cp.flip(a, axis) def outer(self, a, b): return cp.outer(a, b) def clip(self, a, a_min, a_max): return cp.clip(a, a_min, a_max) def repeat(self, a, repeats, axis=None): return cp.repeat(a, repeats, axis) def take_along_axis(self, arr, indices, axis): return cp.take_along_axis(arr, indices, axis) def concatenate(self, arrays, axis=0): return cp.concatenate(arrays, axis) def zero_pad(self, a, pad_width, value=0): return cp.pad(a, pad_width, constant_values=value) def argmax(self, a, axis=None): return cp.argmax(a, axis=axis) def argmin(self, a, axis=None): return cp.argmin(a, axis=axis) def mean(self, a, axis=None): return cp.mean(a, axis=axis) def std(self, a, axis=None): return cp.std(a, axis=axis) def linspace(self, start, stop, num): return cp.linspace(start, stop, num) def meshgrid(self, a, b): return cp.meshgrid(a, b) def diag(self, a, k=0): return cp.diag(a, k) def unique(self, a): return cp.unique(a) def logsumexp(self, a, axis=None): # Taken from # https://github.com/scipy/scipy/blob/v1.7.1/scipy/special/_logsumexp.py#L7-L127 a_max = cp.amax(a, axis=axis, keepdims=True) if a_max.ndim > 0: a_max[~cp.isfinite(a_max)] = 0 elif not cp.isfinite(a_max): a_max = 0 tmp = cp.exp(a - a_max) s = cp.sum(tmp, axis=axis) out = cp.log(s) a_max = cp.squeeze(a_max, axis=axis) out += a_max return out def stack(self, arrays, axis=0): return cp.stack(arrays, axis) def reshape(self, a, shape): return cp.reshape(a, shape) def seed(self, seed=None): if seed is not None: self.rng_.seed(seed) def rand(self, *size, type_as=None): if type_as is None: return self.rng_.rand(*size) else: with cp.cuda.Device(type_as.device): return self.rng_.rand(*size, dtype=type_as.dtype) def randn(self, *size, type_as=None): if type_as is None: return self.rng_.randn(*size) else: with cp.cuda.Device(type_as.device): return self.rng_.randn(*size, dtype=type_as.dtype) def coo_matrix(self, data, rows, cols, shape=None, type_as=None): data = self.from_numpy(data) rows = self.from_numpy(rows) cols = self.from_numpy(cols) if type_as is None: return cupyx.scipy.sparse.coo_matrix( (data, (rows, cols)), shape=shape ) else: with cp.cuda.Device(type_as.device): return cupyx.scipy.sparse.coo_matrix( (data, (rows, cols)), shape=shape, dtype=type_as.dtype ) def issparse(self, a): return cupyx.scipy.sparse.issparse(a) def tocsr(self, a): if self.issparse(a): return a.tocsr() else: return cupyx.scipy.sparse.csr_matrix(a) def eliminate_zeros(self, a, threshold=0.): if threshold > 0: if self.issparse(a): a.data[self.abs(a.data) <= threshold] = 0 else: a[self.abs(a) <= threshold] = 0 if self.issparse(a): a.eliminate_zeros() return a def todense(self, a): if self.issparse(a): return a.toarray() else: return a def where(self, condition, x=None, y=None): if x is None and y is None: return cp.where(condition) else: return cp.where(condition, x, y) def copy(self, a): return a.copy() def allclose(self, a, b, rtol=1e-05, atol=1e-08, equal_nan=False): return cp.allclose(a, b, rtol=rtol, atol=atol, equal_nan=equal_nan) def dtype_device(self, a): return a.dtype, a.device def assert_same_dtype_device(self, a, b): a_dtype, a_device = self.dtype_device(a) b_dtype, b_device = self.dtype_device(b) # cupy has implicit type conversion so # we automatically validate the test for type assert a_device == b_device, f"Device discrepancy. First input is on {str(a_device)}, whereas second input is on {str(b_device)}" def squeeze(self, a, axis=None): return cp.squeeze(a, axis=axis) def bitsize(self, type_as): return type_as.itemsize * 8 def device_type(self, type_as): return "GPU" def _bench(self, callable, *args, n_runs=1, warmup_runs=1): mempool = cp.get_default_memory_pool() pinned_mempool = cp.get_default_pinned_memory_pool() results = dict() for type_as in self.__type_list__: inputs = [self.from_numpy(arg, type_as=type_as) for arg in args] start_gpu = cp.cuda.Event() end_gpu = cp.cuda.Event() for _ in range(warmup_runs): callable(*inputs) start_gpu.synchronize() start_gpu.record() for _ in range(n_runs): callable(*inputs) end_gpu.record() end_gpu.synchronize() key = ("Cupy", self.device_type(type_as), self.bitsize(type_as)) t_gpu = cp.cuda.get_elapsed_time(start_gpu, end_gpu) / 1000. results[key] = t_gpu / n_runs mempool.free_all_blocks() pinned_mempool.free_all_blocks() return results def solve(self, a, b): return cp.linalg.solve(a, b) def trace(self, a): return cp.trace(a) def inv(self, a): return cp.linalg.inv(a) def sqrtm(self, a): L, V = cp.linalg.eigh(a) return (V * self.sqrt(L)[None, :]) @ V.T def kl_div(self, p, q, eps=1e-16): return cp.sum(p * cp.log(p / q + eps)) def isfinite(self, a): return cp.isfinite(a) def array_equal(self, a, b): return cp.array_equal(a, b) def is_floating_point(self, a): return a.dtype.kind == "f" def tile(self, a, reps): return cp.tile(a, reps) def floor(self, a): return cp.floor(a) def prod(self, a, axis=0): return cp.prod(a, axis=axis) def sort2(self, a, axis=-1): return self.sort(a, axis), self.argsort(a, axis) def qr(self, a): return cp.linalg.qr(a) def atan2(self, a, b): return cp.arctan2(a, b) def transpose(self, a, axes=None): return cp.transpose(a, axes) class TensorflowBackend(Backend): __name__ = "tf" __type__ = tf_type __type_list__ = None rng_ = None def __init__(self): self.seed(None) self.__type_list__ = [ tf.convert_to_tensor([1], dtype=tf.float32), tf.convert_to_tensor([1], dtype=tf.float64) ] tmp = self.randn(15, 10) try: tmp.reshape((150, 1)) except AttributeError: warnings.warn( "To use TensorflowBackend, you need to activate the tensorflow " "numpy API. You can activate it by running: \n" "from tensorflow.python.ops.numpy_ops import np_config\n" "np_config.enable_numpy_behavior()", stacklevel=2 ) def _to_numpy(self, a): return a.numpy() def _from_numpy(self, a, type_as=None): if isinstance(a, float): a = np.array(a) if not isinstance(a, self.__type__): if type_as is None: return tf.convert_to_tensor(a) else: return tf.convert_to_tensor(a, dtype=type_as.dtype) else: if type_as is None: return a else: return tf.cast(a, dtype=type_as.dtype) def set_gradients(self, val, inputs, grads): @tf.custom_gradient def tmp(input): def grad(upstream): return grads return val, grad return tmp(inputs) def zeros(self, shape, type_as=None): if type_as is None: return tnp.zeros(shape) else: return tnp.zeros(shape, dtype=type_as.dtype) def ones(self, shape, type_as=None): if type_as is None: return tnp.ones(shape) else: return tnp.ones(shape, dtype=type_as.dtype) def arange(self, stop, start=0, step=1, type_as=None): return tnp.arange(start, stop, step) def full(self, shape, fill_value, type_as=None): if type_as is None: return tnp.full(shape, fill_value) else: return tnp.full(shape, fill_value, dtype=type_as.dtype) def eye(self, N, M=None, type_as=None): if type_as is None: return tnp.eye(N, M) else: return tnp.eye(N, M, dtype=type_as.dtype) def sum(self, a, axis=None, keepdims=False): return tnp.sum(a, axis, keepdims=keepdims) def cumsum(self, a, axis=None): return tnp.cumsum(a, axis) def max(self, a, axis=None, keepdims=False): return tnp.max(a, axis, keepdims=keepdims) def min(self, a, axis=None, keepdims=False): return tnp.min(a, axis, keepdims=keepdims) def maximum(self, a, b): return tnp.maximum(a, b) def minimum(self, a, b): return tnp.minimum(a, b) def dot(self, a, b): if len(b.shape) == 1: if len(a.shape) == 1: # inner product return tf.reduce_sum(tf.multiply(a, b)) else: # matrix vector return tf.linalg.matvec(a, b) else: if len(a.shape) == 1: return tf.linalg.matvec(b.T, a.T).T else: return tf.matmul(a, b) def abs(self, a): return tnp.abs(a) def exp(self, a): return tnp.exp(a) def log(self, a): return tnp.log(a) def sqrt(self, a): return tnp.sqrt(a) def power(self, a, exponents): return tnp.power(a, exponents) def norm(self, a): return tf.math.reduce_euclidean_norm(a) def any(self, a): return tnp.any(a) def isnan(self, a): return tnp.isnan(a) def isinf(self, a): return tnp.isinf(a) def einsum(self, subscripts, *operands): return tnp.einsum(subscripts, *operands) def sort(self, a, axis=-1): return tnp.sort(a, axis) def argsort(self, a, axis=-1): return tnp.argsort(a, axis) def searchsorted(self, a, v, side='left'): return tf.searchsorted(a, v, side=side) def flip(self, a, axis=None): return tnp.flip(a, axis) def outer(self, a, b): return tnp.outer(a, b) def clip(self, a, a_min, a_max): return tnp.clip(a, a_min, a_max) def repeat(self, a, repeats, axis=None): return tnp.repeat(a, repeats, axis) def take_along_axis(self, arr, indices, axis): return tnp.take_along_axis(arr, indices, axis) def concatenate(self, arrays, axis=0): return tnp.concatenate(arrays, axis) def zero_pad(self, a, pad_width, value=0): return tnp.pad(a, pad_width, mode="constant", constant_values=value) def argmax(self, a, axis=None): return tnp.argmax(a, axis=axis) def argmin(self, a, axis=None): return tnp.argmin(a, axis=axis) def mean(self, a, axis=None): return tnp.mean(a, axis=axis) def std(self, a, axis=None): return tnp.std(a, axis=axis) def linspace(self, start, stop, num): return tnp.linspace(start, stop, num) def meshgrid(self, a, b): return tnp.meshgrid(a, b) def diag(self, a, k=0): return tnp.diag(a, k) def unique(self, a): return tf.sort(tf.unique(tf.reshape(a, [-1]))[0]) def logsumexp(self, a, axis=None): return tf.math.reduce_logsumexp(a, axis=axis) def stack(self, arrays, axis=0): return tnp.stack(arrays, axis) def reshape(self, a, shape): return tnp.reshape(a, shape) def seed(self, seed=None): if isinstance(seed, int): self.rng_ = tf.random.Generator.from_seed(seed) elif isinstance(seed, tf.random.Generator): self.rng_ = seed elif seed is None: self.rng_ = tf.random.Generator.from_non_deterministic_state() else: raise ValueError("Non compatible seed : {}".format(seed)) def rand(self, *size, type_as=None): if type_as is None: return self.rng_.uniform(size, minval=0., maxval=1.) else: return self.rng_.uniform( size, minval=0., maxval=1., dtype=type_as.dtype ) def randn(self, *size, type_as=None): if type_as is None: return self.rng_.normal(size) else: return self.rng_.normal(size, dtype=type_as.dtype) def _convert_to_index_for_coo(self, tensor): if isinstance(tensor, self.__type__): return int(self.max(tensor)) + 1 else: return int(np.max(tensor)) + 1 def coo_matrix(self, data, rows, cols, shape=None, type_as=None): if shape is None: shape = ( self._convert_to_index_for_coo(rows), self._convert_to_index_for_coo(cols) ) if type_as is not None: data = self.from_numpy(data, type_as=type_as) sparse_tensor = tf.sparse.SparseTensor( indices=tnp.stack([rows, cols]).T, values=data, dense_shape=shape ) # if type_as is not None: # sparse_tensor = self.from_numpy(sparse_tensor, type_as=type_as) # SparseTensor are not subscriptable so we use dense tensors return self.todense(sparse_tensor) def issparse(self, a): return isinstance(a, tf.sparse.SparseTensor) def tocsr(self, a): return a def eliminate_zeros(self, a, threshold=0.): if self.issparse(a): values = a.values if threshold > 0: mask = self.abs(values) <= threshold else: mask = values == 0 return tf.sparse.retain(a, ~mask) else: if threshold > 0: a = tnp.where(self.abs(a) > threshold, a, 0.) return a def todense(self, a): if self.issparse(a): return tf.sparse.to_dense(tf.sparse.reorder(a)) else: return a def where(self, condition, x=None, y=None): if x is None and y is None: return tnp.where(condition) else: return tnp.where(condition, x, y) def copy(self, a): return tf.identity(a) def allclose(self, a, b, rtol=1e-05, atol=1e-08, equal_nan=False): return tnp.allclose( a, b, rtol=rtol, atol=atol, equal_nan=equal_nan ) def dtype_device(self, a): return a.dtype, a.device.split("device:")[1] def assert_same_dtype_device(self, a, b): a_dtype, a_device = self.dtype_device(a) b_dtype, b_device = self.dtype_device(b) assert a_dtype == b_dtype, "Dtype discrepancy" assert a_device == b_device, f"Device discrepancy. First input is on {str(a_device)}, whereas second input is on {str(b_device)}" def squeeze(self, a, axis=None): return tnp.squeeze(a, axis=axis) def bitsize(self, type_as): return type_as.dtype.size * 8 def device_type(self, type_as): return self.dtype_device(type_as)[1].split(":")[0] def _bench(self, callable, *args, n_runs=1, warmup_runs=1): results = dict() device_contexts = [tf.device("/CPU:0")] if len(tf.config.list_physical_devices('GPU')) > 0: # pragma: no cover device_contexts.append(tf.device("/GPU:0")) for device_context in device_contexts: with device_context: for type_as in self.__type_list__: inputs = [self.from_numpy(arg, type_as=type_as) for arg in args] for _ in range(warmup_runs): callable(*inputs) t0 = time.perf_counter() for _ in range(n_runs): res = callable(*inputs) _ = res.numpy() t1 = time.perf_counter() key = ( "Tensorflow", self.device_type(inputs[0]), self.bitsize(type_as) ) results[key] = (t1 - t0) / n_runs return results def solve(self, a, b): return tf.linalg.solve(a, b) def trace(self, a): return tf.linalg.trace(a) def inv(self, a): return tf.linalg.inv(a) def sqrtm(self, a): return tf.linalg.sqrtm(a) def kl_div(self, p, q, eps=1e-16): return tnp.sum(p * tnp.log(p / q + eps)) def isfinite(self, a): return tnp.isfinite(a) def array_equal(self, a, b): return tnp.array_equal(a, b) def is_floating_point(self, a): return a.dtype.is_floating def tile(self, a, reps): return tnp.tile(a, reps) def floor(self, a): return tf.floor(a) def prod(self, a, axis=0): return tnp.prod(a, axis=axis) def sort2(self, a, axis=-1): return self.sort(a, axis), self.argsort(a, axis) def qr(self, a): return tf.linalg.qr(a) def atan2(self, a, b): return tf.math.atan2(a, b) def transpose(self, a, axes=None): return tf.transpose(a, perm=axes)