diff --git a/CMakeLists.txt b/CMakeLists.txt index 28602c7d..03c19c98 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -120,7 +120,7 @@ else() include(FetchContent) FetchContent_Declare(blosc2 GIT_REPOSITORY https://github.com/Blosc/c-blosc2 - GIT_TAG cdc78596270c1e235d29436d3e730f0f403ddca9 # fix resize + GIT_TAG f27bb87c51443e237dab4c68d445480b65ae7688 # malloc(0) -> NULL ) FetchContent_MakeAvailable(blosc2) include_directories("${blosc2_SOURCE_DIR}/include") diff --git a/bench/ndarray/stringops_bench.py b/bench/ndarray/stringops_bench.py new file mode 100644 index 00000000..c983a0f2 --- /dev/null +++ b/bench/ndarray/stringops_bench.py @@ -0,0 +1,47 @@ +####################################################################### +# Copyright (c) 2019-present, Blosc Development Team +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +####################################################################### + +""" +Compare miniexpr and non-miniexpr paths for string ops. +""" + +import time +import numpy as np +import blosc2 +from blosc2.lazyexpr import _toggle_miniexpr + +# nparr = np.random.randint(low=0, high=128, size=(N, 10), dtype=np.uint32) +# nparr = nparr.view('S40').astype('U10') + +N = int(1e5) +nparr = np.repeat(np.array(['josé', 'pepe', 'francisco']), N) +cparams = blosc2.cparams_dflts +cparams["filters"][-1] = blosc2.Filter.SHUFFLE +cparams["filters_meta"][-1] = 0 # use default (typesize) +arr1 = blosc2.asarray(nparr) +print(f"cratio without filter: {arr1.cratio}") +cparams["filters_meta"][-1] = 4 +arr1 = blosc2.asarray(nparr, cparams=cparams) +print(f"cratio with filter: {arr1.cratio}") + +arr2 = blosc2.full(arr1.shape, 'francisco', blocks=arr1.blocks, chunks=arr1.chunks) + +names = ['==', 'contains', 'startswith', 'endswith'] +functuple = (lambda a, b : a==b, blosc2.contains, blosc2.startswith, blosc2.endswith) +for name, func in zip(names, functuple): + expr = func(arr1, arr2) + dtic = time.time() + res = expr[()] + dtoc = time.time() + print(f'{name} took {round(dtoc-dtic, 3)}s for miniexpr') + _toggle_miniexpr(False) + expr = func(arr1, arr2) + dtic = time.time() + res = expr[()] + dtoc = time.time() + print(f'{name} took {round(dtoc-dtic, 3)}s for normal fast path') + _toggle_miniexpr(True) diff --git a/doc/reference/index_funcs.rst b/doc/reference/index_funcs.rst index 78a8165c..3149af91 100644 --- a/doc/reference/index_funcs.rst +++ b/doc/reference/index_funcs.rst @@ -8,27 +8,23 @@ The following functions are useful for performing indexing and other associated .. autosummary:: broadcast_to - concat count_nonzero expand_dims indices meshgrid sort squeeze - stack take take_along_axis .. autofunction:: blosc2.broadcast_to -.. autofunction:: blosc2.concat .. autofunction:: blosc2.count_nonzero .. autofunction:: blosc2.expand_dims .. autofunction:: blosc2.indices .. autofunction:: blosc2.meshgrid .. autofunction:: blosc2.sort .. autofunction:: blosc2.squeeze -.. autofunction:: blosc2.stack .. autofunction:: blosc2.take .. autofunction:: blosc2.take_along_axis diff --git a/src/blosc2/__init__.py b/src/blosc2/__init__.py index 8d9a68b8..e2579d97 100644 --- a/src/blosc2/__init__.py +++ b/src/blosc2/__init__.py @@ -105,6 +105,21 @@ class Codec(Enum): class Filter(Enum): """ Available filters. + For each of the filters, the integer value passed to ``filters_meta`` has the following meaning: + + - NOFILTER: Not used + - SHUFFLE: Number of byte streams for shuffle (if 0 defaults to typesize of array). + - BITSHUFFLE: Not used + - DELTA: Not used (bitwise XOR) + - TRUNC_PREC: Number of bits to which to truncate float + - NDCELL: Cellshape (i.e. for a 3-dim dataset, meta = 4 implies cellshape is 4x4x4) + - NDMEAN: Cellshape (i.e. for a 3-dim dataset, meta = 4 implies cellshape is 4x4x4) + - BYTEDELTA: Number of byte streams for delta + - INT_TRUNC: Number of bits to which to truncate integer + + For TRUNC_PREC and INT_TRUNC, positive values specify number of bits to keep; negative values specify number of bits to zero. + + For NDCELL/NDMEAN see this explanation for `NDCELL `_ and this for `NDMEAN `_. """ NOFILTER = 0 @@ -598,6 +613,7 @@ def _raise(exc): cumulative_prod, cumulative_sum, divide, + endswith, equal, exp, expm1, @@ -645,6 +661,7 @@ def _raise(exc): sqrt, square, squeeze, + startswith, std, subtract, sum, @@ -769,6 +786,7 @@ def _raise(exc): "detect_number_of_cores", "divide", "dparams_dflts", + "endswith", "empty", "empty_like", "equal", @@ -877,6 +895,7 @@ def _raise(exc): "square", "squeeze", "stack", + "startswith", "std", "storage_dflts", "subtract", diff --git a/src/blosc2/blosc2_ext.pyx b/src/blosc2/blosc2_ext.pyx index 80c8ab5d..e8c028f9 100644 --- a/src/blosc2/blosc2_ext.pyx +++ b/src/blosc2/blosc2_ext.pyx @@ -571,9 +571,6 @@ cdef extern from "miniexpr.h": int ncode void *parameters[1] - int me_compile(const char *expression, const me_variable *variables, - int var_count, me_dtype dtype, int *error, me_expr **out) - int me_compile_nd_jit(const char *expression, const me_variable *variables, int var_count, me_dtype dtype, int ndims, const int64_t *shape, const int32_t *chunkshape, @@ -1436,7 +1433,7 @@ cdef class SChunk: return dst if size < 0: - raise RuntimeError("Error while decompressing the specified chunk") + raise RuntimeError(f"Error while decompressing the specified chunk, error code: {size}") def get_chunk(self, nchunk): cdef uint8_t *chunk @@ -3004,6 +3001,7 @@ cdef class NDArray: var.address = NULL # chunked compile: addresses provided later var.type = 0 # auto-set to ME_VARIABLE inside compiler var.context = NULL + var.itemsize = v.dtype.itemsize if v.dtype.num == 19 else 0 # only store item type if string cdef int error = 0 expression = expression.encode("utf-8") if isinstance(expression, str) else expression @@ -3124,7 +3122,10 @@ cdef b2nd_context_t* create_b2nd_context(shape, chunks, blocks, dtype, kwargs): if 'cparams' in kwargs: kwargs['cparams']['typesize'] = typesize else: - kwargs['cparams'] = {'typesize': typesize} + kwargs['cparams'] = {'typesize': typesize} # last filter is shuffle + if isinstance(dtype, np.dtypes.StrDType) or dtype == np.str_: + kwargs['cparams']['filters'] = [blosc2.Filter.NOFILTER] * 5 + [blosc2.Filter.SHUFFLE] + kwargs['cparams']['filters_meta'] = [0] * 5 + [4] # unicode char bytesize if dtype.kind == 'V': str_dtype = str(dtype) else: diff --git a/src/blosc2/core.py b/src/blosc2/core.py index 95709449..95028c1f 100644 --- a/src/blosc2/core.py +++ b/src/blosc2/core.py @@ -1576,7 +1576,7 @@ def compute_chunks_blocks( # noqa: C901 raise ValueError("blocks cannot be greater than chunks") return chunks, blocks - cparams = kwargs.get("cparams") or copy.deepcopy(blosc2.cparams_dflts) + cparams = kwargs.get("cparams") or blosc2.CParams() # just get defaults if isinstance(cparams, blosc2.CParams): cparams = asdict(cparams) # Typesize in dtype always has preference over typesize in cparams diff --git a/src/blosc2/lazyexpr.py b/src/blosc2/lazyexpr.py index 7ec0f5f3..2bf446df 100644 --- a/src/blosc2/lazyexpr.py +++ b/src/blosc2/lazyexpr.py @@ -51,7 +51,6 @@ from .proxy import _convert_dtype from .utils import ( - NUMPY_GE_2_0, _get_chunk_operands, _sliced_chunk_iter, check_smaller_shape, @@ -65,9 +64,9 @@ linalg_funcs, npcumprod, npcumsum, - npvecdot, process_key, reducers, + safe_numpy_globals, ) if not blosc2.IS_WASM: @@ -75,32 +74,6 @@ global safe_blosc2_globals safe_blosc2_globals = {} -global safe_numpy_globals -# Use numpy eval when running in WebAssembly -safe_numpy_globals = {"np": np} -# Add all first-level numpy functions -safe_numpy_globals.update( - {name: getattr(np, name) for name in dir(np) if callable(getattr(np, name)) and not name.startswith("_")} -) - -if not NUMPY_GE_2_0: # handle non-array-api compliance - safe_numpy_globals["acos"] = np.arccos - safe_numpy_globals["acosh"] = np.arccosh - safe_numpy_globals["asin"] = np.arcsin - safe_numpy_globals["asinh"] = np.arcsinh - safe_numpy_globals["atan"] = np.arctan - safe_numpy_globals["atanh"] = np.arctanh - safe_numpy_globals["atan2"] = np.arctan2 - safe_numpy_globals["permute_dims"] = np.transpose - safe_numpy_globals["pow"] = np.power - safe_numpy_globals["bitwise_left_shift"] = np.left_shift - safe_numpy_globals["bitwise_right_shift"] = np.right_shift - safe_numpy_globals["bitwise_invert"] = np.bitwise_not - safe_numpy_globals["concat"] = np.concatenate - safe_numpy_globals["matrix_transpose"] = np.transpose - safe_numpy_globals["vecdot"] = npvecdot - safe_numpy_globals["cumulative_sum"] = npcumsum - safe_numpy_globals["cumulative_prod"] = npcumprod # Set this to False if miniexpr should not be tried out try_miniexpr = True @@ -108,6 +81,11 @@ try_miniexpr = False +def _toggle_miniexpr(FLAG): + global try_miniexpr + try_miniexpr = FLAG + + def ne_evaluate(expression, local_dict=None, **kwargs): """Safely evaluate expressions using numexpr when possible, falling back to numpy.""" if local_dict is None: @@ -134,28 +112,33 @@ def ne_evaluate(expression, local_dict=None, **kwargs): try: return numexpr.evaluate(expression, local_dict=local_dict, **kwargs) except ValueError as e: - raise e # unsafe expression - except Exception: # non_numexpr functions present - global safe_blosc2_globals - # ne_evaluate will need safe_blosc2_globals for some functions (e.g. clip, logaddexp) - # that are implemented in python-blosc2 not in numexpr - if len(safe_blosc2_globals) == 0: - # First eval call, fill blosc2_safe_globals for ne_evaluate - safe_blosc2_globals = {"blosc2": blosc2} - # Add all first-level blosc2 functions - safe_blosc2_globals.update( - { - name: getattr(blosc2, name) - for name in dir(blosc2) - if callable(getattr(blosc2, name)) and not name.startswith("_") - } - ) - res = eval(expression, safe_blosc2_globals, local_dict) - if "out" in kwargs: - out = kwargs.pop("out") - out[:] = res # will handle calc/decomp if res is lazyarray - return out - return res[()] if isinstance(res, blosc2.Operand) else res + if e.args and e.args[0] == "NumExpr 2 does not support Unicode as a dtype.": + pass + else: + raise e # unsafe expression + except Exception: + pass + # Try with blosc2 funcs as presence of non-numexpr funcs probably caused failure + # ne_evaluate will need safe_blosc2_globals for some functions (e.g. clip, logaddexp, + # startswith, matmul) that are implemented incompletely in numexpr/miniexpr or not implemented at all + global safe_blosc2_globals + if len(safe_blosc2_globals) == 0: + # First eval call, fill blosc2_safe_globals + safe_blosc2_globals = {"blosc2": blosc2} + # Add all first-level blosc2 functions + safe_blosc2_globals.update( + { + name: getattr(blosc2, name) + for name in dir(blosc2) + if callable(getattr(blosc2, name)) and not name.startswith("_") + } + ) + res = eval(expression, safe_blosc2_globals, local_dict) + if "out" in kwargs: + out = kwargs.pop("out") + out[:] = res # will handle calc/decomp if res is lazyarray + return out + return res[()] if isinstance(res, blosc2.Operand) else res def _get_result(expression, chunk_operands, ne_args, where=None, indices=None, _order=None): @@ -271,6 +254,8 @@ def _find_constructor_call(expression: str, constructor: str) -> re.Match | None "hypot", "maximum", "minimum", + "startswith", + "endswith", ) @@ -2801,7 +2786,7 @@ def check_dtype(op, value1, value2): def result_type( - *arrays_and_dtypes: blosc2.NDArray | int | float | complex | bool | blosc2.dtype, + *arrays_and_dtypes: blosc2.NDArray | int | float | complex | bool | str | blosc2.dtype, ) -> blosc2.dtype: """ Returns the dtype that results from applying type promotion rules (see Type Promotion Rules) to the arguments. @@ -2819,7 +2804,7 @@ def result_type( # Follow NumPy rules for scalar-array operations # Create small arrays with the same dtypes and let NumPy's type promotion determine the result type arrs = [ - value + (np.array(value).dtype if isinstance(value, str) else value) if (np.isscalar(value) or not hasattr(value, "dtype")) else np.array([0], dtype=_convert_dtype(value.dtype)) for value in arrays_and_dtypes @@ -2869,6 +2854,8 @@ def __init__(self, new_op): # noqa: C901 if not (isinstance(value1, (blosc2.Operand, np.ndarray)) or np.isscalar(value1)) else value1 ) + # Reset values represented as np.int64 etc. to be set as Python natives + value1 = value1.item() if np.isscalar(value1) and hasattr(value1, "item") else value1 if value2 is None: if isinstance(value1, LazyExpr): self.expression = value1.expression if op is None else f"{op}({value1.expression})" @@ -2881,7 +2868,7 @@ def __init__(self, new_op): # noqa: C901 self.operands = value1.operands else: if np.isscalar(value1): - value1 = ne_evaluate(f"{op}({value1})") + value1 = ne_evaluate(f"{op}({value1!r})") op = None self.operands = {"o0": value1} self.expression = "o0" if op is None else f"{op}(o0)" @@ -2891,6 +2878,9 @@ def __init__(self, new_op): # noqa: C901 if not (isinstance(value2, (blosc2.Operand, np.ndarray)) or np.isscalar(value2)) else value2 ) + # Reset values represented as np.int64 etc. to be set as Python natives + value2 = value2.item() if np.isscalar(value2) and hasattr(value2, "item") else value2 + if isinstance(value1, LazyExpr) or isinstance(value2, LazyExpr): if isinstance(value1, LazyExpr): newexpr = value1.update_expr(new_op) @@ -2903,13 +2893,13 @@ def __init__(self, new_op): # noqa: C901 elif op in funcs_2args: if np.isscalar(value1) and np.isscalar(value2): self.expression = "o0" - self.operands = {"o0": ne_evaluate(f"{op}({value1}, {value2})")} # eager evaluation + self.operands = {"o0": ne_evaluate(f"{op}({value1!r}, {value2!r})")} # eager evaluation elif np.isscalar(value2): self.operands = {"o0": value1} - self.expression = f"{op}(o0, {value2})" + self.expression = f"{op}(o0, {value2!r})" elif np.isscalar(value1): self.operands = {"o0": value2} - self.expression = f"{op}({value1}, o0)" + self.expression = f"{op}({value1!r}, o0)" else: self.operands = {"o0": value1, "o1": value2} self.expression = f"{op}(o0, o1)" @@ -2918,16 +2908,16 @@ def __init__(self, new_op): # noqa: C901 self._dtype = dtype_ if np.isscalar(value1) and np.isscalar(value2): self.expression = "o0" - self.operands = {"o0": ne_evaluate(f"({value1} {op} {value2})")} # eager evaluation + self.operands = {"o0": ne_evaluate(f"({value1!r} {op} {value2!r})")} # eager evaluation elif np.isscalar(value2): self.operands = {"o0": value1} - self.expression = f"(o0 {op} {value2})" + self.expression = f"(o0 {op} {value2!r})" elif hasattr(value2, "shape") and value2.shape == (): self.operands = {"o0": value1} self.expression = f"(o0 {op} {value2[()]})" elif np.isscalar(value1): self.operands = {"o0": value2} - self.expression = f"({value1} {op} o0)" + self.expression = f"({value1!r} {op} o0)" elif hasattr(value1, "shape") and value1.shape == (): self.operands = {"o0": value2} self.expression = f"({value1[()]} {op} o0)" @@ -4095,7 +4085,7 @@ def _numpy_eval_expr(expression, operands, prefer_blosc=False): for key, value in operands.items() } - if "contains" in expression: + if np.any([a in expression for a in ["contains", "startswith", "endswith"]]): _out = ne_evaluate(expression, local_dict=ops) else: # Create a globals dict with blosc2 version of functions preferentially diff --git a/src/blosc2/ndarray.py b/src/blosc2/ndarray.py index be7ab746..b9b5f15e 100644 --- a/src/blosc2/ndarray.py +++ b/src/blosc2/ndarray.py @@ -36,6 +36,7 @@ from .utils import ( _get_local_slice, _get_selection, + _incomplete_lazyfunc, get_chunks_idx, npbinvert, nplshift, @@ -1833,15 +1834,16 @@ def imag(ndarr: blosc2.Array, /) -> blosc2.LazyExpr: return blosc2.LazyExpr(new_op=(ndarr, "imag", None)) +@_incomplete_lazyfunc def contains(ndarr: blosc2.Array, value: str | bytes | blosc2.Array, /) -> blosc2.LazyExpr: """ Check if the array contains a specified value. Parameters ---------- - ndarr: :ref:`NDArray` or :ref:`NDField` or :ref:`C2Array` + ndarr: :ref:`Array` The input array. - value: str or bytes or :ref:`NDArray` or :ref:`NDField` or :ref:`C2Array` + value: str or bytes or :ref:`Array` The value to be checked. Returns @@ -1862,8 +1864,14 @@ def contains(ndarr: blosc2.Array, value: str | bytes | blosc2.Array, /) -> blosc >>> print("Contains 'banana':", result[:]) Contains 'banana': [False True False False] """ + # def chunkwise_contains(inputs, output, offset): + # x1, x2 = inputs + # # output[...] = np.isin(x1, x2, assume_unique=assume_unique, invert=invert, kind=kind) + # output[...] = np.char.find(x1, x2) != -1 + if not isinstance(value, str | bytes | NDArray): raise TypeError("value should be a string, bytes or a NDArray!") + return blosc2.LazyExpr(new_op=(ndarr, "contains", value)) @@ -3052,6 +3060,7 @@ def remainder( return blosc2.LazyExpr(new_op=(x1, "%", x2)) +@_incomplete_lazyfunc def clip( x: blosc2.Array, min: int | float | blosc2.Array | None = None, @@ -3086,12 +3095,14 @@ def clip( def chunkwise_clip(inputs, output, offset): x, min, max = inputs - output[:] = np.clip(x, min, max) + output[...] = np.clip(x, min, max) dtype = blosc2.result_type(x) - return blosc2.lazyudf(chunkwise_clip, (x, min, max), dtype=dtype, shape=x.shape, **kwargs) + shape = () if np.isscalar(x) else None + return blosc2.lazyudf(chunkwise_clip, (x, min, max), dtype=dtype, shape=shape, **kwargs) +@_incomplete_lazyfunc def logaddexp(x1: int | float | blosc2.Array, x2: int | float | blosc2.Array, **kwargs: Any) -> NDArray: """ Calculates the logarithm of the sum of exponentiations log(exp(x1) + exp(x2)) for @@ -3119,7 +3130,7 @@ def logaddexp(x1: int | float | blosc2.Array, x2: int | float | blosc2.Array, ** def chunkwise_logaddexp(inputs, output, offset): x1, x2 = inputs - output[:] = np.logaddexp(x1, x2) + output[...] = np.logaddexp(x1, x2) dtype = blosc2.result_type(x1, x2) if dtype == blosc2.bool_: @@ -3127,7 +3138,8 @@ def chunkwise_logaddexp(inputs, output, offset): if np.issubdtype(dtype, np.integer): dtype = blosc2.float32 - return blosc2.lazyudf(chunkwise_logaddexp, (x1, x2), dtype=dtype, shape=x1.shape, **kwargs) + shape = () if np.isscalar(x1) and np.isscalar(x2) else None + return blosc2.lazyudf(chunkwise_logaddexp, (x1, x2), dtype=dtype, shape=shape, **kwargs) # implemented in python-blosc2 @@ -4957,6 +4969,85 @@ def where( return condition.where(x, y) +@_incomplete_lazyfunc +def startswith( + a: str | blosc2.Array, prefix: str | blosc2.Array +) -> NDArray: # start: int = 0, end: int | None = None, **kwargs) + """ + Copy-pasted from numpy documentation: https://numpy.org/doc/stable/reference/generated/numpy.char.startswith.html + Returns a boolean array which is True where the string element in a starts with prefix, otherwise False. + + Parameters + ---------- + a : blosc2.Array + Input array of bytes_ or str_ dtype + + prefix : blosc2.Array + Prefix array of bytes_ or str_ dtype + + start: int | blosc2.Array + With start, test beginning at that position. + + end: int | blosc2.Array + With end, stop comparing at that position. + + kwargs: Any + kwargs accepted by the :func:`empty` constructor + + Returns + ------- + out: blosc2.Array, bool + Has the same shape as element. + + """ + + # def chunkwise_startswith(inputs, output, offset): + # x1, x2 = inputs + # # output[...] = np.char.startswith(x1, x2, start=start, end=end) + # output[...] = np.char.startswith(x1, x2) + + return blosc2.LazyExpr(new_op=(a, "startswith", prefix)) + + +@_incomplete_lazyfunc +def endswith( + a: str | blosc2.Array, suffix: str | blosc2.Array +) -> NDArray: # start: int = 0, end: int | None = None, **kwargs) -> NDArray: + """ + Copy-pasted from numpy documentation: https://numpy.org/doc/stable/reference/generated/numpy.char.endswith.html + Returns a boolean array which is True where the string element in a ends with suffix, otherwise False. + + Parameters + ---------- + a : blosc2.Array + Input array of bytes_ or str_ dtype + + suffix : blosc2.Array + suffix array of bytes_ or str_ dtype + + start: int | blosc2.Array + With start, test beginning at that position. + + end: int | blosc2.Array + With end, stop comparing at that position. + + kwargs: Any + kwargs accepted by the :func:`empty` constructor + + Returns + ------- + out: blosc2.Array, bool + Has the same shape as element. + + """ + # def chunkwise_endswith(inputs, output, offset): + # x1, x2 = inputs + # # output[...] = np.char.endswith(x1, x2, start=start, end=end) + # output[...] = np.char.endswith(x1, x2) + + return blosc2.LazyExpr(new_op=(a, "endswith", suffix)) + + def lazywhere(value1=None, value2=None): """Decorator to apply a where condition to a LazyExpr.""" @@ -4983,6 +5074,8 @@ def _check_dtype(dtype): dtype = np.dtype(dtype) if dtype.itemsize > blosc2.MAX_TYPESIZE: raise ValueError(f"dtype itemsize {dtype.itemsize} is too large (>{blosc2.MAX_TYPESIZE})!") + if dtype == np.str_: # itemsize is 0 + dtype = np.dtype("...", np.moveaxis(np.conj(a), axis, -1), np.moveaxis(b, axis, -1)) +global safe_numpy_globals +# Use numpy eval when running in WebAssembly +safe_numpy_globals = {"np": np} +# Add all first-level numpy functions +safe_numpy_globals.update( + {name: getattr(np, name) for name in dir(np) if callable(getattr(np, name)) and not name.startswith("_")} +) + +if not NUMPY_GE_2_0: # handle non-array-api compliance + safe_numpy_globals["acos"] = np.arccos + safe_numpy_globals["acosh"] = np.arccosh + safe_numpy_globals["asin"] = np.arcsin + safe_numpy_globals["asinh"] = np.arcsinh + safe_numpy_globals["atan"] = np.arctan + safe_numpy_globals["atanh"] = np.arctanh + safe_numpy_globals["atan2"] = np.arctan2 + safe_numpy_globals["permute_dims"] = np.transpose + safe_numpy_globals["pow"] = np.power + safe_numpy_globals["bitwise_left_shift"] = np.left_shift + safe_numpy_globals["bitwise_right_shift"] = np.right_shift + safe_numpy_globals["bitwise_invert"] = np.bitwise_not + safe_numpy_globals["concat"] = np.concatenate + safe_numpy_globals["matrix_transpose"] = np.transpose + safe_numpy_globals["vecdot"] = npvecdot + safe_numpy_globals["cumulative_sum"] = npcumsum + safe_numpy_globals["cumulative_prod"] = npcumprod + # handle different naming conventions between numpy and blosc2 + safe_numpy_globals["contains"] = lambda *args: np.char.find(*args) != -1 + safe_numpy_globals["startswith"] = np.char.startswith + safe_numpy_globals["endswith"] = np.char.endswith + + elementwise_funcs = [ "abs", "acos", @@ -77,6 +110,7 @@ def npvecdot(a, b, axis=-1): "cos", "cosh", "divide", + "endswith", "equal", "exp", "expm1", @@ -118,6 +152,7 @@ def npvecdot(a, b, axis=-1): "sinh", "sqrt", "square", + "startswith", "subtract", "tan", "tanh", @@ -875,6 +910,49 @@ def process_key(key, shape): return key, mask +incomplete_lazyfunc_map = { + "contains": lambda *args: np.char.find(*args) != -1, + "startswith": lambda *args: np.char.startswith(*args), + "endswith": lambda *args: np.char.endswith(*args), +} | safe_numpy_globals # clip and logaddexp available in safe_numpy_globals + + +def is_inside_ne_evaluate() -> bool: + """ + Whether the current code is being executed from an ne_evaluate call + """ + # Get the current call stack + stack = inspect.stack() + return builtins.any(frame_info.function in {"ne_evaluate"} for frame_info in stack) + + +def _incomplete_lazyfunc(func) -> None: + """Decorator for lazy functions with incomplete numexpr/miniexpr coverage. + + This function will force eager execution when called from ne_evaluate. + + Returns + ------- + out: None + + Examples + -------- + .. code-block:: python + + @incomplete_lazyfunc() + def filler(inputs_tuple, output, offset): + output[:] = inputs_tuple[0] - inputs_tuple[1] + + """ + + def wrapper(*args, **kwargs): + if is_inside_ne_evaluate(): # haven't been able to use miniexpr so use numpy + return incomplete_lazyfunc_map[func.__name__](*args, **kwargs) + return func(*args, **kwargs) + + return wrapper + + def check_smaller_shape(value_shape, shape, slice_shape, slice_): """Check whether the shape of the value is smaller than the shape of the array. diff --git a/tests/ndarray/test_auto_parts.py b/tests/ndarray/test_auto_parts.py index c38939e5..97d9de44 100644 --- a/tests/ndarray/test_auto_parts.py +++ b/tests/ndarray/test_auto_parts.py @@ -5,6 +5,7 @@ # SPDX-License-Identifier: BSD-3-Clause ####################################################################### + import numpy as np import pytest @@ -45,6 +46,7 @@ def test_compute_chunks_blocks(clevel, codec, shape: tuple, dtype): for dim, chunk, block in zip(shape, chunks, blocks, strict=True): assert dim >= chunk assert chunk >= block + assert blosc2.cparams_dflts["clevel"] == blosc2.CParams().clevel # check haven't edited defaults @pytest.mark.parametrize( diff --git a/tests/ndarray/test_full.py b/tests/ndarray/test_full.py index 829a3823..7b9df94c 100644 --- a/tests/ndarray/test_full.py +++ b/tests/ndarray/test_full.py @@ -38,6 +38,17 @@ "full.b2nd", True, ), + ( + (23, 34), + (20, 20), + (10, 10), + "josé", + None, + blosc2.CParams(codec=blosc2.Codec.LZ4HC, clevel=8, use_dict=False, nthreads=2), + {"nthreads": 2}, + "full.b2nd", + True, + ), ( (80, 51, 60), (20, 10, 33), @@ -139,6 +150,7 @@ def test_full(shape, chunks, blocks, fill_value, cparams, dparams, dtype, urlpat [ ((100, 1230), b"0123", None), ((23, 34), b"sun", None), + ((23, 34), "josé", None), ((80, 51, 60), 3.14, "f8"), ((13, 13), 123456789, None), ], @@ -205,6 +217,7 @@ def test_complex_datatype(): ("f_019", " cratio_subopt + assert blosc2.cparams_dflts["typesize"] == blosc2.CParams().typesize # check haven't modified + + +@pytest.mark.parametrize("shape", SHAPES) +def test_frombuffer(shape): + chunks = tuple(max(c // 4, 1) for c in shape) + dtype = np.dtype("