Merge commit '8e226b0f250c70c44a0cb5a958bb7bd5e115e6b3' into updates

src/c-blosc2/ANNOUNCE.md

@@ -1,11 +1,12 @@
-# Announcing C-Blosc2 2.15.0
+# Announcing C-Blosc2 2.15.1
 A fast, compressed and persistent binary data store library for C.
 
 ## What is new?
 
-This is a minor release in which a new io mode
-was added to memory-map files. Furthermore, the `io_cb` read API was changed, so the 
-`SOVERSION` was bumped. In addition, the internal zstd sources were updated to 1.5.6 and some
+This is a maintenance release in which we are fixing calling instructions
+more advanced than available in current CPU, causing a SIGKILL.
+Furthermore, a new `b2nd_nans` function has been added. In addition,
+the internal LZ4 sources were updated to 1.10.0 and some
 other improvements were made.
 
 For more info, please see the release notes in:

src/c-blosc2/README_B2ND_METALAYER.rst

@@ -1,5 +1,5 @@
 b2nd metalayer
-++++++++++++++
+==============
 
 b2nd format is specified as a metalayer on top of a Blosc2 container for storing
 multidimensional information.  Specifically, this metalayer is named 'b2nd'

src/c-blosc2/README_CHUNK_FORMAT.rst

@@ -8,7 +8,7 @@ A regular chunk is composed of a header and a blocks section::
     +---------+--------+
 
 Also, there are the so-called lazy chunks that do not have the actual compressed data,
-but only metainformation about how to read it. Lazy chunks typically appear when reading
+but only meta-information about how to read it. Lazy chunks typically appear when reading
 data from persistent media.  A lazy chunk has header and bstarts sections in place and
 in addition, an additional trailer for allowing to read the data blocks::
 

src/c-blosc2/README_EXTENSION_FILENAMES.rst

@@ -0,0 +1,8 @@
+Extensions for Blosc2 filenames
+===============================
+
+Blosc2 has different file extensions for different purposes.  Here is a list of the currently supported ones:
+
+- `.b2frame` (but also `.b2f` or `.b2`) (Blosc2 frame): this is the main extension for storing `Blosc2 contiguous frames <README_CFRAME_FORMAT.rst>`_.
+
+- `.b2nd` (Blosc2 n-dimensional): this is just a contiguous frame file with `a metalayer for storing n-dimensional information <README_B2ND_METALAYER.rst>`_ like shape, chunkshape, blockshape and dtype.

src/c-blosc2/RELEASE_NOTES.md

@@ -1,6 +1,22 @@
 Release notes for C-Blosc2
 ==========================
 
+Changes from 2.15.0 to 2.15.1
+=============================
+
+* Do not pass `-m` flags when compiling `shuffle.c`. This prevents the
+  compiler from incidentally optimizing the code called independently
+  of the runtime CPU check to these instruction sets, effectively
+  causing `SIGILL` on other CPUs.  Fixes #621.  Thanks to @t20100 and @mgorny.
+
+* Internal LZ4 sources bumped to 1.10.0.
+
+* Allow direct loading of plugins by name, without relying on
+  the presence of python. Thanks to @boxerab.
+
+* Add `b2nd_nans` method (PR #624). Thanks to @waynegm.
+
+
 Changes from 2.14.4 to 2.15.0
 =============================
 

src/c-blosc2/blosc/CMakeLists.txt

@@ -90,7 +90,7 @@ if(LZ4_FOUND)
         target_include_directories(blosc_testing PUBLIC ${LZ4_INCLUDE_DIR})
     endif()
 else()
-    set(LZ4_LOCAL_DIR ${INTERNAL_LIBS}/lz4-1.9.4)
+    set(LZ4_LOCAL_DIR ${INTERNAL_LIBS}/lz4-1.10.0)
     if(BUILD_SHARED)
         target_include_directories(blosc2_shared PRIVATE ${LZ4_LOCAL_DIR})
     endif()
@@ -300,17 +300,11 @@ if(COMPILER_SUPPORT_SSE2)
             set_source_files_properties(
                     shuffle-sse2.c bitshuffle-sse2.c blosclz.c fastcopy.c
                     PROPERTIES COMPILE_OPTIONS "/arch:SSE2")
-            set_property(
-                    SOURCE shuffle.c
-                    APPEND PROPERTY COMPILE_OPTIONS "/arch:SSE2")
         endif()
     else()
         set_source_files_properties(
                 shuffle-sse2.c bitshuffle-sse2.c blosclz.c fastcopy.c
                 PROPERTIES COMPILE_OPTIONS -msse2)
-        set_property(
-                SOURCE shuffle.c
-                APPEND PROPERTY COMPILE_OPTIONS -msse2)
         # Add SIMD flags for the bytedelta filter and Intel (it seems that ARM64 does not need these)
         set_source_files_properties(
                 ${PROJECT_SOURCE_DIR}/plugins/filters/bytedelta/bytedelta.c
@@ -330,16 +324,10 @@ if(COMPILER_SUPPORT_AVX2)
         set_source_files_properties(
                 shuffle-avx2.c bitshuffle-avx2.c
                 PROPERTIES COMPILE_OPTIONS "/arch:AVX2")
-        set_property(
-                SOURCE shuffle.c
-                APPEND PROPERTY COMPILE_OPTIONS "/arch:AVX2")
     else()
         set_source_files_properties(
                 shuffle-avx2.c bitshuffle-avx2.c
                 PROPERTIES COMPILE_OPTIONS -mavx2)
-        set_property(
-                SOURCE shuffle.c
-                APPEND PROPERTY COMPILE_OPTIONS -mavx2)
     endif()
 
     # Define a symbol for the shuffle-dispatch implementation
@@ -354,16 +342,10 @@ if(COMPILER_SUPPORT_AVX512)
         set_source_files_properties(
                 bitshuffle-avx512.c
 		PROPERTIES COMPILE_OPTIONS "/arch:AVX512")
-        set_property(
-                SOURCE shuffle.c
-		APPEND PROPERTY COMPILE_OPTIONS "/arch:AVX512")
     else()
         set_source_files_properties(
                 bitshuffle-avx512.c
                 PROPERTIES COMPILE_OPTIONS "-mavx512f;-mavx512bw")
-        set_property(
-                SOURCE shuffle.c
-                APPEND PROPERTY COMPILE_OPTIONS "-mavx512f;-mavx512bw")
     endif()
 
     # Define a symbol for the shuffle-dispatch implementation
@@ -377,17 +359,11 @@ if(COMPILER_SUPPORT_NEON)
     set_source_files_properties(
             shuffle-neon.c bitshuffle-neon.c
             PROPERTIES COMPILE_OPTIONS "-flax-vector-conversions")
-    set_property(
-            SOURCE shuffle.c
-            APPEND PROPERTY COMPILE_OPTIONS "-flax-vector-conversions")
     if(CMAKE_SYSTEM_PROCESSOR STREQUAL armv7l)
         # Only armv7l needs special -mfpu=neon flag; aarch64 doesn't.
       set_source_files_properties(
             shuffle-neon.c bitshuffle-neon.c
             PROPERTIES COMPILE_OPTIONS "-mfpu=neon;-flax-vector-conversions")
-      set_property(
-            SOURCE shuffle.c
-            APPEND PROPERTY COMPILE_OPTIONS "-mfpu=neon;-flax-vector-conversions")
     endif()
     # Define a symbol for the shuffle-dispatch implementation
     # so it knows NEON is supported even though that file is

src/c-blosc2/blosc/b2nd.c

@@ -395,6 +395,23 @@ int b2nd_zeros(b2nd_context_t *ctx, b2nd_array_t **array) {
 }
 
 
+int b2nd_nans(b2nd_context_t *ctx, b2nd_array_t **array) {
+  BLOSC_ERROR_NULL(ctx, BLOSC2_ERROR_NULL_POINTER);
+  BLOSC_ERROR_NULL(array, BLOSC2_ERROR_NULL_POINTER);
+
+  BLOSC_ERROR(array_new(ctx, BLOSC2_SPECIAL_NAN, array));
+
+  const int32_t typesize = (*array)->sc->typesize;
+  if (typesize != 4 && typesize != 8)
+  {
+    BLOSC_TRACE_ERROR("Unsupported typesize for NaN");
+    return BLOSC2_ERROR_DATA;
+  }
+
+  return BLOSC2_ERROR_SUCCESS;
+}
+
+
 int b2nd_full(b2nd_context_t *ctx, b2nd_array_t **array, const void *fill_value) {
   BLOSC_ERROR_NULL(ctx, BLOSC2_ERROR_NULL_POINTER);
   BLOSC_ERROR_NULL(array, BLOSC2_ERROR_NULL_POINTER);

src/c-blosc2/blosc/bitshuffle-altivec.c

@@ -24,6 +24,7 @@
 
 #include "bitshuffle-altivec.h"
 #include "bitshuffle-generic.h"
+#include <stdlib.h>
 
 /* Make sure ALTIVEC is available for the compilation target and compiler. */
 #if defined(__ALTIVEC__) && defined(__VSX__) && defined(_ARCH_PWR8)
@@ -33,7 +34,6 @@
 #include <altivec.h>
 
 #include <stdint.h>
-#include <stdlib.h>
 
 /* The next is useful for debugging purposes */
 #if 0
@@ -592,4 +592,23 @@ int64_t bshuf_untrans_bit_elem_altivec(const void* in, void* out, const size_t s
   return count;
 }
 
-#endif /* defined(__ALTIVEC__) */
+
+const bool is_bshuf_altivec = true;
+
+#else /* defined(__ALTIVEC__) && defined(__VSX__) && defined(_ARCH_PWR8) */
+
+const bool is_bshuf_altivec = false;
+
+int64_t
+bshuf_trans_bit_elem_altivec(const void* in, void* out, const size_t size,
+                             const size_t elem_size) {
+  abort();
+}
+
+int64_t
+bshuf_untrans_bit_elem_altivec(const void* in, void* out, const size_t size,
+                               const size_t elem_size) {
+  abort();
+}
+
+#endif /* defined(__ALTIVEC__) && defined(__VSX__) && defined(_ARCH_PWR8) */

src/c-blosc2/blosc/bitshuffle-altivec.h

@@ -17,6 +17,12 @@
 
 #include <stddef.h>
 #include <stdint.h>
+#include <stdbool.h>
+
+/**
+ * ALTIVEC-accelerated bit(un)shuffle routines availability.
+*/
+extern const bool is_bshuf_altivec;
 
 BLOSC_NO_EXPORT int64_t
     bshuf_trans_byte_elem_altivec(const void* in, void* out, const size_t size,

src/c-blosc2/blosc/bitshuffle-avx2.c

@@ -23,6 +23,7 @@
 #include "bitshuffle-avx2.h"
 #include "bitshuffle-sse2.h"
 #include "bitshuffle-generic.h"
+#include <stdlib.h>
 
 /* Make sure AVX2 is available for the compilation target and compiler. */
 #if defined(__AVX2__)
@@ -262,4 +263,22 @@ int64_t bshuf_untrans_bit_elem_AVX(const void* in, void* out, const size_t size,
   return count;
 }
 
+const bool is_bshuf_AVX = true;
+
+#else /* defined(__AVX2__) */
+
+const bool is_bshuf_AVX = false;
+
+int64_t
+bshuf_trans_bit_elem_AVX(const void* in, void* out, const size_t size,
+                         const size_t elem_size) {
+  abort();
+}
+
+int64_t
+bshuf_untrans_bit_elem_AVX(const void* in, void* out, const size_t size,
+                           const size_t elem_size) {
+  abort();
+}
+
 #endif /* defined(__AVX2__) */

src/c-blosc2/blosc/bitshuffle-avx2.h

@@ -17,6 +17,13 @@
 
 #include <stddef.h>
 #include <stdint.h>
+#include <stdbool.h>
+
+/**
+ * AVX2-accelerated bit(un)shuffle routines availability.
+*/
+extern const bool is_bshuf_AVX;
+
 
 /**
  * AVX2-accelerated bitshuffle routine.

src/c-blosc2/blosc/bitshuffle-avx512.c

@@ -20,13 +20,15 @@
   rights to use.
 **********************************************************************/
 
-/* Make sure AVX512 is available for the compilation target and compiler. */
-#if defined(__AVX512F__) && defined (__AVX512BW__)
-#include <immintrin.h>
 #include "bitshuffle-avx512.h"
 #include "bitshuffle-avx2.h"
 #include "bitshuffle-sse2.h"
 #include "bitshuffle-generic.h"
+#include <stdlib.h>
+
+/* Make sure AVX512 is available for the compilation target and compiler. */
+#if defined(__AVX512F__) && defined (__AVX512BW__)
+#include <immintrin.h>
 
 
 /* Transpose bits within bytes. */
@@ -158,4 +160,22 @@ int64_t bshuf_untrans_bit_elem_AVX512(const void* in, void* out, const size_t si
   return count;
 }
 
-#endif
+const bool is_bshuf_AVX512 = true;
+
+#else /* defined(__AVX512F__) && defined (__AVX512BW__) */
+
+const bool is_bshuf_AVX512 = false;
+
+int64_t
+bshuf_trans_bit_elem_AVX512(const void* in, void* out, const size_t size,
+                            const size_t elem_size) {
+  abort();
+}
+
+int64_t
+bshuf_untrans_bit_elem_AVX512(const void* in, void* out, const size_t size,
+                              const size_t elem_size) {
+  abort();
+}
+
+#endif /* defined(__AVX512F__) && defined (__AVX512BW__) */

src/c-blosc2/blosc/bitshuffle-avx512.h

@@ -17,6 +17,12 @@
 
 #include <stddef.h>
 #include <stdint.h>
+#include <stdbool.h>
+
+/**
+ * AVX512-accelerated bit(un)shuffle routines availability.
+*/
+extern const bool is_bshuf_AVX512;
 
 BLOSC_NO_EXPORT int64_t
     bshuf_trans_bit_elem_AVX512(const void* in, void* out, const size_t size,

src/c-blosc2/blosc/bitshuffle-neon.c

@@ -22,13 +22,13 @@
 
 #include "bitshuffle-neon.h"
 #include "bitshuffle-generic.h"
+#include <stdlib.h>
 
 /* Make sure NEON is available for the compilation target and compiler. */
 #if defined(__ARM_NEON)
 
 #include <arm_neon.h>
 
-#include <stdlib.h>
 
 /* The next is useful for debugging purposes */
 #if 0
@@ -491,4 +491,20 @@ int64_t bshuf_untrans_bit_elem_NEON(const void* in, void* out, const size_t size
     return count;
 }
 
+const bool is_bshuf_NEON = true;
+
+#else /* defined(__ARM_NEON) */
+
+const bool is_bshuf_NEON = false;
+
+int64_t bshuf_trans_bit_elem_NEON(const void* in, void* out, const size_t size,
+                                  const size_t elem_size) {
+  abort();
+}
+
+int64_t bshuf_untrans_bit_elem_NEON(const void* in, void* out, const size_t size,
+                                    const size_t elem_size) {
+  abort();
+}
+
 #endif /* defined(__ARM_NEON) */

src/c-blosc2/blosc/bitshuffle-neon.h

@@ -17,6 +17,12 @@
 
 #include <stddef.h>
 #include <stdint.h>
+#include <stdbool.h>
+
+/**
+ * NEON-accelerated bit(un)shuffle routines availability.
+*/
+extern const bool is_bshuf_NEON;
 
 /**
   NEON-accelerated bitshuffle routine.