MRZlib: route compress and decompress streams through zlib-ng (native mode)

.gitmodules

@@ -85,3 +85,6 @@
 [submodule "thirdparty/cpp-httplib"]
 	path = thirdparty/cpp-httplib
 	url = https://github.com/yhirose/cpp-httplib
+[submodule "thirdparty/zlib-ng"]
+	path = thirdparty/zlib-ng
+	url = https://github.com/zlib-ng/zlib-ng.git

cmake/Modules/Findzlib-ng.cmake

@@ -0,0 +1,40 @@
+# https://cmake.org/cmake/help/latest/manual/cmake-developer.7.html#a-sample-find-module
+
+find_package(PkgConfig QUIET)
+if(PkgConfig_FOUND)
+  pkg_check_modules(PC_zlib_ng QUIET zlib-ng)
+endif()
+
+find_path(zlib-ng_INCLUDE_DIR
+  NAMES zlib-ng.h
+  HINTS ${PC_zlib_ng_INCLUDE_DIRS}
+)
+find_library(zlib-ng_LIBRARY
+  NAMES z-ng zlib-ng
+  HINTS ${PC_zlib_ng_LIBRARY_DIRS}
+)
+
+if(zlib-ng_INCLUDE_DIR)
+  file(READ "${zlib-ng_INCLUDE_DIR}/zlib-ng.h" zlib-ng_VERSION_FILE)
+  string(REGEX MATCH "ZLIBNG_VERSION \"([0-9\\.]+)\"" _ "${zlib-ng_VERSION_FILE}")
+  set(zlib-ng_VERSION "${CMAKE_MATCH_1}")
+endif()
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(zlib-ng
+  REQUIRED_VARS zlib-ng_INCLUDE_DIR zlib-ng_LIBRARY
+  VERSION_VAR zlib-ng_VERSION
+)
+mark_as_advanced(
+  zlib-ng_INCLUDE_DIR
+  zlib-ng_LIBRARY
+)
+
+if(zlib-ng_FOUND AND NOT TARGET zlib-ng::zlib)
+  add_library(zlib-ng::zlib UNKNOWN IMPORTED)
+  set_target_properties(zlib-ng::zlib PROPERTIES
+    INTERFACE_INCLUDE_DIRECTORIES "${zlib-ng_INCLUDE_DIR}"
+    IMPORTED_LOCATION "${zlib-ng_LIBRARY}"
+    VERSION "${zlib-ng_VERSION}"
+  )
+endif()

requirements/macos.txt

@@ -24,3 +24,4 @@ tbb
 tinyxml2
 tl-expected
 zlib
+zlib-ng

requirements/vcpkg-linux.txt

@@ -37,3 +37,4 @@ tiff
 tinygltf
 tinyxml2
 tl-expected
+zlib-ng

requirements/windows.txt

@@ -30,5 +30,6 @@ tiff
 tinygltf
 tinyxml2
 tl-expected
+zlib-ng
 glad
 glfw3

source/MRMesh/CMakeLists.txt

@@ -33,6 +33,7 @@ find_package(JsonCpp REQUIRED)
 find_package(spdlog 1.9 REQUIRED)
 find_package(TBB 2020.1 COMPONENTS tbb REQUIRED)
 find_package(tl-expected REQUIRED)
+find_package(zlib-ng CONFIG REQUIRED)
 
 target_link_libraries(${PROJECT_NAME}
   PUBLIC
@@ -45,6 +46,7 @@ target_link_libraries(${PROJECT_NAME}
     tl::expected
   PRIVATE
     ${CMAKE_DL_LIBS}
+    zlib-ng::zlib
 )
 
 # libzip on Emscripten is unable to resolve its dependencies by itself

source/MRMesh/MRMeshConfig.cmake.in

@@ -15,6 +15,7 @@ if(EMSCRIPTEN)
   find_package(MbedTLS 1.0 REQUIRED)
   find_package(ZLIB REQUIRED)
   find_dependency(libzip)
+  find_dependency(zlib-ng)
 
   set(MRMESH_NO_GTEST @MRMESH_NO_GTEST@)
   if(NOT MRMESH_NO_GTEST)

source/MRMesh/MRZlib.cpp

@@ -2,7 +2,7 @@
 #include "MRBuffer.h"
 #include "MRFinally.h"
 
-#include <zlib.h>
+#include <zlib-ng.h>
 
 #include <cassert>
 
@@ -22,7 +22,7 @@ constexpr int kRawDeflateBits  = -MAX_WBITS;
 constexpr int kDefaultMemLevel = 8;
 static_assert( kDefaultMemLevel <= MAX_MEM_LEVEL );
 
-std::string zlibToString( int code )
+std::string zngToString( int code )
 {
     switch ( code )
     {
@@ -62,17 +62,17 @@ namespace MR
 Expected<void> zlibCompressStream( std::istream& in, std::ostream& out, const ZlibCompressParams& params )
 {
     Buffer<char> inChunk( cChunkSize ), outChunk( cChunkSize );
-    z_stream stream {
+    zng_stream stream {
         .zalloc = Z_NULL,
         .zfree = Z_NULL,
         .opaque = Z_NULL,
     };
     int ret;
-    if ( Z_OK != ( ret = deflateInit2( &stream, params.level, Z_DEFLATED, windowBitsFor( params.rawDeflate ), kDefaultMemLevel, Z_DEFAULT_STRATEGY ) ) )
-        return unexpected( zlibToString( ret ) );
+    if ( Z_OK != ( ret = zng_deflateInit2( &stream, params.level, Z_DEFLATED, windowBitsFor( params.rawDeflate ), kDefaultMemLevel, Z_DEFAULT_STRATEGY ) ) )
+        return unexpected( zngToString( ret ) );
 
     MR_FINALLY {
-        deflateEnd( &stream );
+        zng_deflateEnd( &stream );
     };
 
     if ( params.stats )
@@ -89,7 +89,7 @@ Expected<void> zlibCompressStream( std::istream& in, std::ostream& out, const Zl
 
         if ( params.stats )
         {
-            params.stats->crc32 = (uint32_t)crc32( params.stats->crc32, stream.next_in, stream.avail_in );
+            params.stats->crc32 = (uint32_t)zng_crc32( params.stats->crc32, stream.next_in, stream.avail_in );
             params.stats->uncompressedSize += stream.avail_in;
         }
 
@@ -98,9 +98,9 @@ Expected<void> zlibCompressStream( std::istream& in, std::ostream& out, const Zl
         {
             stream.next_out = reinterpret_cast<uint8_t*>( outChunk.data() );
             stream.avail_out = (unsigned)outChunk.size();
-            ret = deflate( &stream, flush );
+            ret = zng_deflate( &stream, flush );
             if ( Z_OK != ret && Z_STREAM_END != ret )
-                return unexpected( zlibToString( ret ) );
+                return unexpected( zngToString( ret ) );
 
             assert( stream.avail_out <= (unsigned)outChunk.size() );
             const unsigned written = (unsigned)outChunk.size() - stream.avail_out;
@@ -124,17 +124,17 @@ Expected<void> zlibCompressStream( std::istream& in, std::ostream& out, int leve
 Expected<void> zlibDecompressStream( std::istream& in, std::ostream& out, const ZlibParams& params )
 {
     Buffer<char> inChunk( cChunkSize ), outChunk( cChunkSize );
-    z_stream stream {
+    zng_stream stream {
         .zalloc = Z_NULL,
         .zfree = Z_NULL,
         .opaque = Z_NULL,
     };
     int ret;
-    if ( Z_OK != ( ret = inflateInit2( &stream, windowBitsFor( params.rawDeflate ) ) ) )
-        return unexpected( zlibToString( ret ) );
+    if ( Z_OK != ( ret = zng_inflateInit2( &stream, windowBitsFor( params.rawDeflate ) ) ) )
+        return unexpected( zngToString( ret ) );
 
     MR_FINALLY {
-        inflateEnd( &stream );
+        zng_inflateEnd( &stream );
     };
 
     while ( !in.eof() )
@@ -150,9 +150,9 @@ Expected<void> zlibDecompressStream( std::istream& in, std::ostream& out, const
         {
             stream.next_out = reinterpret_cast<uint8_t*>( outChunk.data() );
             stream.avail_out = (unsigned)outChunk.size();
-            ret = inflate( &stream, Z_NO_FLUSH );
+            ret = zng_inflate( &stream, Z_NO_FLUSH );
             if ( Z_OK != ret && Z_STREAM_END != ret )
-                return unexpected( zlibToString( ret ) );
+                return unexpected( zngToString( ret ) );
 
             assert( stream.avail_out <= (unsigned)outChunk.size() );
             out.write( outChunk.data(), (unsigned)outChunk.size() - stream.avail_out );

thirdparty/CMakeLists.txt

@@ -85,6 +85,24 @@ ENDIF()
 
 add_subdirectory(./OpenCTM-git ./OpenCTM)
 
+IF(NOT APPLE)
+  set(ZLIB_COMPAT       OFF CACHE BOOL "")
+  set(ZLIB_ENABLE_TESTS OFF CACHE BOOL "")
+  set(ZLIBNG_ENABLE_TESTS OFF CACHE BOOL "")
+  set(WITH_GTEST        OFF CACHE BOOL "")
+  set(WITH_GZFILEOP     OFF CACHE BOOL "")
+  IF(EMSCRIPTEN)
+    # Force static: undefined BUILD_SHARED_LIBS makes zlib-ng emit duplicate
+    # libz-ng.a rules under Emscripten. Restore to undefined (not empty)
+    # afterwards -- jsoncpp downstream breaks on empty BUILD_SHARED_LIBS.
+    set(BUILD_SHARED_LIBS OFF)
+    add_subdirectory(./zlib-ng)
+    unset(BUILD_SHARED_LIBS)
+  ELSE()
+    add_subdirectory(./zlib-ng)
+  ENDIF()
+ENDIF()
+
 option(PHMAP_INSTALL "" ON)
 add_subdirectory(./parallel-hashmap)
 

thirdparty/install.bat

@@ -88,7 +88,7 @@ REM Install vcpkg core dependencies
 vcpkg install vcpkg-cmake vcpkg-cmake-config --host-triplet %VCPKG_DEFAULT_TRIPLET% --overlay-triplets "%~dp0vcpkg\triplets" --debug --x-abi-tools-use-exact-versions || goto :error
 
 REM Install all required dependencies
-vcpkg install !packages! --host-triplet %VCPKG_DEFAULT_TRIPLET% --overlay-triplets "%~dp0vcpkg\triplets" --debug --x-abi-tools-use-exact-versions || goto :error
+vcpkg install !packages! --host-triplet %VCPKG_DEFAULT_TRIPLET% --overlay-triplets "%~dp0vcpkg\triplets" --overlay-ports "%~dp0vcpkg\ports" --debug --x-abi-tools-use-exact-versions || goto :error
 
 endlocal
 goto :EOF

thirdparty/vcpkg/ports/zlib-ng/portfile.cmake

@@ -0,0 +1,66 @@
+vcpkg_from_github(
+    OUT_SOURCE_PATH SOURCE_PATH
+    REPO zlib-ng/zlib-ng
+    REF "${VERSION}"
+    SHA512 e2057c764f1d5aaee738edee7e977182c5b097e3c95489dcd8de813f237d92a05daaa86d68d44b331d9fec5d1802586a8f6cfb658ba849874aaa14e72a8107f5
+    HEAD_REF develop
+)
+
+# Set ZLIB_COMPAT in the triplet file to turn on
+if(NOT DEFINED ZLIB_COMPAT)
+    set(ZLIB_COMPAT OFF)
+endif()
+
+vcpkg_cmake_configure(
+    SOURCE_PATH "${SOURCE_PATH}"
+    OPTIONS
+        "-DZLIB_FULL_VERSION=${ZLIB_FULL_VERSION}"
+        -DZLIB_ENABLE_TESTS=OFF
+        -DWITH_NEW_STRATEGIES=ON
+        -DZLIB_COMPAT=${ZLIB_COMPAT}
+    OPTIONS_RELEASE
+        -DWITH_OPTIM=ON
+)
+vcpkg_cmake_install()
+vcpkg_copy_pdbs()
+
+# Condition in `WIN32`, from https://github.com/zlib-ng/zlib-ng/blob/2.1.5/CMakeLists.txt#L1081-L1100
+# (dynamic) for `zlib` or (static `MSVC) for `zlibstatic` or default `z`
+# i.e. (windows) and not (static mingw) https://learn.microsoft.com/en-us/vcpkg/maintainers/variables#vcpkg_target_is_system
+if(VCPKG_TARGET_IS_WINDOWS AND (NOT (VCPKG_LIBRARY_LINKAGE STREQUAL static AND VCPKG_TARGET_IS_MINGW)))
+    set(_port_suffix)
+    if(ZLIB_COMPAT)
+        set(_port_suffix "")
+    else()
+        set(_port_suffix "-ng")
+    endif()
+
+    set(_port_output_name)
+    if(VCPKG_LIBRARY_LINKAGE STREQUAL "dynamic")
+        set(_port_output_name "zlib${_port_suffix}")
+    else()
+        set(_port_output_name "zlibstatic${_port_suffix}")
+    endif()
+
+    # CMAKE_DEBUG_POSTFIX from https://github.com/zlib-ng/zlib-ng/blob/2.1.5/CMakeLists.txt#L494
+    if(NOT DEFINED VCPKG_BUILD_TYPE OR VCPKG_BUILD_TYPE STREQUAL "release")
+        vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/lib/pkgconfig/zlib${_port_suffix}.pc" " -lz${_port_suffix}" " -l${_port_output_name}")
+    endif()
+    if(NOT DEFINED VCPKG_BUILD_TYPE OR VCPKG_BUILD_TYPE STREQUAL "debug")
+        vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/debug/lib/pkgconfig/zlib${_port_suffix}.pc" " -lz${_port_suffix}" " -l${_port_output_name}d")
+    endif()
+endif()
+
+vcpkg_fixup_pkgconfig()
+
+if(ZLIB_COMPAT)
+    set(_cmake_dir "ZLIB")
+else()
+    set(_cmake_dir "zlib-ng")
+endif()
+vcpkg_cmake_config_fixup(CONFIG_PATH lib/cmake/${_cmake_dir})
+
+file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/share"
+                    "${CURRENT_PACKAGES_DIR}/debug/include"
+)
+vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/LICENSE.md")

thirdparty/vcpkg/ports/zlib-ng/vcpkg.json

@@ -0,0 +1,18 @@
+{
+  "name": "zlib-ng",
+  "version": "2.3.3",
+  "port-version": 1,
+  "description": "zlib replacement with optimizations for 'next generation' systems",
+  "homepage": "https://github.com/zlib-ng/zlib-ng",
+  "license": "Zlib",
+  "dependencies": [
+    {
+      "name": "vcpkg-cmake",
+      "host": true
+    },
+    {
+      "name": "vcpkg-cmake-config",
+      "host": true
+    }
+  ]
+}

thirdparty/vcpkg/triplets/arm64-linux-meshlib.cmake

@@ -2,6 +2,12 @@ set(VCPKG_TARGET_ARCHITECTURE arm64)
 set(VCPKG_CRT_LINKAGE dynamic)
 set(VCPKG_LIBRARY_LINKAGE dynamic)
 
+# Have to build zlib-ng as a static library due to the auditwheel issue:
+# https://github.com/pypa/auditwheel/issues/613
+if(PORT STREQUAL "zlib-ng")
+  set(VCPKG_LIBRARY_LINKAGE static)
+endif()
+
 set(VCPKG_CMAKE_SYSTEM_NAME Linux)
 set(VCPKG_BUILD_TYPE release)
 

thirdparty/vcpkg/triplets/x64-linux-meshlib.cmake

@@ -2,6 +2,12 @@ set(VCPKG_TARGET_ARCHITECTURE x64)
 set(VCPKG_CRT_LINKAGE dynamic)
 set(VCPKG_LIBRARY_LINKAGE dynamic)
 
+# Have to build zlib-ng as a static library due to the auditwheel issue:
+# https://github.com/pypa/auditwheel/issues/613
+if(PORT STREQUAL "zlib-ng")
+  set(VCPKG_LIBRARY_LINKAGE static)
+endif()
+
 set(VCPKG_CMAKE_SYSTEM_NAME Linux)
 set(VCPKG_BUILD_TYPE release)