From 3179b10d8a12a4be2d43cfbb381c6190f80feed2 Mon Sep 17 00:00:00 2001 From: TL Date: Thu, 16 Jan 2025 13:41:03 +0800 Subject: [PATCH 01/20] shared heap create from preallocated buffer and use as a single memory region --- .../shared-heap-preallocated/CMakeLists.txt | 127 +++++++ samples/shared-heap-preallocated/src/main.c | 335 ++++++++++++++++++ .../wasm-apps/CMakeLists.txt | 43 +++ .../wasm-apps/test1.c | 71 ++++ .../wasm-apps/test2.c | 18 + 5 files changed, 594 insertions(+) create mode 100644 samples/shared-heap-preallocated/CMakeLists.txt create mode 100644 samples/shared-heap-preallocated/src/main.c create mode 100644 samples/shared-heap-preallocated/wasm-apps/CMakeLists.txt create mode 100644 samples/shared-heap-preallocated/wasm-apps/test1.c create mode 100644 samples/shared-heap-preallocated/wasm-apps/test2.c diff --git a/samples/shared-heap-preallocated/CMakeLists.txt b/samples/shared-heap-preallocated/CMakeLists.txt new file mode 100644 index 0000000000..6346d077e7 --- /dev/null +++ b/samples/shared-heap-preallocated/CMakeLists.txt @@ -0,0 +1,127 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required (VERSION 3.14) + +include(CheckPIESupported) + +if (NOT WAMR_BUILD_PLATFORM STREQUAL "windows") + project (shared_heap_test) +else() + project (shared_heap_test C ASM) +endif() + +################ runtime settings ################ +string (TOLOWER ${CMAKE_HOST_SYSTEM_NAME} WAMR_BUILD_PLATFORM) +if (APPLE) + add_definitions(-DBH_PLATFORM_DARWIN) +endif () + +# Reset default linker flags +set (CMAKE_SHARED_LIBRARY_LINK_C_FLAGS "") +set (CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS "") + +# WAMR features switch + +# Set WAMR_BUILD_TARGET, currently values supported: +# "X86_64", "AMD_64", "X86_32", "AARCH64[sub]", "ARM[sub]", "THUMB[sub]", +# "MIPS", "XTENSA", "RISCV64[sub]", "RISCV32[sub]" + +if (NOT DEFINED WAMR_BUILD_TARGET) + if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(arm64|aarch64)") + set (WAMR_BUILD_TARGET "AARCH64") + elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "riscv64") + set (WAMR_BUILD_TARGET "RISCV64") + elseif (CMAKE_SIZEOF_VOID_P EQUAL 8) + # Build as X86_64 by default in 64-bit platform + set (WAMR_BUILD_TARGET "X86_64") + elseif (CMAKE_SIZEOF_VOID_P EQUAL 4) + # Build as X86_32 by default in 32-bit platform + set (WAMR_BUILD_TARGET "X86_32") + else () + message(SEND_ERROR "Unsupported build target platform!") + endif () +endif () + +if (NOT CMAKE_BUILD_TYPE) + set (CMAKE_BUILD_TYPE Debug) +endif () + +set (WAMR_BUILD_INTERP 1) +set (WAMR_BUILD_FAST_INTERP 1) +set (WAMR_BUILD_AOT 1) +set (WAMR_BUILD_JIT 0) +set (WAMR_BUILD_LIBC_BUILTIN 1) +set (WAMR_BUILD_LIBC_WASI 0) +set (WAMR_BUILD_SHARED_HEAP 1) +set (WAMR_BUILD_GC_HEAP_VERIFY 1) + +if (NOT MSVC) + # linker flags + if (NOT (CMAKE_C_COMPILER MATCHES ".*clang.*" OR CMAKE_C_COMPILER_ID MATCHES ".*Clang")) + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gc-sections") + endif () + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra -Wformat -Wformat-security") + if (WAMR_BUILD_TARGET MATCHES "X86_.*" OR WAMR_BUILD_TARGET STREQUAL "AMD_64") + if (NOT (CMAKE_C_COMPILER MATCHES ".*clang.*" OR CMAKE_C_COMPILER_ID MATCHES ".*Clang")) + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mindirect-branch-register") + endif () + endif () +endif () + +# build out vmlib +set (WAMR_ROOT_DIR ${CMAKE_CURRENT_LIST_DIR}/../..) +include (${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake) + +add_library(vmlib STATIC ${WAMR_RUNTIME_LIB_SOURCE}) +if (MSVC) + target_compile_definitions(vmlib PRIVATE WASM_API_EXTERN=) +endif() +target_link_libraries(vmlib ${LLVM_AVAILABLE_LIBS} ${UV_A_LIBS} -lm -ldl -lpthread) + +################ application related ################ +include_directories(${CMAKE_CURRENT_LIST_DIR}/src) +include (${SHARED_DIR}/utils/uncommon/shared_uncommon.cmake) + +add_executable (shared_heap_test src/main.c ${UNCOMMON_SHARED_SOURCE}) + +check_pie_supported() +set_target_properties (shared_heap_test PROPERTIES POSITION_INDEPENDENT_CODE ON) + +if (APPLE) + target_link_libraries (shared_heap_test vmlib -lm -ldl -lpthread) +else () + target_link_libraries (shared_heap_test vmlib -lm -ldl -lpthread -lrt) +endif () + +add_subdirectory(wasm-apps) + +if (WAMR_BUILD_AOT EQUAL 1) + set (WAMR_COMPILER_DIR ${CMAKE_CURRENT_LIST_DIR}/../../wamr-compiler/build) + message (CHECK_START "Detecting WAMR_COMPILER at ${WAMR_COMPILER_DIR}") + find_file (WAMR_COMPILER + wamrc + PATHS "${CMAKE_CURRENT_LIST_DIR}/../../wamr-compiler/build" + NO_DEFAULT_PATH + NO_CMAKE_FIND_ROOT_PATH + ) + if (WAMR_COMPILER) + message (CHECK_PASS "found") + else() + message (CHECK_FAIL "not found") + endif() + if (NOT EXISTS ${WAMR_COMPILER}) + message (FATAL_ERROR "Please build wamrc under ${WAMR_ROOT_DIR}/wamr-compiler") + else() + message (STATUS "WAMR_COMPILER is ${WAMR_COMPILER}") + endif() + + add_custom_target( + wasm_to_aot + ALL + DEPENDS wasm-apps/test1.wasm wasm-apps/test2.wasm ${WAMR_COMPILER} + COMMAND ${WAMR_COMPILER} --enable-shared-heap -o wasm-apps/test1.aot wasm-apps/test1.wasm + COMMAND ${WAMR_COMPILER} --enable-shared-heap -o wasm-apps/test2.aot wasm-apps/test2.wasm + WORKING_DIRECTORY ${CMAKE_BINARY_DIR} + ) +endif() diff --git a/samples/shared-heap-preallocated/src/main.c b/samples/shared-heap-preallocated/src/main.c new file mode 100644 index 0000000000..9842f33c30 --- /dev/null +++ b/samples/shared-heap-preallocated/src/main.c @@ -0,0 +1,335 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "wasm_export.h" +#include "bh_platform.h" +#include "bh_read_file.h" + +typedef struct thread_arg { + bh_queue *queue; + wasm_module_inst_t module_inst; +} thread_arg; + +static void * +thread1_callback(void *arg) +{ + thread_arg *targ = arg; + wasm_module_inst_t module_inst = targ->module_inst; + bh_queue *queue = targ->queue; + wasm_exec_env_t exec_env; + wasm_function_inst_t my_shared_heap_malloc_func; + wasm_function_inst_t my_shared_heap_free_func; + uint32 i, argv[2]; + + /* lookup wasm functions */ + if (!(my_shared_heap_malloc_func = wasm_runtime_lookup_function( + module_inst, "my_shared_heap_malloc")) + || !(my_shared_heap_free_func = wasm_runtime_lookup_function( + module_inst, "my_shared_heap_free"))) { + printf("Failed to lookup function.\n"); + } + + /* create exec env */ + if (!(exec_env = wasm_runtime_create_exec_env(module_inst, 32768))) { + printf("Failed to create exec env.\n"); + return NULL; + } + + /* allocate memory with wasm_runtime_shared_heap_malloc and send it + to wasm app2 */ + for (i = 0; i < 1; i++) { + uint8 *buf; + uint64 offset; + + offset = wasm_runtime_shared_heap_malloc(module_inst, 4096 * (i + 1), + (void **)&buf); + + if (offset == 0) { + printf("Failed to allocate memory from shared heap\n"); + break; + } + + snprintf(buf, 1024, "Hello, this is buf %u allocated from shared heap", + i + 1); + + printf("wasm app1 send buf: %s\n\n", buf); + if (!bh_post_msg(queue, 1, buf, 1024 * i)) { + printf("Failed to post message to queue\n"); + wasm_runtime_shared_heap_free(module_inst, offset); + break; + } + } + + /* allocate memory by calling my_shared_heap_malloc function and send it + to wasm app2 */ + for (i = 1; i < 2; i++) { + uint8 *buf; + + argv[0] = 1024 * (i + 1); + argv[1] = i + 1; + wasm_runtime_call_wasm(exec_env, my_shared_heap_malloc_func, 2, argv); + + if (wasm_runtime_get_exception(module_inst)) { + printf("Failed to call 'my_shared_heap_malloc' function: %s\n", + wasm_runtime_get_exception(module_inst)); + break; + } + if (argv[0] == 0) { + printf("Failed to allocate memory from shared heap\n"); + break; + } + + buf = wasm_runtime_addr_app_to_native(module_inst, argv[0]); + + printf("wasm app1 send buf: %s\n\n", buf); + if (!bh_post_msg(queue, 1, buf, 1024 * i)) { + printf("Failed to post message to queue\n"); + wasm_runtime_shared_heap_free(module_inst, argv[0]); + break; + } + } + + wasm_runtime_destroy_exec_env(exec_env); + + return NULL; +} + +static void +queue_callback(void *message, void *arg) +{ + bh_message_t msg = (bh_message_t)message; + wasm_exec_env_t exec_env = arg; + wasm_module_inst_t module_inst = wasm_runtime_get_module_inst(exec_env); + wasm_function_inst_t print_buf_func; + uint32 argv[2]; + + /* lookup wasm function */ + if (!(print_buf_func = + wasm_runtime_lookup_function(module_inst, "print_buf"))) { + printf("Failed to lookup function.\n"); + return; + } + + char *buf = bh_message_payload(msg); + printf("wasm app's native queue received buf: %s\n\n", buf); + + /* call wasm function */ + argv[0] = wasm_runtime_addr_native_to_app(module_inst, buf); + wasm_runtime_call_wasm(exec_env, print_buf_func, 1, argv); + if (wasm_runtime_get_exception(module_inst)) { + printf("Failed to call 'print_buf' function: %s\n", + wasm_runtime_get_exception(module_inst)); + } +} + +static void * +thread2_callback(void *arg) +{ + thread_arg *targ = arg; + bh_queue *queue = targ->queue; + wasm_module_inst_t module_inst = targ->module_inst; + wasm_exec_env_t exec_env; + + /* create exec env */ + if (!(exec_env = wasm_runtime_create_exec_env(module_inst, 32768))) { + printf("Failed to create exec env.\n"); + return NULL; + } + + /* enter queue's message loop until bh_queue_exit_loop_run + is called */ + bh_queue_enter_loop_run(queue, queue_callback, exec_env); + + wasm_runtime_destroy_exec_env(exec_env); + + return NULL; +} + +static char global_heap_buf[512 * 1024]; + +int +main(int argc, char **argv) +{ + char *wasm_file1 = NULL, *wasm_file2 = NULL; + uint8 *wasm_file1_buf = NULL, *wasm_file2_buf = NULL; + uint32 wasm_file1_size, wasm_file2_size; + wasm_module_t wasm_module1 = NULL, wasm_module2 = NULL; + wasm_module_inst_t module_inst1 = NULL; + wasm_module_inst_t module_inst2 = NULL; + wasm_shared_heap_t shared_heap = NULL; + bh_queue *queue = NULL; + RuntimeInitArgs init_args; + SharedHeapInitArgs heap_init_args; + char error_buf[128] = { 0 }; + bool aot_mode = false; + int ret = -1; + +#if WASM_DISABLE_HW_BOUND_CHECK == 0 +#if defined(BUILD_TARGET_X86_64) + printf("HW BOUND CHECK\n"); +#endif +#endif + + if (argc > 1 && !strcmp(argv[1], "--aot")) + aot_mode = true; + + if (!aot_mode) + printf("Test shared heap in interpreter mode\n\n"); + else + printf("Test shared heap in AOT mode\n\n"); + + bh_log_set_verbose_level(5); + memset(&init_args, 0, sizeof(RuntimeInitArgs)); + + init_args.mem_alloc_type = Alloc_With_Pool; + init_args.mem_alloc_option.pool.heap_buf = global_heap_buf; + init_args.mem_alloc_option.pool.heap_size = sizeof(global_heap_buf); + + /* init wasm runtime */ + if (!wasm_runtime_full_init(&init_args)) { + printf("Init runtime environment failed.\n"); + return -1; + } + + /* create queue */ + if (!(queue = bh_queue_create())) { + printf("Create queue failed.\n"); + goto fail; + } + + /* read wasm file */ + if (!aot_mode) + wasm_file1 = "./wasm-apps/test1.wasm"; + else + wasm_file1 = "./wasm-apps/test1.aot"; + if (!(wasm_file1_buf = + bh_read_file_to_buffer(wasm_file1, &wasm_file1_size))) { + printf("Open wasm file %s failed.\n", wasm_file1); + goto fail; + } + + /* load wasm file */ + wasm_module1 = wasm_runtime_load((uint8 *)wasm_file1_buf, wasm_file1_size, + error_buf, sizeof(error_buf)); + if (!wasm_module1) { + printf("Load wasm module failed. error: %s\n", error_buf); + goto fail; + } + + /* instantiate module */ + module_inst1 = wasm_runtime_instantiate(wasm_module1, 65536, 4096, + error_buf, sizeof(error_buf)); + if (!module_inst1) { + printf("Instantiate wasm module failed. error: %s\n", error_buf); + goto fail; + } + + /* read wasm file */ + if (!aot_mode) + wasm_file2 = "./wasm-apps/test2.wasm"; + else + wasm_file2 = "./wasm-apps/test2.aot"; + if (!(wasm_file2_buf = + bh_read_file_to_buffer(wasm_file2, &wasm_file2_size))) { + printf("Open wasm file %s failed.\n", wasm_file1); + goto fail; + } + + /* load wasm file */ + wasm_module2 = wasm_runtime_load((uint8 *)wasm_file2_buf, wasm_file2_size, + error_buf, sizeof(error_buf)); + if (!wasm_module2) { + printf("Load wasm module failed. error: %s\n", error_buf); + goto fail; + } + + /* instantiate module */ + module_inst2 = wasm_runtime_instantiate(wasm_module2, 65536, 4096, + error_buf, sizeof(error_buf)); + if (!module_inst2) { + printf("Instantiate wasm module failed. error: %s\n", error_buf); + goto fail; + } + + /* create shared heap */ + memset(&heap_init_args, 0, sizeof(heap_init_args)); + heap_init_args.size = 4096 + 1; + shared_heap = wasm_runtime_create_shared_heap(&heap_init_args); + if (!shared_heap) { + printf("Create shared heap failed. error: %s\n", error_buf); + goto fail; + } + + /* attach module instance 1 to the shared heap */ + if (!wasm_runtime_attach_shared_heap(module_inst1, shared_heap)) { + printf("Attach shared heap failed.\n"); + goto fail; + } + + /* attach module instance 2 to the shared heap */ + if (!wasm_runtime_attach_shared_heap(module_inst2, shared_heap)) { + printf("Attach shared heap failed.\n"); + goto fail; + } + + /* create thread 1 */ + struct thread_arg targ1 = { 0 }; + korp_tid tid1; + targ1.queue = queue; + targ1.module_inst = module_inst1; + if (os_thread_create(&tid1, thread1_callback, &targ1, + APP_THREAD_STACK_SIZE_DEFAULT)) { + printf("Failed to create thread 1\n"); + goto fail; + } + + /* create thread 2 */ + struct thread_arg targ2 = { 0 }; + korp_tid tid2; + targ2.queue = queue; + targ2.module_inst = module_inst2; + if (os_thread_create(&tid2, thread2_callback, &targ2, + APP_THREAD_STACK_SIZE_DEFAULT)) { + printf("Failed to create thread 2\n"); + os_thread_join(tid1, NULL); + goto fail; + } + + /* wait until all messages are post to wasm app2 and wasm app2 + handles all of them, then exit the queue message loop */ + usleep(10000); + bh_queue_exit_loop_run(queue); + + os_thread_join(tid1, NULL); + os_thread_join(tid2, NULL); + + ret = 0; + +fail: + if (module_inst2) + wasm_runtime_deinstantiate(module_inst2); + + if (module_inst1) + wasm_runtime_deinstantiate(module_inst1); + + if (wasm_module2) + wasm_runtime_unload(wasm_module2); + + if (wasm_module1) + wasm_runtime_unload(wasm_module1); + + if (wasm_file2_buf) + wasm_runtime_free(wasm_file2_buf); + + if (wasm_file1_buf) + wasm_runtime_free(wasm_file1_buf); + + if (queue) + bh_queue_destroy(queue); + + wasm_runtime_destroy(); + + return ret; +} diff --git a/samples/shared-heap-preallocated/wasm-apps/CMakeLists.txt b/samples/shared-heap-preallocated/wasm-apps/CMakeLists.txt new file mode 100644 index 0000000000..a90c82e1ad --- /dev/null +++ b/samples/shared-heap-preallocated/wasm-apps/CMakeLists.txt @@ -0,0 +1,43 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required(VERSION 3.14) +project(wasm-apps) + +set(WAMR_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../..) + +if (APPLE) + set (HAVE_FLAG_SEARCH_PATHS_FIRST 0) + set (CMAKE_C_LINK_FLAGS "") + set (CMAKE_CXX_LINK_FLAGS "") +endif () + +set (CMAKE_SYSTEM_PROCESSOR wasm32) +set (CMAKE_SYSROOT ${WAMR_ROOT_DIR}/wamr-sdk/app/libc-builtin-sysroot) + +if (NOT DEFINED WASI_SDK_DIR) + set (WASI_SDK_DIR "/opt/wasi-sdk") +endif () + +set (CMAKE_C_COMPILER_TARGET "wasm32") +set (CMAKE_C_COMPILER "${WASI_SDK_DIR}/bin/clang") +set (CMAKE_C_FLAGS "-nostdlib -Qunused-arguments -z stack-size=32768") + +set (DEFINED_SYMBOLS "${WAMR_ROOT_DIR}/wamr-sdk/app/libc-builtin-sysroot/share/defined-symbols.txt") + +set (CMAKE_EXE_LINKER_FLAGS + "-O0 -Wl,--initial-memory=65536, \ + -Wl,--no-entry,--strip-all, \ + -Wl,--export=__heap_base,--export=__data_end \ + -Wl,--export=__wasm_call_ctors \ + -Wl,--export=my_shared_heap_malloc \ + -Wl,--export=my_shared_heap_free \ + -Wl,--export=print_buf \ + -Wl,--allow-undefined" +) + +add_executable(test1.wasm test1.c) +target_link_libraries(test1.wasm) + +add_executable(test2.wasm test2.c) +target_link_libraries(test2.wasm) diff --git a/samples/shared-heap-preallocated/wasm-apps/test1.c b/samples/shared-heap-preallocated/wasm-apps/test1.c new file mode 100644 index 0000000000..2e3fd2112e --- /dev/null +++ b/samples/shared-heap-preallocated/wasm-apps/test1.c @@ -0,0 +1,71 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include +#include +#include + +extern void * +shared_heap_malloc(uint32_t size); +extern void +shared_heap_free(void *ptr); + +void * +my_shared_heap_malloc(uint32_t size, uint32_t index) +{ + char *buf1 = NULL, *buf2 = NULL, *buf; + + char *buf3 = NULL; + buf3 = malloc(2048); + + buf1 = shared_heap_malloc(1024); + if (!buf1) + return NULL; + + buf1[0] = 'H'; + buf1[1] = 'e'; + buf1[2] = 'l'; + buf1[3] = 'l'; + buf1[4] = 'o'; + buf1[5] = ','; + buf1[6] = ' '; + + buf2 = shared_heap_malloc(1024); + if (!buf2) { + shared_heap_free(buf1); + return NULL; + } + + snprintf(buf2, 1024, "this is buf %u allocated from shared heap", index); + + buf = shared_heap_malloc(size); + if (!buf) { + shared_heap_free(buf1); + shared_heap_free(buf2); + return NULL; + } + + memset(buf, 0, size); + memcpy(buf, buf1, strlen(buf1)); + memcpy(buf + strlen(buf1), buf2, strlen(buf2)); + + uint32_t max_value = UINT32_MAX; + char *address = (char *)max_value; + printf("The address of buf1, buf2, buf, UINT32_MAX is: %p,%p,%p,%p\n", buf1, + buf2, buf, address); + address[0] = 'c'; + printf("The content address of UINT32_MAX is: %c\n", *address); + + shared_heap_free(buf1); + shared_heap_free(buf2); + return buf; +} + +void +my_shared_heap_free(void *ptr) +{ + shared_heap_free(ptr); +} diff --git a/samples/shared-heap-preallocated/wasm-apps/test2.c b/samples/shared-heap-preallocated/wasm-apps/test2.c new file mode 100644 index 0000000000..b63efcd1a2 --- /dev/null +++ b/samples/shared-heap-preallocated/wasm-apps/test2.c @@ -0,0 +1,18 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include + +#include + +extern void +shared_heap_free(void *ptr); + +void +print_buf(char *buf) +{ + printf("wasm app2's wasm func received buf: %s\n\n", buf); + shared_heap_free(buf); +} From 013306d67cf749d5971510b47fb8859ac6d840a7 Mon Sep 17 00:00:00 2001 From: TL Date: Thu, 23 Jan 2025 18:20:01 +0800 Subject: [PATCH 02/20] first draft of shared heap enhancement in interpreter and runtime --- core/iwasm/common/wasm_memory.c | 263 ++++++++++++++----- core/iwasm/interpreter/wasm_interp_classic.c | 50 +++- 2 files changed, 230 insertions(+), 83 deletions(-) diff --git a/core/iwasm/common/wasm_memory.c b/core/iwasm/common/wasm_memory.c index 74df84e56c..a24b3c32d8 100644 --- a/core/iwasm/common/wasm_memory.c +++ b/core/iwasm/common/wasm_memory.c @@ -143,7 +143,7 @@ is_bounds_checks_enabled(WASMModuleInstanceCommon *module_inst) #if WASM_ENABLE_SHARED_HEAP != 0 static void * -wasm_mmap_linear_memory(uint64_t map_size, uint64 commit_size); +wasm_mmap_linear_memory(uint64 map_size, uint64 commit_size); static void wasm_munmap_linear_memory(void *mapped_mem, uint64 commit_size, uint64 map_size); @@ -177,39 +177,53 @@ wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args) goto fail1; } - if (!(heap->heap_handle = - runtime_malloc(mem_allocator_get_heap_struct_size()))) { - goto fail2; - } - size = align_uint(size, os_getpagesize()); heap->size = size; heap->start_off_mem64 = UINT64_MAX - heap->size + 1; heap->start_off_mem32 = UINT32_MAX - heap->size + 1; + heap->attached_count = 0; + + if (init_args->pre_allocated_addr != NULL) { + /* Create shared heap from a pre allocated buffer, its size need to + * align with system page */ + if (size != init_args->size) { + LOG_WARNING("Pre allocated size need to be aligned with system " + "page size to create shared heap"); + goto fail1; + } - if (size > APP_HEAP_SIZE_MAX || size < APP_HEAP_SIZE_MIN) { - LOG_WARNING("Invalid size of shared heap"); - goto fail3; + heap->heap_handle = NULL; } + else { + if (!(heap->heap_handle = + runtime_malloc(mem_allocator_get_heap_struct_size()))) { + goto fail2; + } + + if (size > APP_HEAP_SIZE_MAX || size < APP_HEAP_SIZE_MIN) { + LOG_WARNING("Invalid size of shared heap"); + goto fail3; + } #ifndef OS_ENABLE_HW_BOUND_CHECK - map_size = size; + map_size = size; #else - /* Totally 8G is mapped, the opcode load/store address range is 0 to 8G: - * ea = i + memarg.offset - * both i and memarg.offset are u32 in range 0 to 4G - * so the range of ea is 0 to 8G - */ - map_size = 8 * (uint64)BH_GB; + /* Totally 8G is mapped, the opcode load/store address range is 0 to 8G: + * ea = i + memarg.offset + * both i and memarg.offset are u32 in range 0 to 4G + * so the range of ea is 0 to 8G + */ + map_size = 8 * (uint64)BH_GB; #endif - if (!(heap->base_addr = wasm_mmap_linear_memory(map_size, size))) { - goto fail3; - } - if (!mem_allocator_create_with_struct_and_pool( - heap->heap_handle, heap_struct_size, heap->base_addr, size)) { - LOG_WARNING("init share heap failed"); - goto fail4; + if (!(heap->base_addr = wasm_mmap_linear_memory(map_size, size))) { + goto fail3; + } + if (!mem_allocator_create_with_struct_and_pool( + heap->heap_handle, heap_struct_size, heap->base_addr, size)) { + LOG_WARNING("init share heap failed"); + goto fail4; + } } os_mutex_lock(&shared_heap_list_lock); @@ -233,6 +247,74 @@ wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args) return NULL; } +WASMSharedHeap * +wasm_runtime_chain_shared_heaps(WASMSharedHeap *head, WASMSharedHeap *body) +{ + WASMSharedHeap *cur; + bool heap_handle_exist = false; + + if (!head || !body) { + LOG_WARNING("Invalid shared heap to chain."); + return NULL; + } + + os_mutex_lock(&shared_heap_list_lock); + if (head->attached_count != 0 || body->attached_count != 0) { + LOG_WARNING("To create shared heap chain, all shared heap need to be " + "detached first."); + os_mutex_unlock(&shared_heap_list_lock); + return NULL; + } + + for (cur = head; cur; cur = cur->chain_next) { + if (cur->heap_handle && heap_handle_exist) { + LOG_WARNING( + "To create shared heap chain, only one of shared heap can " + "dynamically shared_heap_malloc and shared_heap_free, the rest " + "can only be pre-allocated shared heap"); + os_mutex_unlock(&shared_heap_list_lock); + return NULL; + } + if (cur->heap_handle) + heap_handle_exist = true; + } + + head->start_off_mem64 = body->start_off_mem64 - head->size + 1; + head->start_off_mem32 = body->start_off_mem32 - head->size + 1; + head->chain_next = body; + os_mutex_unlock(&shared_heap_list_lock); + return head; +} + +WASMSharedHeap * +wasm_runtime_unchain_shared_heaps(WASMSharedHeap *head, bool entire_chain) +{ + WASMSharedHeap *cur; + + if (!head || !head->chain_next) { + LOG_WARNING("Invalid shared heap chain to disconnect the head from."); + return NULL; + } + + os_mutex_lock(&shared_heap_list_lock); + if (head->attached_count != 0) { + LOG_WARNING("To disconnect the shared heap head from the shared heap " + "chain, the shared heap chain needs to be detached first."); + os_mutex_unlock(&shared_heap_list_lock); + return NULL; + } + + for (cur = head; cur && cur->chain_next; cur = cur->chain_next) { + cur->start_off_mem64 = UINT64_MAX - cur->size + 1; + cur->start_off_mem32 = UINT32_MAX - cur->size + 1; + if (!entire_chain) + break; + } + + os_mutex_unlock(&shared_heap_list_lock); + return cur; +} + bool wasm_runtime_attach_shared_heap_internal(WASMModuleInstanceCommon *module_inst, WASMSharedHeap *shared_heap) @@ -303,6 +385,9 @@ wasm_runtime_attach_shared_heap_internal(WASMModuleInstanceCommon *module_inst, } #endif /* end of WASM_ENABLE_AOT != 0 */ + os_mutex_lock(&shared_heap_list_lock); + shared_heap->attached_count++; + os_mutex_unlock(&shared_heap_list_lock); return true; } @@ -324,6 +409,11 @@ wasm_runtime_detach_shared_heap_internal(WASMModuleInstanceCommon *module_inst) if (module_inst->module_type == Wasm_Module_Bytecode) { WASMModuleInstanceExtra *e = (WASMModuleInstanceExtra *)((WASMModuleInstance *)module_inst)->e; + if (e->shared_heap != NULL) { + os_mutex_lock(&shared_heap_list_lock); + e->shared_heap->attached_count--; + os_mutex_unlock(&shared_heap_list_lock); + } e->shared_heap = NULL; #if WASM_ENABLE_JIT != 0 #if UINTPTR_MAX == UINT64_MAX @@ -339,6 +429,11 @@ wasm_runtime_detach_shared_heap_internal(WASMModuleInstanceCommon *module_inst) if (module_inst->module_type == Wasm_Module_AoT) { AOTModuleInstanceExtra *e = (AOTModuleInstanceExtra *)((AOTModuleInstance *)module_inst)->e; + if (e->shared_heap != NULL) { + os_mutex_lock(&shared_heap_list_lock); + e->shared_heap->attached_count--; + os_mutex_unlock(&shared_heap_list_lock); + } e->shared_heap = NULL; #if UINTPTR_MAX == UINT64_MAX e->shared_heap_start_off.u64 = UINT64_MAX; @@ -387,9 +482,11 @@ wasm_runtime_get_shared_heap(WASMModuleInstanceCommon *module_inst_comm) static bool is_app_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst, - bool is_memory64, uint64 app_offset, uint32 bytes) + bool is_memory64, uint64 app_offset, uint32 bytes, + WASMSharedHeap **target_heap) { - WASMSharedHeap *heap = get_shared_heap(module_inst); + WASMSharedHeap *heap = get_shared_heap(module_inst), *cur; + uint64 shared_heap_start, shared_heap_end; if (!heap) { return false; @@ -399,57 +496,66 @@ is_app_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst, bytes = 1; } - if (!is_memory64) { - if (app_offset >= heap->start_off_mem32 - && app_offset <= UINT32_MAX - bytes + 1) { - return true; - } - } - else { - if (app_offset >= heap->start_off_mem64 - && app_offset <= UINT64_MAX - bytes + 1) { + for (cur = heap; cur; cur = cur->chain_next) { + shared_heap_start = + cur->start_off_mem64 ? is_memory64 : cur->start_off_mem32; + shared_heap_end = shared_heap_start + cur->size; + if (app_offset >= shared_heap_start && app_offset <= shared_heap_end) { + if (target_heap) + *target_heap = cur; return true; } } + if (target_heap) + *target_heap = NULL; return false; } static bool is_native_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst, - uint8 *addr, uint32 bytes) + uint8 *addr, uint32 bytes, + WASMSharedHeap **target_heap) { - WASMSharedHeap *heap = get_shared_heap(module_inst); - uintptr_t base_addr; - uintptr_t addr_int; - uintptr_t end_addr; + WASMSharedHeap *cur, *heap_head = get_shared_heap(module_inst); + uintptr_t base_addr, addr_int, end_addr; - if (!heap) { + if (!heap_head) { return false; } - base_addr = (uintptr_t)heap->base_addr; - addr_int = (uintptr_t)addr; - if (addr_int < base_addr) { - return false; - } + /* Iterate through shared heap chain to find whether native addr in one of + * shared heap */ + for (cur = heap_head; cur != NULL; cur = cur->chain_next) { + base_addr = (uintptr_t)cur->base_addr; + addr_int = (uintptr_t)addr; + if (addr_int < base_addr) { + continue; + } - end_addr = addr_int + bytes; - /* Check for overflow */ - if (end_addr <= addr_int) { - return false; - } + end_addr = addr_int + bytes; + /* Check for overflow */ + if (end_addr <= addr_int) { + continue; + } - if (end_addr > base_addr + heap->size) { - return false; + if (end_addr > base_addr + cur->size) { + continue; + } + + if (target_heap) + *target_heap = cur; + return true; } - return true; + if (target_heap) + *target_heap = NULL; + return false; } uint64 wasm_runtime_shared_heap_malloc(WASMModuleInstanceCommon *module_inst, - uint64_t size, void **p_native_addr) + uint64 size, void **p_native_addr) { WASMMemoryInstance *memory = wasm_get_default_memory((WASMModuleInstance *)module_inst); @@ -459,6 +565,14 @@ wasm_runtime_shared_heap_malloc(WASMModuleInstanceCommon *module_inst, if (!memory || !shared_heap) return 0; + while (shared_heap && !shared_heap->heap_handle) { + shared_heap = shared_heap->chain_next; + } + if (!shared_heap) { + LOG_WARNING("Can't allocate from pre allocated shared heap"); + return 0; + } + native_addr = mem_allocator_malloc(shared_heap->heap_handle, size); if (!native_addr) return 0; @@ -487,6 +601,14 @@ wasm_runtime_shared_heap_free(WASMModuleInstanceCommon *module_inst, uint64 ptr) return; } + while (shared_heap && !shared_heap->heap_handle) { + shared_heap = shared_heap->chain_next; + } + if (!shared_heap) { + LOG_WARNING("The address to free is from pre allocated shared heap"); + return; + } + if (memory->is_memory64) { if (ptr < shared_heap->start_off_mem64) { /* ptr can not > UINT64_MAX */ LOG_WARNING("The address to free isn't in shared heap"); @@ -564,14 +686,16 @@ destroy_shared_heaps() while (heap) { cur = heap; heap = heap->next; - mem_allocator_destroy(cur->heap_handle); - wasm_runtime_free(cur->heap_handle); + if (cur->heap_handle) { + mem_allocator_destroy(cur->heap_handle); + wasm_runtime_free(cur->heap_handle); #ifndef OS_ENABLE_HW_BOUND_CHECK - map_size = cur->size; + map_size = cur->size; #else - map_size = 8 * (uint64)BH_GB; + map_size = 8 * (uint64)BH_GB; #endif - wasm_munmap_linear_memory(cur->base_addr, cur->size, map_size); + wasm_munmap_linear_memory(cur->base_addr, cur->size, map_size); + } wasm_runtime_free(cur); } os_mutex_destroy(&shared_heap_list_lock); @@ -761,7 +885,7 @@ wasm_runtime_validate_app_addr(WASMModuleInstanceCommon *module_inst_comm, #if WASM_ENABLE_SHARED_HEAP != 0 if (is_app_addr_in_shared_heap(module_inst_comm, memory_inst->is_memory64, - app_offset, size)) { + app_offset, size, NULL)) { return true; } #endif @@ -812,9 +936,9 @@ wasm_runtime_validate_app_str_addr(WASMModuleInstanceCommon *module_inst_comm, } #if WASM_ENABLE_SHARED_HEAP != 0 + WASMSharedHeap *shared_heap; if (is_app_addr_in_shared_heap(module_inst_comm, memory_inst->is_memory64, - app_str_offset, 1)) { - WASMSharedHeap *shared_heap = get_shared_heap(module_inst_comm); + app_str_offset, 1, &shared_heap)) { str = (char *)shared_heap->base_addr + (memory_inst->is_memory64 ? (app_str_offset - shared_heap->start_off_mem64) @@ -884,7 +1008,8 @@ wasm_runtime_validate_native_addr(WASMModuleInstanceCommon *module_inst_comm, } #if WASM_ENABLE_SHARED_HEAP != 0 - if (is_native_addr_in_shared_heap(module_inst_comm, native_ptr, size)) { + if (is_native_addr_in_shared_heap(module_inst_comm, native_ptr, size, + NULL)) { return true; } #endif @@ -910,6 +1035,7 @@ wasm_runtime_addr_app_to_native(WASMModuleInstanceCommon *module_inst_comm, { WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; WASMMemoryInstance *memory_inst; + WASMSharedHeap *shared_heap; uint8 *addr; bool bounds_checks; @@ -925,8 +1051,7 @@ wasm_runtime_addr_app_to_native(WASMModuleInstanceCommon *module_inst_comm, #if WASM_ENABLE_SHARED_HEAP != 0 if (is_app_addr_in_shared_heap(module_inst_comm, memory_inst->is_memory64, - app_offset, 1)) { - WASMSharedHeap *shared_heap = get_shared_heap(module_inst_comm); + app_offset, 1, &shared_heap)) { uint64 shared_heap_start = 0; if (memory_inst && !memory_inst->is_memory64) { @@ -985,8 +1110,9 @@ wasm_runtime_addr_native_to_app(WASMModuleInstanceCommon *module_inst_comm, } #if WASM_ENABLE_SHARED_HEAP != 0 - if (is_native_addr_in_shared_heap(module_inst_comm, addr, 1)) { - WASMSharedHeap *shared_heap = get_shared_heap(module_inst_comm); + WASMSharedHeap *shared_heap; + if (is_native_addr_in_shared_heap(module_inst_comm, addr, 1, + &shared_heap)) { uint64 shared_heap_start = 0; if (memory_inst && !memory_inst->is_memory64) { @@ -1112,8 +1238,7 @@ wasm_check_app_addr_and_convert(WASMModuleInstance *module_inst, bool is_str, #if WASM_ENABLE_SHARED_HEAP != 0 if (is_app_addr_in_shared_heap((WASMModuleInstanceCommon *)module_inst, memory_inst->is_memory64, app_buf_addr, - app_buf_size)) { - shared_heap = get_shared_heap((WASMModuleInstanceCommon *)module_inst); + app_buf_size, &shared_heap)) { native_addr = shared_heap->base_addr + (memory_inst->is_memory64 ? (app_buf_addr - shared_heap->start_off_mem64) diff --git a/core/iwasm/interpreter/wasm_interp_classic.c b/core/iwasm/interpreter/wasm_interp_classic.c index 98668470fb..e45a4a18f4 100644 --- a/core/iwasm/interpreter/wasm_interp_classic.c +++ b/core/iwasm/interpreter/wasm_interp_classic.c @@ -53,9 +53,34 @@ typedef float64 CellType_F64; #else #define is_default_memory true #endif -#define app_addr_in_shared_heap(app_addr, bytes) \ - (shared_heap && is_default_memory && (app_addr) >= shared_heap_start_off \ - && (app_addr) <= shared_heap_end_off - bytes + 1) +#if WASM_ENABLE_MEMORY64 +#define get_shared_heap_start_off(shared_heap) \ + (is_memory64 ? shared_heap->start_off_mem64 : shared_heap->start_off_mem32) +#else +#define get_shared_heap_start_off(shared_heap) (shared_heap->start_off_mem32) +#endif +#define app_addr_in_shared_heap(app_addr, bytes) \ + (shared_heap && is_default_memory && (app_addr) >= shared_heap_start_off \ + && (app_addr) <= shared_heap_end_off - bytes + 0) \ + || ({ \ + bool in_chain = false; \ + WASMSharedHeap *cur; \ + uint64 cur_shared_heap_start_off, cur_shared_heap_end_off; \ + for (cur = shared_heap; cur; cur = cur->chain_next) { \ + cur_shared_heap_start_off = get_shared_heap_start_off(cur); \ + cur_shared_heap_end_off = \ + cur_shared_heap_start_off + cur->size - 1; \ + if ((app_addr) >= cur_shared_heap_start_off \ + && (app_addr) <= cur_shared_heap_end_off - bytes + 0) { \ + shared_heap_start_off = cur_shared_heap_start_off; \ + shared_heap_end_off = cur_shared_heap_end_off; \ + shared_heap_base_addr = cur->base_addr; \ + in_chain = true; \ + break; \ + } \ + } \ + in_chain; \ + }) #define shared_heap_addr_app_to_native(app_addr, native_addr) \ native_addr = shared_heap_base_addr + ((app_addr)-shared_heap_start_off) @@ -1647,18 +1672,12 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, #if WASM_ENABLE_SHARED_HEAP != 0 WASMSharedHeap *shared_heap = module->e->shared_heap; uint8 *shared_heap_base_addr = shared_heap ? shared_heap->base_addr : NULL; -#if WASM_ENABLE_MEMORY64 != 0 uint64 shared_heap_start_off = - shared_heap ? (is_memory64 ? shared_heap->start_off_mem64 - : shared_heap->start_off_mem32) - : 0; + shared_heap ? get_shared_heap_start_off(shared_heap) : 0; uint64 shared_heap_end_off = - shared_heap ? (is_memory64 ? UINT64_MAX : UINT32_MAX) : 0; -#else - uint64 shared_heap_start_off = - shared_heap ? shared_heap->start_off_mem32 : 0; - uint64 shared_heap_end_off = shared_heap ? UINT32_MAX : 0; -#endif + shared_heap + ? (get_shared_heap_start_off(shared_heap) + shared_heap->size - 1) + : 0; #endif /* end of WASM_ENABLE_SHARED_HEAP != 0 */ #if WASM_ENABLE_MULTI_MEMORY != 0 uint32 memidx = 0; @@ -1697,7 +1716,10 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, goto got_exception; } - HANDLE_OP(WASM_OP_NOP) { HANDLE_OP_END(); } + HANDLE_OP(WASM_OP_NOP) + { + HANDLE_OP_END(); + } #if WASM_ENABLE_EXCE_HANDLING != 0 HANDLE_OP(WASM_OP_RETHROW) From 5c22e611cd2e4347374ce05b26351b0ae12d21c3 Mon Sep 17 00:00:00 2001 From: TL Date: Fri, 24 Jan 2025 14:45:08 +0800 Subject: [PATCH 03/20] first draft of shared heap enhancement in interpreter and runtime: update function signature --- core/iwasm/common/wasm_memory.c | 14 ++++++++---- core/iwasm/common/wasm_memory.h | 6 +++++ core/iwasm/include/wasm_export.h | 39 +++++++++++++++++++++++++++++--- 3 files changed, 52 insertions(+), 7 deletions(-) diff --git a/core/iwasm/common/wasm_memory.c b/core/iwasm/common/wasm_memory.c index a24b3c32d8..49df9acfb2 100644 --- a/core/iwasm/common/wasm_memory.c +++ b/core/iwasm/common/wasm_memory.c @@ -251,7 +251,7 @@ WASMSharedHeap * wasm_runtime_chain_shared_heaps(WASMSharedHeap *head, WASMSharedHeap *body) { WASMSharedHeap *cur; - bool heap_handle_exist = false; + bool heap_handle_exist = head->heap_handle != NULL; if (!head || !body) { LOG_WARNING("Invalid shared heap to chain."); @@ -265,8 +265,15 @@ wasm_runtime_chain_shared_heaps(WASMSharedHeap *head, WASMSharedHeap *body) os_mutex_unlock(&shared_heap_list_lock); return NULL; } - - for (cur = head; cur; cur = cur->chain_next) { + for (cur = shared_heap_list; cur; cur = cur->next) { + if (cur->chain_next == body) { + LOG_WARNING("To create shared heap chain, the `body` shared heap " + "can't already be in a chain"); + os_mutex_unlock(&shared_heap_list_lock); + return NULL; + } + } + for (cur = body; cur; cur = cur->chain_next) { if (cur->heap_handle && heap_handle_exist) { LOG_WARNING( "To create shared heap chain, only one of shared heap can " @@ -310,7 +317,6 @@ wasm_runtime_unchain_shared_heaps(WASMSharedHeap *head, bool entire_chain) if (!entire_chain) break; } - os_mutex_unlock(&shared_heap_list_lock); return cur; } diff --git a/core/iwasm/common/wasm_memory.h b/core/iwasm/common/wasm_memory.h index bceea0ee43..88d0f3c5df 100644 --- a/core/iwasm/common/wasm_memory.h +++ b/core/iwasm/common/wasm_memory.h @@ -45,6 +45,12 @@ SET_LINEAR_MEMORY_SIZE(WASMMemoryInstance *memory, uint64 size) WASMSharedHeap * wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args); +WASMSharedHeap * +wasm_runtime_chain_shared_heaps(WASMSharedHeap *head, WASMSharedHeap *body); + +WASMSharedHeap * +wasm_runtime_unchain_shared_heaps(WASMSharedHeap *head, bool entire_chain); + bool wasm_runtime_attach_shared_heap(WASMModuleInstanceCommon *module_inst, WASMSharedHeap *shared_heap); diff --git a/core/iwasm/include/wasm_export.h b/core/iwasm/include/wasm_export.h index 273657246d..15759f7775 100644 --- a/core/iwasm/include/wasm_export.h +++ b/core/iwasm/include/wasm_export.h @@ -336,6 +336,7 @@ typedef enum { typedef struct SharedHeapInitArgs { uint32_t size; + void *pre_allocated_addr; } SharedHeapInitArgs; /** @@ -2258,7 +2259,37 @@ WASM_RUNTIME_API_EXTERN wasm_shared_heap_t wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args); /** - * Attach a shared heap to a module instance + * This function links two shared heaps, `head` and `body`, where `head` is a + * shared heap and `body` can be shared heap chain head, into a single chain. + * The `head` heap will be the head of the chain, and the `body` heap will be + * appended to it. At most one shared heap in shared heap chain can be + * dynamically allocated, the rest have to be the pre-allocated shared heap * + * + * @param head The head of the shared heap chain. + * @param body The body of the shared heap chain to be appended. + * @return The new head of the shared heap chain. NULL if failed. + */ +WASM_RUNTIME_API_EXTERN wasm_shared_heap_t +wasm_runtime_chain_shared_heaps(wasm_shared_heap_t head, + wasm_shared_heap_t body); + +/** + * This function unchains the shared heaps from the given head. If + * `entire_chain` is true, it will unchain the entire chain of shared heaps. + * Otherwise, it will unchain only the first shared heap in the chain. + * + * @param head The head of the shared heap chain. + * @param entire_chain A boolean flag indicating whether to unchain the entire + * chain. + * @return The new head of the shared heap chain. Or the last shared heap in the + * chain if `entire_chain` is true. + */ +wasm_shared_heap_t +wasm_runtime_unchain_shared_heaps(wasm_shared_heap_t head, bool entire_chain); + +/** + * Attach a shared heap, it can be the head of shared heap chain, in that case, + * attach the shared heap chain, to a module instance * * @param module_inst the module instance * @param shared_heap the shared heap @@ -2277,7 +2308,8 @@ WASM_RUNTIME_API_EXTERN void wasm_runtime_detach_shared_heap(wasm_module_inst_t module_inst); /** - * Allocate memory from a shared heap + * Allocate memory from a shared heap, or the non-preallocated shared heap from + * the shared heap chain * * @param module_inst the module instance * @param size required memory size @@ -2294,7 +2326,8 @@ wasm_runtime_shared_heap_malloc(wasm_module_inst_t module_inst, uint64_t size, void **p_native_addr); /** - * Free the memory allocated from shared heap + * Free the memory allocated from shared heap, or the non-preallocated shared + * heap from the shared heap chain * * @param module_inst the module instance * @param ptr the offset in wasm app From 1566ff1606e1d35d897b866ac25560c6acddd50f Mon Sep 17 00:00:00 2001 From: TL Date: Sun, 26 Jan 2025 11:11:40 +0800 Subject: [PATCH 04/20] first draft of shared heap enhancement in interpreter and runtime: update interpreter --- core/iwasm/common/wasm_memory.c | 2 +- core/iwasm/interpreter/wasm_interp_classic.c | 14 +++--- core/iwasm/interpreter/wasm_interp_fast.c | 45 ++++++++++++++------ core/iwasm/interpreter/wasm_runtime.h | 9 ++++ 4 files changed, 48 insertions(+), 22 deletions(-) diff --git a/core/iwasm/common/wasm_memory.c b/core/iwasm/common/wasm_memory.c index 49df9acfb2..d3d7e2a51e 100644 --- a/core/iwasm/common/wasm_memory.c +++ b/core/iwasm/common/wasm_memory.c @@ -504,7 +504,7 @@ is_app_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst, for (cur = heap; cur; cur = cur->chain_next) { shared_heap_start = - cur->start_off_mem64 ? is_memory64 : cur->start_off_mem32; + is_memory64 ? cur->start_off_mem64 : cur->start_off_mem32; shared_heap_end = shared_heap_start + cur->size; if (app_offset >= shared_heap_start && app_offset <= shared_heap_end) { if (target_heap) diff --git a/core/iwasm/interpreter/wasm_interp_classic.c b/core/iwasm/interpreter/wasm_interp_classic.c index e45a4a18f4..d75fc55e7e 100644 --- a/core/iwasm/interpreter/wasm_interp_classic.c +++ b/core/iwasm/interpreter/wasm_interp_classic.c @@ -53,15 +53,18 @@ typedef float64 CellType_F64; #else #define is_default_memory true #endif -#if WASM_ENABLE_MEMORY64 +#if WASM_ENABLE_MEMORY64 != 0 #define get_shared_heap_start_off(shared_heap) \ (is_memory64 ? shared_heap->start_off_mem64 : shared_heap->start_off_mem32) #else #define get_shared_heap_start_off(shared_heap) (shared_heap->start_off_mem32) #endif +/* Check whether the app addr in the last visited shared heap, if not, check the + * shared heap chain to find which(if any) shared heap the app addr in, and + * update the last visited shared heap info if found. */ #define app_addr_in_shared_heap(app_addr, bytes) \ (shared_heap && is_default_memory && (app_addr) >= shared_heap_start_off \ - && (app_addr) <= shared_heap_end_off - bytes + 0) \ + && (app_addr) <= shared_heap_end_off - bytes + 1) \ || ({ \ bool in_chain = false; \ WASMSharedHeap *cur; \ @@ -71,7 +74,7 @@ typedef float64 CellType_F64; cur_shared_heap_end_off = \ cur_shared_heap_start_off + cur->size - 1; \ if ((app_addr) >= cur_shared_heap_start_off \ - && (app_addr) <= cur_shared_heap_end_off - bytes + 0) { \ + && (app_addr) <= cur_shared_heap_end_off - bytes + 1) { \ shared_heap_start_off = cur_shared_heap_start_off; \ shared_heap_end_off = cur_shared_heap_end_off; \ shared_heap_base_addr = cur->base_addr; \ @@ -1716,10 +1719,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, goto got_exception; } - HANDLE_OP(WASM_OP_NOP) - { - HANDLE_OP_END(); - } + HANDLE_OP(WASM_OP_NOP) { HANDLE_OP_END(); } #if WASM_ENABLE_EXCE_HANDLING != 0 HANDLE_OP(WASM_OP_RETHROW) diff --git a/core/iwasm/interpreter/wasm_interp_fast.c b/core/iwasm/interpreter/wasm_interp_fast.c index 359a6979c7..3a9715a66f 100644 --- a/core/iwasm/interpreter/wasm_interp_fast.c +++ b/core/iwasm/interpreter/wasm_interp_fast.c @@ -38,9 +38,33 @@ typedef float64 CellType_F64; #endif #if WASM_ENABLE_SHARED_HEAP != 0 -#define app_addr_in_shared_heap(app_addr, bytes) \ - (shared_heap && (app_addr) >= shared_heap_start_off \ - && (app_addr) <= shared_heap_end_off - bytes + 1) +/* TODO: add code for 64 bit version when memory64 is enabled for fast-interp */ +#define get_shared_heap_start_off(shared_heap) (shared_heap->start_off_mem32) +/* Check whether the app addr in the last visited shared heap, if not, check the + * shared heap chain to find which(if any) shared heap the app addr in, and + * update the last visited shared heap info if found. */ +#define app_addr_in_shared_heap(app_addr, bytes) \ + (shared_heap && (app_addr) >= shared_heap_start_off \ + && (app_addr) <= shared_heap_end_off - bytes + 1) \ + || ({ \ + bool in_chain = false; \ + WASMSharedHeap *cur; \ + uint64 cur_shared_heap_start_off, cur_shared_heap_end_off; \ + for (cur = shared_heap; cur; cur = cur->chain_next) { \ + cur_shared_heap_start_off = get_shared_heap_start_off(cur); \ + cur_shared_heap_end_off = \ + cur_shared_heap_start_off + cur->size - 1; \ + if ((app_addr) >= cur_shared_heap_start_off \ + && (app_addr) <= cur_shared_heap_end_off - bytes + 1) { \ + shared_heap_start_off = cur_shared_heap_start_off; \ + shared_heap_end_off = cur_shared_heap_end_off; \ + shared_heap_base_addr = cur->base_addr; \ + in_chain = true; \ + break; \ + } \ + } \ + in_chain; \ + }) #define shared_heap_addr_app_to_native(app_addr, native_addr) \ native_addr = shared_heap_base_addr + ((app_addr)-shared_heap_start_off) @@ -1540,19 +1564,12 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, #if WASM_ENABLE_SHARED_HEAP != 0 WASMSharedHeap *shared_heap = module->e ? module->e->shared_heap : NULL; uint8 *shared_heap_base_addr = shared_heap ? shared_heap->base_addr : NULL; - /* -#if WASM_ENABLE_MEMORY64 != 0 uint64 shared_heap_start_off = - shared_heap ? (is_memory64 ? shared_heap->start_off_mem64 - : shared_heap->start_off_mem32) - : 0; + shared_heap ? get_shared_heap_start_off(shared_heap) : 0; uint64 shared_heap_end_off = - shared_heap ? (is_memory64 ? UINT64_MAX : UINT32_MAX) : 0; -#else - */ /* TODO: uncomment the code when memory64 is enabled for fast-interp */ - uint64 shared_heap_start_off = - shared_heap ? shared_heap->start_off_mem32 : 0; - uint64 shared_heap_end_off = shared_heap ? UINT32_MAX : 0; + shared_heap + ? (get_shared_heap_start_off(shared_heap) + shared_heap->size - 1) + : 0; /* #endif */ #endif /* end of WASM_ENABLE_SHARED_HEAP != 0 */ diff --git a/core/iwasm/interpreter/wasm_runtime.h b/core/iwasm/interpreter/wasm_runtime.h index 00e9ad107f..b9d49ec229 100644 --- a/core/iwasm/interpreter/wasm_runtime.h +++ b/core/iwasm/interpreter/wasm_runtime.h @@ -93,12 +93,21 @@ typedef union { } MemBound; typedef struct WASMSharedHeap { + /* The global shared heap list maintained in runtime, used for runtime + * destroy */ struct WASMSharedHeap *next; + /* The logical shared heap chain the shared heap in */ + struct WASMSharedHeap *chain_next; + /* Will be null if shared heap is created from pre allocated memory chunk + * and don't need to dynamic malloc and free */ void *heap_handle; uint8 *base_addr; uint64 size; uint64 start_off_mem64; uint64 start_off_mem32; + /* The number of wasm apps it attached to, for a shared heap chain, only the + * list head need to maintain the valid attached_count */ + uint8 attached_count; } WASMSharedHeap; struct WASMMemoryInstance { From aebb391cf5ddc622b32577f656befc090df222ff Mon Sep 17 00:00:00 2001 From: TL Date: Sun, 26 Jan 2025 11:12:49 +0800 Subject: [PATCH 05/20] unify shared heap sample --- .../shared-heap-preallocated/CMakeLists.txt | 127 ------- samples/shared-heap-preallocated/src/main.c | 335 ------------------ .../wasm-apps/CMakeLists.txt | 43 --- .../wasm-apps/test1.c | 71 ---- .../wasm-apps/test2.c | 18 - 5 files changed, 594 deletions(-) delete mode 100644 samples/shared-heap-preallocated/CMakeLists.txt delete mode 100644 samples/shared-heap-preallocated/src/main.c delete mode 100644 samples/shared-heap-preallocated/wasm-apps/CMakeLists.txt delete mode 100644 samples/shared-heap-preallocated/wasm-apps/test1.c delete mode 100644 samples/shared-heap-preallocated/wasm-apps/test2.c diff --git a/samples/shared-heap-preallocated/CMakeLists.txt b/samples/shared-heap-preallocated/CMakeLists.txt deleted file mode 100644 index 6346d077e7..0000000000 --- a/samples/shared-heap-preallocated/CMakeLists.txt +++ /dev/null @@ -1,127 +0,0 @@ -# Copyright (C) 2019 Intel Corporation. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -cmake_minimum_required (VERSION 3.14) - -include(CheckPIESupported) - -if (NOT WAMR_BUILD_PLATFORM STREQUAL "windows") - project (shared_heap_test) -else() - project (shared_heap_test C ASM) -endif() - -################ runtime settings ################ -string (TOLOWER ${CMAKE_HOST_SYSTEM_NAME} WAMR_BUILD_PLATFORM) -if (APPLE) - add_definitions(-DBH_PLATFORM_DARWIN) -endif () - -# Reset default linker flags -set (CMAKE_SHARED_LIBRARY_LINK_C_FLAGS "") -set (CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS "") - -# WAMR features switch - -# Set WAMR_BUILD_TARGET, currently values supported: -# "X86_64", "AMD_64", "X86_32", "AARCH64[sub]", "ARM[sub]", "THUMB[sub]", -# "MIPS", "XTENSA", "RISCV64[sub]", "RISCV32[sub]" - -if (NOT DEFINED WAMR_BUILD_TARGET) - if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(arm64|aarch64)") - set (WAMR_BUILD_TARGET "AARCH64") - elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "riscv64") - set (WAMR_BUILD_TARGET "RISCV64") - elseif (CMAKE_SIZEOF_VOID_P EQUAL 8) - # Build as X86_64 by default in 64-bit platform - set (WAMR_BUILD_TARGET "X86_64") - elseif (CMAKE_SIZEOF_VOID_P EQUAL 4) - # Build as X86_32 by default in 32-bit platform - set (WAMR_BUILD_TARGET "X86_32") - else () - message(SEND_ERROR "Unsupported build target platform!") - endif () -endif () - -if (NOT CMAKE_BUILD_TYPE) - set (CMAKE_BUILD_TYPE Debug) -endif () - -set (WAMR_BUILD_INTERP 1) -set (WAMR_BUILD_FAST_INTERP 1) -set (WAMR_BUILD_AOT 1) -set (WAMR_BUILD_JIT 0) -set (WAMR_BUILD_LIBC_BUILTIN 1) -set (WAMR_BUILD_LIBC_WASI 0) -set (WAMR_BUILD_SHARED_HEAP 1) -set (WAMR_BUILD_GC_HEAP_VERIFY 1) - -if (NOT MSVC) - # linker flags - if (NOT (CMAKE_C_COMPILER MATCHES ".*clang.*" OR CMAKE_C_COMPILER_ID MATCHES ".*Clang")) - set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gc-sections") - endif () - set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra -Wformat -Wformat-security") - if (WAMR_BUILD_TARGET MATCHES "X86_.*" OR WAMR_BUILD_TARGET STREQUAL "AMD_64") - if (NOT (CMAKE_C_COMPILER MATCHES ".*clang.*" OR CMAKE_C_COMPILER_ID MATCHES ".*Clang")) - set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mindirect-branch-register") - endif () - endif () -endif () - -# build out vmlib -set (WAMR_ROOT_DIR ${CMAKE_CURRENT_LIST_DIR}/../..) -include (${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake) - -add_library(vmlib STATIC ${WAMR_RUNTIME_LIB_SOURCE}) -if (MSVC) - target_compile_definitions(vmlib PRIVATE WASM_API_EXTERN=) -endif() -target_link_libraries(vmlib ${LLVM_AVAILABLE_LIBS} ${UV_A_LIBS} -lm -ldl -lpthread) - -################ application related ################ -include_directories(${CMAKE_CURRENT_LIST_DIR}/src) -include (${SHARED_DIR}/utils/uncommon/shared_uncommon.cmake) - -add_executable (shared_heap_test src/main.c ${UNCOMMON_SHARED_SOURCE}) - -check_pie_supported() -set_target_properties (shared_heap_test PROPERTIES POSITION_INDEPENDENT_CODE ON) - -if (APPLE) - target_link_libraries (shared_heap_test vmlib -lm -ldl -lpthread) -else () - target_link_libraries (shared_heap_test vmlib -lm -ldl -lpthread -lrt) -endif () - -add_subdirectory(wasm-apps) - -if (WAMR_BUILD_AOT EQUAL 1) - set (WAMR_COMPILER_DIR ${CMAKE_CURRENT_LIST_DIR}/../../wamr-compiler/build) - message (CHECK_START "Detecting WAMR_COMPILER at ${WAMR_COMPILER_DIR}") - find_file (WAMR_COMPILER - wamrc - PATHS "${CMAKE_CURRENT_LIST_DIR}/../../wamr-compiler/build" - NO_DEFAULT_PATH - NO_CMAKE_FIND_ROOT_PATH - ) - if (WAMR_COMPILER) - message (CHECK_PASS "found") - else() - message (CHECK_FAIL "not found") - endif() - if (NOT EXISTS ${WAMR_COMPILER}) - message (FATAL_ERROR "Please build wamrc under ${WAMR_ROOT_DIR}/wamr-compiler") - else() - message (STATUS "WAMR_COMPILER is ${WAMR_COMPILER}") - endif() - - add_custom_target( - wasm_to_aot - ALL - DEPENDS wasm-apps/test1.wasm wasm-apps/test2.wasm ${WAMR_COMPILER} - COMMAND ${WAMR_COMPILER} --enable-shared-heap -o wasm-apps/test1.aot wasm-apps/test1.wasm - COMMAND ${WAMR_COMPILER} --enable-shared-heap -o wasm-apps/test2.aot wasm-apps/test2.wasm - WORKING_DIRECTORY ${CMAKE_BINARY_DIR} - ) -endif() diff --git a/samples/shared-heap-preallocated/src/main.c b/samples/shared-heap-preallocated/src/main.c deleted file mode 100644 index 9842f33c30..0000000000 --- a/samples/shared-heap-preallocated/src/main.c +++ /dev/null @@ -1,335 +0,0 @@ -/* - * Copyright (C) 2019 Intel Corporation. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - */ - -#include "wasm_export.h" -#include "bh_platform.h" -#include "bh_read_file.h" - -typedef struct thread_arg { - bh_queue *queue; - wasm_module_inst_t module_inst; -} thread_arg; - -static void * -thread1_callback(void *arg) -{ - thread_arg *targ = arg; - wasm_module_inst_t module_inst = targ->module_inst; - bh_queue *queue = targ->queue; - wasm_exec_env_t exec_env; - wasm_function_inst_t my_shared_heap_malloc_func; - wasm_function_inst_t my_shared_heap_free_func; - uint32 i, argv[2]; - - /* lookup wasm functions */ - if (!(my_shared_heap_malloc_func = wasm_runtime_lookup_function( - module_inst, "my_shared_heap_malloc")) - || !(my_shared_heap_free_func = wasm_runtime_lookup_function( - module_inst, "my_shared_heap_free"))) { - printf("Failed to lookup function.\n"); - } - - /* create exec env */ - if (!(exec_env = wasm_runtime_create_exec_env(module_inst, 32768))) { - printf("Failed to create exec env.\n"); - return NULL; - } - - /* allocate memory with wasm_runtime_shared_heap_malloc and send it - to wasm app2 */ - for (i = 0; i < 1; i++) { - uint8 *buf; - uint64 offset; - - offset = wasm_runtime_shared_heap_malloc(module_inst, 4096 * (i + 1), - (void **)&buf); - - if (offset == 0) { - printf("Failed to allocate memory from shared heap\n"); - break; - } - - snprintf(buf, 1024, "Hello, this is buf %u allocated from shared heap", - i + 1); - - printf("wasm app1 send buf: %s\n\n", buf); - if (!bh_post_msg(queue, 1, buf, 1024 * i)) { - printf("Failed to post message to queue\n"); - wasm_runtime_shared_heap_free(module_inst, offset); - break; - } - } - - /* allocate memory by calling my_shared_heap_malloc function and send it - to wasm app2 */ - for (i = 1; i < 2; i++) { - uint8 *buf; - - argv[0] = 1024 * (i + 1); - argv[1] = i + 1; - wasm_runtime_call_wasm(exec_env, my_shared_heap_malloc_func, 2, argv); - - if (wasm_runtime_get_exception(module_inst)) { - printf("Failed to call 'my_shared_heap_malloc' function: %s\n", - wasm_runtime_get_exception(module_inst)); - break; - } - if (argv[0] == 0) { - printf("Failed to allocate memory from shared heap\n"); - break; - } - - buf = wasm_runtime_addr_app_to_native(module_inst, argv[0]); - - printf("wasm app1 send buf: %s\n\n", buf); - if (!bh_post_msg(queue, 1, buf, 1024 * i)) { - printf("Failed to post message to queue\n"); - wasm_runtime_shared_heap_free(module_inst, argv[0]); - break; - } - } - - wasm_runtime_destroy_exec_env(exec_env); - - return NULL; -} - -static void -queue_callback(void *message, void *arg) -{ - bh_message_t msg = (bh_message_t)message; - wasm_exec_env_t exec_env = arg; - wasm_module_inst_t module_inst = wasm_runtime_get_module_inst(exec_env); - wasm_function_inst_t print_buf_func; - uint32 argv[2]; - - /* lookup wasm function */ - if (!(print_buf_func = - wasm_runtime_lookup_function(module_inst, "print_buf"))) { - printf("Failed to lookup function.\n"); - return; - } - - char *buf = bh_message_payload(msg); - printf("wasm app's native queue received buf: %s\n\n", buf); - - /* call wasm function */ - argv[0] = wasm_runtime_addr_native_to_app(module_inst, buf); - wasm_runtime_call_wasm(exec_env, print_buf_func, 1, argv); - if (wasm_runtime_get_exception(module_inst)) { - printf("Failed to call 'print_buf' function: %s\n", - wasm_runtime_get_exception(module_inst)); - } -} - -static void * -thread2_callback(void *arg) -{ - thread_arg *targ = arg; - bh_queue *queue = targ->queue; - wasm_module_inst_t module_inst = targ->module_inst; - wasm_exec_env_t exec_env; - - /* create exec env */ - if (!(exec_env = wasm_runtime_create_exec_env(module_inst, 32768))) { - printf("Failed to create exec env.\n"); - return NULL; - } - - /* enter queue's message loop until bh_queue_exit_loop_run - is called */ - bh_queue_enter_loop_run(queue, queue_callback, exec_env); - - wasm_runtime_destroy_exec_env(exec_env); - - return NULL; -} - -static char global_heap_buf[512 * 1024]; - -int -main(int argc, char **argv) -{ - char *wasm_file1 = NULL, *wasm_file2 = NULL; - uint8 *wasm_file1_buf = NULL, *wasm_file2_buf = NULL; - uint32 wasm_file1_size, wasm_file2_size; - wasm_module_t wasm_module1 = NULL, wasm_module2 = NULL; - wasm_module_inst_t module_inst1 = NULL; - wasm_module_inst_t module_inst2 = NULL; - wasm_shared_heap_t shared_heap = NULL; - bh_queue *queue = NULL; - RuntimeInitArgs init_args; - SharedHeapInitArgs heap_init_args; - char error_buf[128] = { 0 }; - bool aot_mode = false; - int ret = -1; - -#if WASM_DISABLE_HW_BOUND_CHECK == 0 -#if defined(BUILD_TARGET_X86_64) - printf("HW BOUND CHECK\n"); -#endif -#endif - - if (argc > 1 && !strcmp(argv[1], "--aot")) - aot_mode = true; - - if (!aot_mode) - printf("Test shared heap in interpreter mode\n\n"); - else - printf("Test shared heap in AOT mode\n\n"); - - bh_log_set_verbose_level(5); - memset(&init_args, 0, sizeof(RuntimeInitArgs)); - - init_args.mem_alloc_type = Alloc_With_Pool; - init_args.mem_alloc_option.pool.heap_buf = global_heap_buf; - init_args.mem_alloc_option.pool.heap_size = sizeof(global_heap_buf); - - /* init wasm runtime */ - if (!wasm_runtime_full_init(&init_args)) { - printf("Init runtime environment failed.\n"); - return -1; - } - - /* create queue */ - if (!(queue = bh_queue_create())) { - printf("Create queue failed.\n"); - goto fail; - } - - /* read wasm file */ - if (!aot_mode) - wasm_file1 = "./wasm-apps/test1.wasm"; - else - wasm_file1 = "./wasm-apps/test1.aot"; - if (!(wasm_file1_buf = - bh_read_file_to_buffer(wasm_file1, &wasm_file1_size))) { - printf("Open wasm file %s failed.\n", wasm_file1); - goto fail; - } - - /* load wasm file */ - wasm_module1 = wasm_runtime_load((uint8 *)wasm_file1_buf, wasm_file1_size, - error_buf, sizeof(error_buf)); - if (!wasm_module1) { - printf("Load wasm module failed. error: %s\n", error_buf); - goto fail; - } - - /* instantiate module */ - module_inst1 = wasm_runtime_instantiate(wasm_module1, 65536, 4096, - error_buf, sizeof(error_buf)); - if (!module_inst1) { - printf("Instantiate wasm module failed. error: %s\n", error_buf); - goto fail; - } - - /* read wasm file */ - if (!aot_mode) - wasm_file2 = "./wasm-apps/test2.wasm"; - else - wasm_file2 = "./wasm-apps/test2.aot"; - if (!(wasm_file2_buf = - bh_read_file_to_buffer(wasm_file2, &wasm_file2_size))) { - printf("Open wasm file %s failed.\n", wasm_file1); - goto fail; - } - - /* load wasm file */ - wasm_module2 = wasm_runtime_load((uint8 *)wasm_file2_buf, wasm_file2_size, - error_buf, sizeof(error_buf)); - if (!wasm_module2) { - printf("Load wasm module failed. error: %s\n", error_buf); - goto fail; - } - - /* instantiate module */ - module_inst2 = wasm_runtime_instantiate(wasm_module2, 65536, 4096, - error_buf, sizeof(error_buf)); - if (!module_inst2) { - printf("Instantiate wasm module failed. error: %s\n", error_buf); - goto fail; - } - - /* create shared heap */ - memset(&heap_init_args, 0, sizeof(heap_init_args)); - heap_init_args.size = 4096 + 1; - shared_heap = wasm_runtime_create_shared_heap(&heap_init_args); - if (!shared_heap) { - printf("Create shared heap failed. error: %s\n", error_buf); - goto fail; - } - - /* attach module instance 1 to the shared heap */ - if (!wasm_runtime_attach_shared_heap(module_inst1, shared_heap)) { - printf("Attach shared heap failed.\n"); - goto fail; - } - - /* attach module instance 2 to the shared heap */ - if (!wasm_runtime_attach_shared_heap(module_inst2, shared_heap)) { - printf("Attach shared heap failed.\n"); - goto fail; - } - - /* create thread 1 */ - struct thread_arg targ1 = { 0 }; - korp_tid tid1; - targ1.queue = queue; - targ1.module_inst = module_inst1; - if (os_thread_create(&tid1, thread1_callback, &targ1, - APP_THREAD_STACK_SIZE_DEFAULT)) { - printf("Failed to create thread 1\n"); - goto fail; - } - - /* create thread 2 */ - struct thread_arg targ2 = { 0 }; - korp_tid tid2; - targ2.queue = queue; - targ2.module_inst = module_inst2; - if (os_thread_create(&tid2, thread2_callback, &targ2, - APP_THREAD_STACK_SIZE_DEFAULT)) { - printf("Failed to create thread 2\n"); - os_thread_join(tid1, NULL); - goto fail; - } - - /* wait until all messages are post to wasm app2 and wasm app2 - handles all of them, then exit the queue message loop */ - usleep(10000); - bh_queue_exit_loop_run(queue); - - os_thread_join(tid1, NULL); - os_thread_join(tid2, NULL); - - ret = 0; - -fail: - if (module_inst2) - wasm_runtime_deinstantiate(module_inst2); - - if (module_inst1) - wasm_runtime_deinstantiate(module_inst1); - - if (wasm_module2) - wasm_runtime_unload(wasm_module2); - - if (wasm_module1) - wasm_runtime_unload(wasm_module1); - - if (wasm_file2_buf) - wasm_runtime_free(wasm_file2_buf); - - if (wasm_file1_buf) - wasm_runtime_free(wasm_file1_buf); - - if (queue) - bh_queue_destroy(queue); - - wasm_runtime_destroy(); - - return ret; -} diff --git a/samples/shared-heap-preallocated/wasm-apps/CMakeLists.txt b/samples/shared-heap-preallocated/wasm-apps/CMakeLists.txt deleted file mode 100644 index a90c82e1ad..0000000000 --- a/samples/shared-heap-preallocated/wasm-apps/CMakeLists.txt +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright (C) 2019 Intel Corporation. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -cmake_minimum_required(VERSION 3.14) -project(wasm-apps) - -set(WAMR_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../..) - -if (APPLE) - set (HAVE_FLAG_SEARCH_PATHS_FIRST 0) - set (CMAKE_C_LINK_FLAGS "") - set (CMAKE_CXX_LINK_FLAGS "") -endif () - -set (CMAKE_SYSTEM_PROCESSOR wasm32) -set (CMAKE_SYSROOT ${WAMR_ROOT_DIR}/wamr-sdk/app/libc-builtin-sysroot) - -if (NOT DEFINED WASI_SDK_DIR) - set (WASI_SDK_DIR "/opt/wasi-sdk") -endif () - -set (CMAKE_C_COMPILER_TARGET "wasm32") -set (CMAKE_C_COMPILER "${WASI_SDK_DIR}/bin/clang") -set (CMAKE_C_FLAGS "-nostdlib -Qunused-arguments -z stack-size=32768") - -set (DEFINED_SYMBOLS "${WAMR_ROOT_DIR}/wamr-sdk/app/libc-builtin-sysroot/share/defined-symbols.txt") - -set (CMAKE_EXE_LINKER_FLAGS - "-O0 -Wl,--initial-memory=65536, \ - -Wl,--no-entry,--strip-all, \ - -Wl,--export=__heap_base,--export=__data_end \ - -Wl,--export=__wasm_call_ctors \ - -Wl,--export=my_shared_heap_malloc \ - -Wl,--export=my_shared_heap_free \ - -Wl,--export=print_buf \ - -Wl,--allow-undefined" -) - -add_executable(test1.wasm test1.c) -target_link_libraries(test1.wasm) - -add_executable(test2.wasm test2.c) -target_link_libraries(test2.wasm) diff --git a/samples/shared-heap-preallocated/wasm-apps/test1.c b/samples/shared-heap-preallocated/wasm-apps/test1.c deleted file mode 100644 index 2e3fd2112e..0000000000 --- a/samples/shared-heap-preallocated/wasm-apps/test1.c +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (C) 2019 Intel Corporation. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - */ - -#include -#include -#include -#include - -extern void * -shared_heap_malloc(uint32_t size); -extern void -shared_heap_free(void *ptr); - -void * -my_shared_heap_malloc(uint32_t size, uint32_t index) -{ - char *buf1 = NULL, *buf2 = NULL, *buf; - - char *buf3 = NULL; - buf3 = malloc(2048); - - buf1 = shared_heap_malloc(1024); - if (!buf1) - return NULL; - - buf1[0] = 'H'; - buf1[1] = 'e'; - buf1[2] = 'l'; - buf1[3] = 'l'; - buf1[4] = 'o'; - buf1[5] = ','; - buf1[6] = ' '; - - buf2 = shared_heap_malloc(1024); - if (!buf2) { - shared_heap_free(buf1); - return NULL; - } - - snprintf(buf2, 1024, "this is buf %u allocated from shared heap", index); - - buf = shared_heap_malloc(size); - if (!buf) { - shared_heap_free(buf1); - shared_heap_free(buf2); - return NULL; - } - - memset(buf, 0, size); - memcpy(buf, buf1, strlen(buf1)); - memcpy(buf + strlen(buf1), buf2, strlen(buf2)); - - uint32_t max_value = UINT32_MAX; - char *address = (char *)max_value; - printf("The address of buf1, buf2, buf, UINT32_MAX is: %p,%p,%p,%p\n", buf1, - buf2, buf, address); - address[0] = 'c'; - printf("The content address of UINT32_MAX is: %c\n", *address); - - shared_heap_free(buf1); - shared_heap_free(buf2); - return buf; -} - -void -my_shared_heap_free(void *ptr) -{ - shared_heap_free(ptr); -} diff --git a/samples/shared-heap-preallocated/wasm-apps/test2.c b/samples/shared-heap-preallocated/wasm-apps/test2.c deleted file mode 100644 index b63efcd1a2..0000000000 --- a/samples/shared-heap-preallocated/wasm-apps/test2.c +++ /dev/null @@ -1,18 +0,0 @@ -/* - * Copyright (C) 2019 Intel Corporation. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - */ - -#include - -#include - -extern void -shared_heap_free(void *ptr); - -void -print_buf(char *buf) -{ - printf("wasm app2's wasm func received buf: %s\n\n", buf); - shared_heap_free(buf); -} From add69585b9a267de20bf9fd2cdf1b06133ab23f9 Mon Sep 17 00:00:00 2001 From: TL Date: Mon, 27 Jan 2025 14:12:22 +0800 Subject: [PATCH 06/20] fix shared heap enhancement bugs and format code --- core/iwasm/common/wasm_memory.c | 33 +++++++------ core/iwasm/interpreter/wasm_interp_classic.c | 14 ++++-- core/iwasm/interpreter/wasm_interp_fast.c | 6 +-- samples/shared-heap/src/main.c | 8 +-- tests/unit/shared-heap/shared_heap_test.cc | 52 ++++++++++---------- 5 files changed, 61 insertions(+), 52 deletions(-) diff --git a/core/iwasm/common/wasm_memory.c b/core/iwasm/common/wasm_memory.c index d3d7e2a51e..ec57e9e67b 100644 --- a/core/iwasm/common/wasm_memory.c +++ b/core/iwasm/common/wasm_memory.c @@ -183,6 +183,11 @@ wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args) heap->start_off_mem32 = UINT32_MAX - heap->size + 1; heap->attached_count = 0; + if (size > APP_HEAP_SIZE_MAX || size < APP_HEAP_SIZE_MIN) { + LOG_WARNING("Invalid size of shared heap"); + goto fail1; + } + if (init_args->pre_allocated_addr != NULL) { /* Create shared heap from a pre allocated buffer, its size need to * align with system page */ @@ -193,6 +198,7 @@ wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args) } heap->heap_handle = NULL; + heap->base_addr = init_args->pre_allocated_addr; } else { if (!(heap->heap_handle = @@ -200,11 +206,6 @@ wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args) goto fail2; } - if (size > APP_HEAP_SIZE_MAX || size < APP_HEAP_SIZE_MIN) { - LOG_WARNING("Invalid size of shared heap"); - goto fail3; - } - #ifndef OS_ENABLE_HW_BOUND_CHECK map_size = size; #else @@ -286,8 +287,8 @@ wasm_runtime_chain_shared_heaps(WASMSharedHeap *head, WASMSharedHeap *body) heap_handle_exist = true; } - head->start_off_mem64 = body->start_off_mem64 - head->size + 1; - head->start_off_mem32 = body->start_off_mem32 - head->size + 1; + head->start_off_mem64 = body->start_off_mem64 - head->size; + head->start_off_mem32 = body->start_off_mem32 - head->size; head->chain_next = body; os_mutex_unlock(&shared_heap_list_lock); return head; @@ -505,8 +506,9 @@ is_app_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst, for (cur = heap; cur; cur = cur->chain_next) { shared_heap_start = is_memory64 ? cur->start_off_mem64 : cur->start_off_mem32; - shared_heap_end = shared_heap_start + cur->size; - if (app_offset >= shared_heap_start && app_offset <= shared_heap_end) { + shared_heap_end = shared_heap_start - 1 + cur->size; + if (app_offset >= shared_heap_start + && app_offset <= shared_heap_end - bytes + 1) { if (target_heap) *target_heap = cur; return true; @@ -535,19 +537,16 @@ is_native_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst, for (cur = heap_head; cur != NULL; cur = cur->chain_next) { base_addr = (uintptr_t)cur->base_addr; addr_int = (uintptr_t)addr; - if (addr_int < base_addr) { + if (addr_int < base_addr) continue; - } end_addr = addr_int + bytes; /* Check for overflow */ - if (end_addr <= addr_int) { + if (end_addr <= addr_int) continue; - } - if (end_addr > base_addr + cur->size) { + if (end_addr > base_addr + cur->size) continue; - } if (target_heap) *target_heap = cur; @@ -1041,9 +1040,11 @@ wasm_runtime_addr_app_to_native(WASMModuleInstanceCommon *module_inst_comm, { WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; WASMMemoryInstance *memory_inst; - WASMSharedHeap *shared_heap; uint8 *addr; bool bounds_checks; +#if WASM_ENABLE_SHARED_HEAP != 0 + WASMSharedHeap *shared_heap; +#endif bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode || module_inst_comm->module_type == Wasm_Module_AoT); diff --git a/core/iwasm/interpreter/wasm_interp_classic.c b/core/iwasm/interpreter/wasm_interp_classic.c index d75fc55e7e..76803cca74 100644 --- a/core/iwasm/interpreter/wasm_interp_classic.c +++ b/core/iwasm/interpreter/wasm_interp_classic.c @@ -72,7 +72,7 @@ typedef float64 CellType_F64; for (cur = shared_heap; cur; cur = cur->chain_next) { \ cur_shared_heap_start_off = get_shared_heap_start_off(cur); \ cur_shared_heap_end_off = \ - cur_shared_heap_start_off + cur->size - 1; \ + cur_shared_heap_start_off - 1 + cur->size; \ if ((app_addr) >= cur_shared_heap_start_off \ && (app_addr) <= cur_shared_heap_end_off - bytes + 1) { \ shared_heap_start_off = cur_shared_heap_start_off; \ @@ -1679,7 +1679,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, shared_heap ? get_shared_heap_start_off(shared_heap) : 0; uint64 shared_heap_end_off = shared_heap - ? (get_shared_heap_start_off(shared_heap) + shared_heap->size - 1) + ? (get_shared_heap_start_off(shared_heap) - 1 + shared_heap->size) : 0; #endif /* end of WASM_ENABLE_SHARED_HEAP != 0 */ #if WASM_ENABLE_MULTI_MEMORY != 0 @@ -1719,7 +1719,10 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, goto got_exception; } - HANDLE_OP(WASM_OP_NOP) { HANDLE_OP_END(); } + HANDLE_OP(WASM_OP_NOP) + { + HANDLE_OP_END(); + } #if WASM_ENABLE_EXCE_HANDLING != 0 HANDLE_OP(WASM_OP_RETHROW) @@ -5656,7 +5659,10 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, HANDLE_OP(WASM_OP_I32_REINTERPRET_F32) HANDLE_OP(WASM_OP_I64_REINTERPRET_F64) HANDLE_OP(WASM_OP_F32_REINTERPRET_I32) - HANDLE_OP(WASM_OP_F64_REINTERPRET_I64) { HANDLE_OP_END(); } + HANDLE_OP(WASM_OP_F64_REINTERPRET_I64) + { + HANDLE_OP_END(); + } HANDLE_OP(WASM_OP_I32_EXTEND8_S) { diff --git a/core/iwasm/interpreter/wasm_interp_fast.c b/core/iwasm/interpreter/wasm_interp_fast.c index 3a9715a66f..6ab5d236e4 100644 --- a/core/iwasm/interpreter/wasm_interp_fast.c +++ b/core/iwasm/interpreter/wasm_interp_fast.c @@ -53,7 +53,7 @@ typedef float64 CellType_F64; for (cur = shared_heap; cur; cur = cur->chain_next) { \ cur_shared_heap_start_off = get_shared_heap_start_off(cur); \ cur_shared_heap_end_off = \ - cur_shared_heap_start_off + cur->size - 1; \ + cur_shared_heap_start_off - 1 + cur->size; \ if ((app_addr) >= cur_shared_heap_start_off \ && (app_addr) <= cur_shared_heap_end_off - bytes + 1) { \ shared_heap_start_off = cur_shared_heap_start_off; \ @@ -67,7 +67,7 @@ typedef float64 CellType_F64; }) #define shared_heap_addr_app_to_native(app_addr, native_addr) \ - native_addr = shared_heap_base_addr + ((app_addr)-shared_heap_start_off) + native_addr = shared_heap_base_addr + ((app_addr) - shared_heap_start_off) #define CHECK_SHARED_HEAP_OVERFLOW(app_addr, bytes, native_addr) \ if (app_addr_in_shared_heap(app_addr, bytes)) \ @@ -1568,7 +1568,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, shared_heap ? get_shared_heap_start_off(shared_heap) : 0; uint64 shared_heap_end_off = shared_heap - ? (get_shared_heap_start_off(shared_heap) + shared_heap->size - 1) + ? (get_shared_heap_start_off(shared_heap) - 1 + shared_heap->size) : 0; /* #endif */ #endif /* end of WASM_ENABLE_SHARED_HEAP != 0 */ diff --git a/samples/shared-heap/src/main.c b/samples/shared-heap/src/main.c index f4024f08c2..ed64cc5da0 100644 --- a/samples/shared-heap/src/main.c +++ b/samples/shared-heap/src/main.c @@ -55,7 +55,7 @@ thread1_callback(void *arg) i + 1); printf("wasm app1 send buf: %s\n\n", buf); - if (!bh_post_msg(queue, 1, buf, 1024 * i)) { + if (!bh_post_msg(queue, 1, buf, 1024 * (i + 1))) { printf("Failed to post message to queue\n"); wasm_runtime_shared_heap_free(module_inst, offset); break; @@ -84,7 +84,7 @@ thread1_callback(void *arg) buf = wasm_runtime_addr_app_to_native(module_inst, argv[0]); printf("wasm app1 send buf: %s\n\n", buf); - if (!bh_post_msg(queue, 1, buf, 1024 * i)) { + if (!bh_post_msg(queue, 1, buf, 1024 * (i+1))) { printf("Failed to post message to queue\n"); wasm_runtime_shared_heap_free(module_inst, argv[0]); break; @@ -268,7 +268,7 @@ main(int argc, char **argv) } /* create thread 1 */ - struct thread_arg targ1 = { 0 }; + thread_arg targ1 = { 0 }; korp_tid tid1; targ1.queue = queue; targ1.module_inst = module_inst1; @@ -279,7 +279,7 @@ main(int argc, char **argv) } /* create thread 2 */ - struct thread_arg targ2 = { 0 }; + thread_arg targ2 = { 0 }; korp_tid tid2; targ2.queue = queue; targ2.module_inst = module_inst2; diff --git a/tests/unit/shared-heap/shared_heap_test.cc b/tests/unit/shared-heap/shared_heap_test.cc index deb4bbb388..76357711db 100644 --- a/tests/unit/shared-heap/shared_heap_test.cc +++ b/tests/unit/shared-heap/shared_heap_test.cc @@ -92,7 +92,9 @@ destroy_module_env(struct ret_env module_env) } } -static void test_shared_heap(WASMSharedHeap *shared_heap, const char *file, const char *func_name, uint32 argc, uint32 argv[]) +static void +test_shared_heap(WASMSharedHeap *shared_heap, const char *file, + const char *func_name, uint32 argc, uint32 argv[]) { struct ret_env tmp_module_env; WASMFunctionInstanceCommon *func_test = nullptr; @@ -101,7 +103,8 @@ static void test_shared_heap(WASMSharedHeap *shared_heap, const char *file, cons tmp_module_env = load_wasm((char *)file, 0); - if (!wasm_runtime_attach_shared_heap(tmp_module_env.wasm_module_inst, shared_heap)) { + if (!wasm_runtime_attach_shared_heap(tmp_module_env.wasm_module_inst, + shared_heap)) { printf("Failed to attach shared heap\n"); goto test_failed; } @@ -116,7 +119,8 @@ static void test_shared_heap(WASMSharedHeap *shared_heap, const char *file, cons wasm_runtime_call_wasm(tmp_module_env.exec_env, func_test, argc, argv); if (!ret) { printf("\nFailed to wasm_runtime_call_wasm!\n"); - const char *s = wasm_runtime_get_exception(tmp_module_env.wasm_module_inst); + const char *s = + wasm_runtime_get_exception(tmp_module_env.wasm_module_inst); printf("exception: %s\n", s); goto test_failed; } @@ -131,7 +135,7 @@ static void test_shared_heap(WASMSharedHeap *shared_heap, const char *file, cons TEST_F(shared_heap_test, test_shared_heap_basic) { - SharedHeapInitArgs args; + SharedHeapInitArgs args = { 0 }; WASMSharedHeap *shared_heap = nullptr; uint32 argv[1] = { 0 }; @@ -150,12 +154,11 @@ TEST_F(shared_heap_test, test_shared_heap_basic) // test aot test_shared_heap(shared_heap, "test.aot", "test", 1, argv); EXPECT_EQ(10, argv[0]); - } TEST_F(shared_heap_test, test_shared_heap_malloc_fail) { - SharedHeapInitArgs args; + SharedHeapInitArgs args = { 0 }; WASMSharedHeap *shared_heap = nullptr; uint32 argv[1] = { 0 }; @@ -177,36 +180,36 @@ TEST_F(shared_heap_test, test_shared_heap_malloc_fail) } #ifndef native_function +/* clang-format off */ #define native_function(func_name, signature) \ { #func_name, (void *)glue_##func_name, signature, NULL } - +/* clang-format on */ #endif #ifndef nitems #define nitems(_a) (sizeof(_a) / sizeof(0 [(_a)])) #endif /* nitems */ -uintptr_t glue_test_addr_conv(wasm_exec_env_t env, uintptr_t addr) +uintptr_t +glue_test_addr_conv(wasm_exec_env_t env, uintptr_t addr) { - wasm_module_inst_t module_inst = get_module_inst(env); - uintptr_t ret; - void *native_addr = (void *)addr; - uintptr_t app_addr = addr_native_to_app(native_addr); - - native_addr = addr_app_to_native(app_addr); - if (native_addr != (void *)addr) - { - EXPECT_EQ(1, 0); - } - return app_addr; + wasm_module_inst_t module_inst = get_module_inst(env); + uintptr_t ret; + void *native_addr = (void *)addr; + uintptr_t app_addr = addr_native_to_app(native_addr); + + native_addr = addr_app_to_native(app_addr); + if (native_addr != (void *)addr) { + EXPECT_EQ(1, 0); + } + return app_addr; } -static NativeSymbol g_test_native_symbols[] = -{ - native_function(test_addr_conv,"(*)i"), +static NativeSymbol g_test_native_symbols[] = { + native_function(test_addr_conv, "(*)i"), }; TEST_F(shared_heap_test, test_addr_conv) { - SharedHeapInitArgs args; + SharedHeapInitArgs args = { 0 }; WASMSharedHeap *shared_heap = nullptr; uint32 argv[1] = { 0 }; struct ret_env tmp_module_env; @@ -217,8 +220,7 @@ TEST_F(shared_heap_test, test_addr_conv) ret = wasm_native_register_natives("env", g_test_native_symbols, nitems(g_test_native_symbols)); - if (!ret) - { + if (!ret) { EXPECT_EQ(1, 0); return; } From e91181331799316252bf65c9a3978ee3278b2de0 Mon Sep 17 00:00:00 2001 From: TL Date: Wed, 5 Feb 2025 15:06:25 +0800 Subject: [PATCH 07/20] format and update shared heap boundary check in runtime API --- core/iwasm/common/wasm_memory.c | 34 ++++++++++++++------ core/iwasm/interpreter/wasm_interp_classic.c | 10 ++---- 2 files changed, 26 insertions(+), 18 deletions(-) diff --git a/core/iwasm/common/wasm_memory.c b/core/iwasm/common/wasm_memory.c index ec57e9e67b..7c09ff5e3f 100644 --- a/core/iwasm/common/wasm_memory.c +++ b/core/iwasm/common/wasm_memory.c @@ -496,25 +496,38 @@ is_app_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst, uint64 shared_heap_start, shared_heap_end; if (!heap) { - return false; + goto fail; } if (bytes == 0) { bytes = 1; } - for (cur = heap; cur; cur = cur->chain_next) { - shared_heap_start = - is_memory64 ? cur->start_off_mem64 : cur->start_off_mem32; - shared_heap_end = shared_heap_start - 1 + cur->size; - if (app_offset >= shared_heap_start - && app_offset <= shared_heap_end - bytes + 1) { - if (target_heap) + /* Early stop for app start address not in the shared heap(chain) at all */ + shared_heap_start = + is_memory64 ? heap->start_off_mem64 : heap->start_off_mem32; + shared_heap_end = is_memory64 ? UINT64_MAX : UINT32_MAX; + if (app_offset < shared_heap_start + || app_offset > shared_heap_end - bytes + 1) { + goto fail; + } + + /* Find the exact shared heap that app addr is in */ + if (target_heap) { + for (cur = heap; cur; cur = cur->chain_next) { + shared_heap_start = + is_memory64 ? cur->start_off_mem64 : cur->start_off_mem32; + shared_heap_end = shared_heap_start - 1 + cur->size; + if (app_offset >= shared_heap_start + && app_offset <= shared_heap_end - bytes + 1) { *target_heap = cur; - return true; + return true; + } } } + return true; +fail: if (target_heap) *target_heap = NULL; return false; @@ -529,7 +542,7 @@ is_native_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst, uintptr_t base_addr, addr_int, end_addr; if (!heap_head) { - return false; + goto fail; } /* Iterate through shared heap chain to find whether native addr in one of @@ -553,6 +566,7 @@ is_native_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst, return true; } +fail: if (target_heap) *target_heap = NULL; return false; diff --git a/core/iwasm/interpreter/wasm_interp_classic.c b/core/iwasm/interpreter/wasm_interp_classic.c index 76803cca74..349e360f6c 100644 --- a/core/iwasm/interpreter/wasm_interp_classic.c +++ b/core/iwasm/interpreter/wasm_interp_classic.c @@ -1719,10 +1719,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, goto got_exception; } - HANDLE_OP(WASM_OP_NOP) - { - HANDLE_OP_END(); - } + HANDLE_OP(WASM_OP_NOP) { HANDLE_OP_END(); } #if WASM_ENABLE_EXCE_HANDLING != 0 HANDLE_OP(WASM_OP_RETHROW) @@ -5659,10 +5656,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, HANDLE_OP(WASM_OP_I32_REINTERPRET_F32) HANDLE_OP(WASM_OP_I64_REINTERPRET_F64) HANDLE_OP(WASM_OP_F32_REINTERPRET_I32) - HANDLE_OP(WASM_OP_F64_REINTERPRET_I64) - { - HANDLE_OP_END(); - } + HANDLE_OP(WASM_OP_F64_REINTERPRET_I64) { HANDLE_OP_END(); } HANDLE_OP(WASM_OP_I32_EXTEND8_S) { From 0b0ef2d11742c9989162b7ef3890c834f3d6af2b Mon Sep 17 00:00:00 2001 From: TL Date: Wed, 5 Feb 2025 15:49:01 +0800 Subject: [PATCH 08/20] format --- core/iwasm/interpreter/wasm_interp_fast.c | 2 +- samples/shared-heap/src/main.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/iwasm/interpreter/wasm_interp_fast.c b/core/iwasm/interpreter/wasm_interp_fast.c index 6ab5d236e4..6bdb034565 100644 --- a/core/iwasm/interpreter/wasm_interp_fast.c +++ b/core/iwasm/interpreter/wasm_interp_fast.c @@ -67,7 +67,7 @@ typedef float64 CellType_F64; }) #define shared_heap_addr_app_to_native(app_addr, native_addr) \ - native_addr = shared_heap_base_addr + ((app_addr) - shared_heap_start_off) + native_addr = shared_heap_base_addr + ((app_addr)-shared_heap_start_off) #define CHECK_SHARED_HEAP_OVERFLOW(app_addr, bytes, native_addr) \ if (app_addr_in_shared_heap(app_addr, bytes)) \ diff --git a/samples/shared-heap/src/main.c b/samples/shared-heap/src/main.c index ed64cc5da0..1d93a6a2f7 100644 --- a/samples/shared-heap/src/main.c +++ b/samples/shared-heap/src/main.c @@ -84,7 +84,7 @@ thread1_callback(void *arg) buf = wasm_runtime_addr_app_to_native(module_inst, argv[0]); printf("wasm app1 send buf: %s\n\n", buf); - if (!bh_post_msg(queue, 1, buf, 1024 * (i+1))) { + if (!bh_post_msg(queue, 1, buf, 1024 * (i + 1))) { printf("Failed to post message to queue\n"); wasm_runtime_shared_heap_free(module_inst, argv[0]); break; From 91f1234dd0c7885b0de16f39e09f54391c57c5ea Mon Sep 17 00:00:00 2001 From: TL Date: Sat, 8 Feb 2025 12:17:01 +0800 Subject: [PATCH 09/20] some refactor --- core/iwasm/common/wasm_memory.c | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/core/iwasm/common/wasm_memory.c b/core/iwasm/common/wasm_memory.c index 7c09ff5e3f..32e2114126 100644 --- a/core/iwasm/common/wasm_memory.c +++ b/core/iwasm/common/wasm_memory.c @@ -267,9 +267,10 @@ wasm_runtime_chain_shared_heaps(WASMSharedHeap *head, WASMSharedHeap *body) return NULL; } for (cur = shared_heap_list; cur; cur = cur->next) { - if (cur->chain_next == body) { - LOG_WARNING("To create shared heap chain, the `body` shared heap " - "can't already be in a chain"); + if (cur->chain_next == body || cur->chain_next == head) { + LOG_WARNING( + "To create shared heap chain, both the 'head' and 'body' " + "shared heap can't already be the 'body' in another a chain"); os_mutex_unlock(&shared_heap_list_lock); return NULL; } @@ -600,12 +601,10 @@ wasm_runtime_shared_heap_malloc(WASMModuleInstanceCommon *module_inst, *p_native_addr = native_addr; } - if (memory->is_memory64) - return shared_heap->start_off_mem64 - + ((uint8 *)native_addr - shared_heap->base_addr); - else - return shared_heap->start_off_mem32 - + ((uint8 *)native_addr - shared_heap->base_addr); + return memory->is_memory64 + ? shared_heap->start_off_mem64 + : shared_heap->start_off_mem32 + + ((uint8 *)native_addr - shared_heap->base_addr); } void From 3296925c8a64672c310dfeeabd5b7c7248a84122 Mon Sep 17 00:00:00 2001 From: TL Date: Sat, 8 Feb 2025 12:53:58 +0800 Subject: [PATCH 10/20] some refactor --- core/iwasm/common/wasm_memory.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/core/iwasm/common/wasm_memory.c b/core/iwasm/common/wasm_memory.c index 32e2114126..e8b8849fa8 100644 --- a/core/iwasm/common/wasm_memory.c +++ b/core/iwasm/common/wasm_memory.c @@ -298,7 +298,7 @@ wasm_runtime_chain_shared_heaps(WASMSharedHeap *head, WASMSharedHeap *body) WASMSharedHeap * wasm_runtime_unchain_shared_heaps(WASMSharedHeap *head, bool entire_chain) { - WASMSharedHeap *cur; + WASMSharedHeap *cur, *tmp; if (!head || !head->chain_next) { LOG_WARNING("Invalid shared heap chain to disconnect the head from."); @@ -313,9 +313,13 @@ wasm_runtime_unchain_shared_heaps(WASMSharedHeap *head, bool entire_chain) return NULL; } - for (cur = head; cur && cur->chain_next; cur = cur->chain_next) { + cur = head; + while (cur && cur->chain_next) { cur->start_off_mem64 = UINT64_MAX - cur->size + 1; cur->start_off_mem32 = UINT32_MAX - cur->size + 1; + tmp = cur; + cur = cur->chain_next; + tmp->chain_next = NULL; if (!entire_chain) break; } From a9d776eda322298eb842c3c35c79aea5ce7c9bec Mon Sep 17 00:00:00 2001 From: TL Date: Wed, 12 Feb 2025 16:20:29 +0800 Subject: [PATCH 11/20] cr suggestions and some refactor --- core/iwasm/common/wasm_memory.c | 13 ++--- core/iwasm/common/wasm_shared_memory.h | 48 +++++++++++++++++++ core/iwasm/include/wasm_export.h | 10 ++-- core/iwasm/interpreter/wasm_interp_classic.c | 50 -------------------- core/iwasm/interpreter/wasm_interp_fast.c | 46 +++--------------- 5 files changed, 66 insertions(+), 101 deletions(-) diff --git a/core/iwasm/common/wasm_memory.c b/core/iwasm/common/wasm_memory.c index e8b8849fa8..d44d1778e6 100644 --- a/core/iwasm/common/wasm_memory.c +++ b/core/iwasm/common/wasm_memory.c @@ -185,7 +185,7 @@ wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args) if (size > APP_HEAP_SIZE_MAX || size < APP_HEAP_SIZE_MIN) { LOG_WARNING("Invalid size of shared heap"); - goto fail1; + goto fail2; } if (init_args->pre_allocated_addr != NULL) { @@ -194,7 +194,7 @@ wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args) if (size != init_args->size) { LOG_WARNING("Pre allocated size need to be aligned with system " "page size to create shared heap"); - goto fail1; + goto fail2; } heap->heap_handle = NULL; @@ -252,12 +252,13 @@ WASMSharedHeap * wasm_runtime_chain_shared_heaps(WASMSharedHeap *head, WASMSharedHeap *body) { WASMSharedHeap *cur; - bool heap_handle_exist = head->heap_handle != NULL; + bool heap_handle_exist = false; if (!head || !body) { LOG_WARNING("Invalid shared heap to chain."); return NULL; } + heap_handle_exist = head->heap_handle != NULL; os_mutex_lock(&shared_heap_list_lock); if (head->attached_count != 0 || body->attached_count != 0) { @@ -543,16 +544,16 @@ is_native_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst, uint8 *addr, uint32 bytes, WASMSharedHeap **target_heap) { - WASMSharedHeap *cur, *heap_head = get_shared_heap(module_inst); + WASMSharedHeap *cur, *heap = get_shared_heap(module_inst); uintptr_t base_addr, addr_int, end_addr; - if (!heap_head) { + if (!heap) { goto fail; } /* Iterate through shared heap chain to find whether native addr in one of * shared heap */ - for (cur = heap_head; cur != NULL; cur = cur->chain_next) { + for (cur = heap; cur != NULL; cur = cur->chain_next) { base_addr = (uintptr_t)cur->base_addr; addr_int = (uintptr_t)addr; if (addr_int < base_addr) diff --git a/core/iwasm/common/wasm_shared_memory.h b/core/iwasm/common/wasm_shared_memory.h index e1c5154a53..8b8af3efa6 100644 --- a/core/iwasm/common/wasm_shared_memory.h +++ b/core/iwasm/common/wasm_shared_memory.h @@ -56,6 +56,54 @@ uint32 wasm_runtime_atomic_notify(WASMModuleInstanceCommon *module, void *address, uint32 count); +#if WASM_ENABLE_MULTI_MEMORY != 0 +/* Only enable shared heap for the default memory */ +#define is_default_memory (memidx == 0) +#else +#define is_default_memory true +#endif +#if WASM_ENABLE_MEMORY64 != 0 +#define get_shared_heap_start_off(shared_heap) \ + (is_memory64 ? shared_heap->start_off_mem64 : shared_heap->start_off_mem32) +#else +#define get_shared_heap_start_off(shared_heap) (shared_heap->start_off_mem32) +#endif +/* Check whether the app addr in the last visited shared heap, if not, check the + * shared heap chain to find which(if any) shared heap the app addr in, and + * update the last visited shared heap info if found. */ +#define app_addr_in_shared_heap(app_addr, bytes) \ + (shared_heap && is_default_memory && (app_addr) >= shared_heap_start_off \ + && (app_addr) <= shared_heap_end_off - bytes + 1) \ + || ({ \ + bool in_chain = false; \ + WASMSharedHeap *cur; \ + uint64 cur_shared_heap_start_off, cur_shared_heap_end_off; \ + for (cur = shared_heap; cur; cur = cur->chain_next) { \ + cur_shared_heap_start_off = get_shared_heap_start_off(cur); \ + cur_shared_heap_end_off = \ + cur_shared_heap_start_off - 1 + cur->size; \ + if ((app_addr) >= cur_shared_heap_start_off \ + && (app_addr) <= cur_shared_heap_end_off - bytes + 1) { \ + shared_heap_start_off = cur_shared_heap_start_off; \ + shared_heap_end_off = cur_shared_heap_end_off; \ + shared_heap_base_addr = cur->base_addr; \ + in_chain = true; \ + break; \ + } \ + } \ + in_chain; \ + }) + +#define shared_heap_addr_app_to_native(app_addr, native_addr) \ + native_addr = shared_heap_base_addr + ((app_addr)-shared_heap_start_off) + +#define CHECK_SHARED_HEAP_OVERFLOW(app_addr, bytes, native_addr) \ + if (app_addr_in_shared_heap(app_addr, bytes)) \ + shared_heap_addr_app_to_native(app_addr, native_addr); \ + else +#else +#define CHECK_SHARED_HEAP_OVERFLOW(app_addr, bytes, native_addr) + #ifdef __cplusplus } #endif diff --git a/core/iwasm/include/wasm_export.h b/core/iwasm/include/wasm_export.h index 15759f7775..65bb0e7674 100644 --- a/core/iwasm/include/wasm_export.h +++ b/core/iwasm/include/wasm_export.h @@ -2259,11 +2259,11 @@ WASM_RUNTIME_API_EXTERN wasm_shared_heap_t wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args); /** - * This function links two shared heaps, `head` and `body`, where `head` is a - * shared heap and `body` can be shared heap chain head, into a single chain. - * The `head` heap will be the head of the chain, and the `body` heap will be - * appended to it. At most one shared heap in shared heap chain can be - * dynamically allocated, the rest have to be the pre-allocated shared heap * + * This function links two shared heap(lists), `head` and `body` in to a single + * shared heap list, where `head` becomes the new shared heap list head. The + * shared heap list remains one continuous shared heap in wasm app's point of + * view. At most one shared heap in shared heap list can be dynamically + * allocated, the rest have to be the pre-allocated shared heap. * * * @param head The head of the shared heap chain. * @param body The body of the shared heap chain to be appended. diff --git a/core/iwasm/interpreter/wasm_interp_classic.c b/core/iwasm/interpreter/wasm_interp_classic.c index 349e360f6c..ab1486aa97 100644 --- a/core/iwasm/interpreter/wasm_interp_classic.c +++ b/core/iwasm/interpreter/wasm_interp_classic.c @@ -46,56 +46,6 @@ typedef float64 CellType_F64; #define get_linear_mem_size() GET_LINEAR_MEMORY_SIZE(memory) #endif -#if WASM_ENABLE_SHARED_HEAP != 0 -#if WASM_ENABLE_MULTI_MEMORY != 0 -/* Only enable shared heap for the default memory */ -#define is_default_memory (memidx == 0) -#else -#define is_default_memory true -#endif -#if WASM_ENABLE_MEMORY64 != 0 -#define get_shared_heap_start_off(shared_heap) \ - (is_memory64 ? shared_heap->start_off_mem64 : shared_heap->start_off_mem32) -#else -#define get_shared_heap_start_off(shared_heap) (shared_heap->start_off_mem32) -#endif -/* Check whether the app addr in the last visited shared heap, if not, check the - * shared heap chain to find which(if any) shared heap the app addr in, and - * update the last visited shared heap info if found. */ -#define app_addr_in_shared_heap(app_addr, bytes) \ - (shared_heap && is_default_memory && (app_addr) >= shared_heap_start_off \ - && (app_addr) <= shared_heap_end_off - bytes + 1) \ - || ({ \ - bool in_chain = false; \ - WASMSharedHeap *cur; \ - uint64 cur_shared_heap_start_off, cur_shared_heap_end_off; \ - for (cur = shared_heap; cur; cur = cur->chain_next) { \ - cur_shared_heap_start_off = get_shared_heap_start_off(cur); \ - cur_shared_heap_end_off = \ - cur_shared_heap_start_off - 1 + cur->size; \ - if ((app_addr) >= cur_shared_heap_start_off \ - && (app_addr) <= cur_shared_heap_end_off - bytes + 1) { \ - shared_heap_start_off = cur_shared_heap_start_off; \ - shared_heap_end_off = cur_shared_heap_end_off; \ - shared_heap_base_addr = cur->base_addr; \ - in_chain = true; \ - break; \ - } \ - } \ - in_chain; \ - }) - -#define shared_heap_addr_app_to_native(app_addr, native_addr) \ - native_addr = shared_heap_base_addr + ((app_addr)-shared_heap_start_off) - -#define CHECK_SHARED_HEAP_OVERFLOW(app_addr, bytes, native_addr) \ - if (app_addr_in_shared_heap(app_addr, bytes)) \ - shared_heap_addr_app_to_native(app_addr, native_addr); \ - else -#else -#define CHECK_SHARED_HEAP_OVERFLOW(app_addr, bytes, native_addr) -#endif - #if WASM_ENABLE_MEMORY64 == 0 #if (!defined(OS_ENABLE_HW_BOUND_CHECK) \ diff --git a/core/iwasm/interpreter/wasm_interp_fast.c b/core/iwasm/interpreter/wasm_interp_fast.c index 6bdb034565..a62b6b2dec 100644 --- a/core/iwasm/interpreter/wasm_interp_fast.c +++ b/core/iwasm/interpreter/wasm_interp_fast.c @@ -37,46 +37,6 @@ typedef float64 CellType_F64; #define get_linear_mem_size() GET_LINEAR_MEMORY_SIZE(memory) #endif -#if WASM_ENABLE_SHARED_HEAP != 0 -/* TODO: add code for 64 bit version when memory64 is enabled for fast-interp */ -#define get_shared_heap_start_off(shared_heap) (shared_heap->start_off_mem32) -/* Check whether the app addr in the last visited shared heap, if not, check the - * shared heap chain to find which(if any) shared heap the app addr in, and - * update the last visited shared heap info if found. */ -#define app_addr_in_shared_heap(app_addr, bytes) \ - (shared_heap && (app_addr) >= shared_heap_start_off \ - && (app_addr) <= shared_heap_end_off - bytes + 1) \ - || ({ \ - bool in_chain = false; \ - WASMSharedHeap *cur; \ - uint64 cur_shared_heap_start_off, cur_shared_heap_end_off; \ - for (cur = shared_heap; cur; cur = cur->chain_next) { \ - cur_shared_heap_start_off = get_shared_heap_start_off(cur); \ - cur_shared_heap_end_off = \ - cur_shared_heap_start_off - 1 + cur->size; \ - if ((app_addr) >= cur_shared_heap_start_off \ - && (app_addr) <= cur_shared_heap_end_off - bytes + 1) { \ - shared_heap_start_off = cur_shared_heap_start_off; \ - shared_heap_end_off = cur_shared_heap_end_off; \ - shared_heap_base_addr = cur->base_addr; \ - in_chain = true; \ - break; \ - } \ - } \ - in_chain; \ - }) - -#define shared_heap_addr_app_to_native(app_addr, native_addr) \ - native_addr = shared_heap_base_addr + ((app_addr)-shared_heap_start_off) - -#define CHECK_SHARED_HEAP_OVERFLOW(app_addr, bytes, native_addr) \ - if (app_addr_in_shared_heap(app_addr, bytes)) \ - shared_heap_addr_app_to_native(app_addr, native_addr); \ - else -#else -#define CHECK_SHARED_HEAP_OVERFLOW(app_addr, bytes, native_addr) -#endif - #if !defined(OS_ENABLE_HW_BOUND_CHECK) \ || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 #define CHECK_MEMORY_OVERFLOW(bytes) \ @@ -1562,6 +1522,12 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, bool is_return_call = false; #endif #if WASM_ENABLE_SHARED_HEAP != 0 + /* TODO: currently flowing two variables are only dummy for shared heap + * boundary check, need to be updated when multi-memory or memory64 + * proposals are to be implemented */ + bool is_memory64 = false; + uint32 memidx = 0; + WASMSharedHeap *shared_heap = module->e ? module->e->shared_heap : NULL; uint8 *shared_heap_base_addr = shared_heap ? shared_heap->base_addr : NULL; uint64 shared_heap_start_off = From 3abea0cd310173da5e3f016868c199abdcc52ae0 Mon Sep 17 00:00:00 2001 From: TL Date: Wed, 12 Feb 2025 16:54:58 +0800 Subject: [PATCH 12/20] some update --- core/iwasm/common/wasm_shared_memory.h | 48 ------------------------- core/iwasm/interpreter/wasm_loader.h | 50 ++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 48 deletions(-) diff --git a/core/iwasm/common/wasm_shared_memory.h b/core/iwasm/common/wasm_shared_memory.h index 8b8af3efa6..e1c5154a53 100644 --- a/core/iwasm/common/wasm_shared_memory.h +++ b/core/iwasm/common/wasm_shared_memory.h @@ -56,54 +56,6 @@ uint32 wasm_runtime_atomic_notify(WASMModuleInstanceCommon *module, void *address, uint32 count); -#if WASM_ENABLE_MULTI_MEMORY != 0 -/* Only enable shared heap for the default memory */ -#define is_default_memory (memidx == 0) -#else -#define is_default_memory true -#endif -#if WASM_ENABLE_MEMORY64 != 0 -#define get_shared_heap_start_off(shared_heap) \ - (is_memory64 ? shared_heap->start_off_mem64 : shared_heap->start_off_mem32) -#else -#define get_shared_heap_start_off(shared_heap) (shared_heap->start_off_mem32) -#endif -/* Check whether the app addr in the last visited shared heap, if not, check the - * shared heap chain to find which(if any) shared heap the app addr in, and - * update the last visited shared heap info if found. */ -#define app_addr_in_shared_heap(app_addr, bytes) \ - (shared_heap && is_default_memory && (app_addr) >= shared_heap_start_off \ - && (app_addr) <= shared_heap_end_off - bytes + 1) \ - || ({ \ - bool in_chain = false; \ - WASMSharedHeap *cur; \ - uint64 cur_shared_heap_start_off, cur_shared_heap_end_off; \ - for (cur = shared_heap; cur; cur = cur->chain_next) { \ - cur_shared_heap_start_off = get_shared_heap_start_off(cur); \ - cur_shared_heap_end_off = \ - cur_shared_heap_start_off - 1 + cur->size; \ - if ((app_addr) >= cur_shared_heap_start_off \ - && (app_addr) <= cur_shared_heap_end_off - bytes + 1) { \ - shared_heap_start_off = cur_shared_heap_start_off; \ - shared_heap_end_off = cur_shared_heap_end_off; \ - shared_heap_base_addr = cur->base_addr; \ - in_chain = true; \ - break; \ - } \ - } \ - in_chain; \ - }) - -#define shared_heap_addr_app_to_native(app_addr, native_addr) \ - native_addr = shared_heap_base_addr + ((app_addr)-shared_heap_start_off) - -#define CHECK_SHARED_HEAP_OVERFLOW(app_addr, bytes, native_addr) \ - if (app_addr_in_shared_heap(app_addr, bytes)) \ - shared_heap_addr_app_to_native(app_addr, native_addr); \ - else -#else -#define CHECK_SHARED_HEAP_OVERFLOW(app_addr, bytes, native_addr) - #ifdef __cplusplus } #endif diff --git a/core/iwasm/interpreter/wasm_loader.h b/core/iwasm/interpreter/wasm_loader.h index 676770ee22..9171d71d37 100644 --- a/core/iwasm/interpreter/wasm_loader.h +++ b/core/iwasm/interpreter/wasm_loader.h @@ -73,6 +73,56 @@ wasm_loader_find_block_addr(WASMExecEnv *exec_env, BlockAddr *block_addr_cache, uint8 block_type, uint8 **p_else_addr, uint8 **p_end_addr); +#if WASM_ENABLE_SHARED_HEAP != 0 +#if WASM_ENABLE_MULTI_MEMORY != 0 +/* Only enable shared heap for the default memory */ +#define is_default_memory (memidx == 0) +#else +#define is_default_memory true +#endif +#if WASM_ENABLE_MEMORY64 != 0 +#define get_shared_heap_start_off(shared_heap) \ + (is_memory64 ? shared_heap->start_off_mem64 : shared_heap->start_off_mem32) +#else +#define get_shared_heap_start_off(shared_heap) (shared_heap->start_off_mem32) +#endif +/* Check whether the app addr in the last visited shared heap, if not, check the + * shared heap chain to find which(if any) shared heap the app addr in, and + * update the last visited shared heap info if found. */ +#define app_addr_in_shared_heap(app_addr, bytes) \ + (shared_heap && is_default_memory && (app_addr) >= shared_heap_start_off \ + && (app_addr) <= shared_heap_end_off - bytes + 1) \ + || ({ \ + bool in_chain = false; \ + WASMSharedHeap *cur; \ + uint64 cur_shared_heap_start_off, cur_shared_heap_end_off; \ + for (cur = shared_heap; cur; cur = cur->chain_next) { \ + cur_shared_heap_start_off = get_shared_heap_start_off(cur); \ + cur_shared_heap_end_off = \ + cur_shared_heap_start_off - 1 + cur->size; \ + if ((app_addr) >= cur_shared_heap_start_off \ + && (app_addr) <= cur_shared_heap_end_off - bytes + 1) { \ + shared_heap_start_off = cur_shared_heap_start_off; \ + shared_heap_end_off = cur_shared_heap_end_off; \ + shared_heap_base_addr = cur->base_addr; \ + in_chain = true; \ + break; \ + } \ + } \ + in_chain; \ + }) + +#define shared_heap_addr_app_to_native(app_addr, native_addr) \ + native_addr = shared_heap_base_addr + ((app_addr)-shared_heap_start_off) + +#define CHECK_SHARED_HEAP_OVERFLOW(app_addr, bytes, native_addr) \ + if (app_addr_in_shared_heap(app_addr, bytes)) \ + shared_heap_addr_app_to_native(app_addr, native_addr); \ + else +#else +#define CHECK_SHARED_HEAP_OVERFLOW(app_addr, bytes, native_addr) +#endif /* end of WASM_ENABLE_SHARED_HEAP != 0 */ + #ifdef __cplusplus } #endif From 88a7f161cf4f521f46c86ba7b1e4de48821d300c Mon Sep 17 00:00:00 2001 From: TL Date: Fri, 21 Feb 2025 11:24:14 +0800 Subject: [PATCH 13/20] some refactor --- core/iwasm/common/wasm_memory.c | 278 ++++++++++++------- core/iwasm/interpreter/wasm_interp_classic.c | 15 +- core/iwasm/interpreter/wasm_interp_fast.c | 17 +- core/iwasm/interpreter/wasm_loader.h | 47 +++- core/iwasm/interpreter/wasm_runtime.h | 5 +- 5 files changed, 232 insertions(+), 130 deletions(-) diff --git a/core/iwasm/common/wasm_memory.c b/core/iwasm/common/wasm_memory.c index d44d1778e6..69801a2f7a 100644 --- a/core/iwasm/common/wasm_memory.c +++ b/core/iwasm/common/wasm_memory.c @@ -328,6 +328,132 @@ wasm_runtime_unchain_shared_heaps(WASMSharedHeap *head, bool entire_chain) return cur; } +static uint8 * +get_last_used_shared_heap_base_addr_adj(WASMModuleInstanceCommon *module_inst) +{ +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + WASMModuleInstanceExtra *e = + (WASMModuleInstanceExtra *)((WASMModuleInstance *)module_inst)->e; + return e->shared_heap_base_addr_adj; + } +#endif /* end of WASM_ENABLE_INTERP != 0 */ +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + AOTModuleInstanceExtra *e = + (AOTModuleInstanceExtra *)((AOTModuleInstance *)module_inst)->e; + return e->shared_heap_base_addr_adj; + } +#endif /* end of WASM_ENABLE_AOT != 0 */ + return 0; +} + +static uintptr_t +get_last_used_shared_heap_start_offset(WASMModuleInstanceCommon *module_inst) +{ +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + WASMModuleInstanceExtra *e = + (WASMModuleInstanceExtra *)((WASMModuleInstance *)module_inst)->e; +#if UINTPTR_MAX == UINT64_MAX + return e->shared_heap_start_off.u64; +#else + return e->shared_heap_start_off.u32[0]; +#endif + } +#endif /* end of WASM_ENABLE_INTERP != 0 */ +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + AOTModuleInstanceExtra *e = + (AOTModuleInstanceExtra *)((AOTModuleInstance *)module_inst)->e; +#if UINTPTR_MAX == UINT64_MAX + return e->shared_heap_start_off.u64; +#else + return e->shared_heap_start_off.u32[0]; +#endif + } +#endif /* end of WASM_ENABLE_AOT != 0 */ + return 0; +} + +static uintptr_t +get_last_used_shared_heap_end_offset(WASMModuleInstanceCommon *module_inst) +{ +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + WASMModuleInstanceExtra *e = + (WASMModuleInstanceExtra *)((WASMModuleInstance *)module_inst)->e; +#if UINTPTR_MAX == UINT64_MAX + return e->shared_heap_end_off.u64; +#else + return e->shared_heap_end_off.u32[0]; +#endif + } +#endif /* end of WASM_ENABLE_INTERP != 0 */ +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + AOTModuleInstanceExtra *e = + (AOTModuleInstanceExtra *)((AOTModuleInstance *)module_inst)->e; +#if UINTPTR_MAX == UINT64_MAX + return e->shared_heap_end_off.u64; +#else + return e->shared_heap_end_off.u32[0]; +#endif + } +#endif /* end of WASM_ENABLE_AOT != 0 */ + return 0; +} + +static void +update_last_used_shared_heap(WASMModuleInstanceCommon *module_inst, + WASMSharedHeap *shared_heap, bool is_memory64) +{ +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + WASMModuleInstanceExtra *e = + (WASMModuleInstanceExtra *)((WASMModuleInstance *)module_inst)->e; +#if UINTPTR_MAX == UINT64_MAX + if (is_memory64) + e->shared_heap_start_off.u64 = shared_heap->start_off_mem64; + else + e->shared_heap_start_off.u64 = shared_heap->start_off_mem32; + e->shared_heap_end_off.u64 = + e->shared_heap_start_off.u64 - 1 + shared_heap->size; + e->shared_heap_base_addr_adj = + shared_heap->base_addr - e->shared_heap_start_off.u64; +#else + e->shared_heap_start_off.u32[0] = (uint32)shared_heap->start_off_mem32; + e->shared_heap_end_off.u32[0] = + e->shared_heap_start_off.u32[0] - 1 + shared_heap->size; + e->shared_heap_base_addr_adj = + shared_heap->base_addr - e->shared_heap_start_off.u32[0]; +#endif + } +#endif /* end of WASM_ENABLE_INTERP != 0 */ +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + AOTModuleInstanceExtra *e = + (AOTModuleInstanceExtra *)((AOTModuleInstance *)module_inst)->e; +#if UINTPTR_MAX == UINT64_MAX + if (is_memory64) + e->shared_heap_start_off.u64 = shared_heap->start_off_mem64; + else + e->shared_heap_start_off.u64 = shared_heap->start_off_mem32; + e->shared_heap_end_off.u64 = + e->shared_heap_start_off.u64 - 1 + shared_heap->size; + e->shared_heap_base_addr_adj = + shared_heap->base_addr - e->shared_heap_start_off.u64; +#else + e->shared_heap_start_off.u32[0] = (uint32)shared_heap->start_off_mem32; + e->shared_heap_end_off.u32[0] = + e->shared_heap_start_off.u32[0] - 1 + shared_heap->size; + e->shared_heap_base_addr_adj = + shared_heap->base_addr - e->shared_heap_start_off.u32[0]; +#endif + } +#endif /* end of WASM_ENABLE_AOT != 0 */ +} + bool wasm_runtime_attach_shared_heap_internal(WASMModuleInstanceCommon *module_inst, WASMSharedHeap *shared_heap) @@ -358,20 +484,6 @@ wasm_runtime_attach_shared_heap_internal(WASMModuleInstanceCommon *module_inst, return false; } e->shared_heap = shared_heap; -#if WASM_ENABLE_JIT != 0 -#if UINTPTR_MAX == UINT64_MAX - if (memory->is_memory64) - e->shared_heap_start_off.u64 = shared_heap->start_off_mem64; - else - e->shared_heap_start_off.u64 = shared_heap->start_off_mem32; - e->shared_heap_base_addr_adj = - shared_heap->base_addr - e->shared_heap_start_off.u64; -#else - e->shared_heap_start_off.u32[0] = (uint32)shared_heap->start_off_mem32; - e->shared_heap_base_addr_adj = - shared_heap->base_addr - e->shared_heap_start_off.u32[0]; -#endif -#endif /* end of WASM_ENABLE_JIT != 0 */ } #endif /* end of WASM_ENABLE_INTERP != 0 */ #if WASM_ENABLE_AOT != 0 @@ -383,20 +495,9 @@ wasm_runtime_attach_shared_heap_internal(WASMModuleInstanceCommon *module_inst, return false; } e->shared_heap = shared_heap; -#if UINTPTR_MAX == UINT64_MAX - if (memory->is_memory64) - e->shared_heap_start_off.u64 = shared_heap->start_off_mem64; - else - e->shared_heap_start_off.u64 = shared_heap->start_off_mem32; - e->shared_heap_base_addr_adj = - shared_heap->base_addr - e->shared_heap_start_off.u64; -#else - e->shared_heap_start_off.u32[0] = (uint32)shared_heap->start_off_mem32; - e->shared_heap_base_addr_adj = - shared_heap->base_addr - e->shared_heap_start_off.u32[0]; -#endif } #endif /* end of WASM_ENABLE_AOT != 0 */ + update_last_used_shared_heap(module_inst, shared_heap, memory->is_memory64); os_mutex_lock(&shared_heap_list_lock); shared_heap->attached_count++; @@ -428,14 +529,14 @@ wasm_runtime_detach_shared_heap_internal(WASMModuleInstanceCommon *module_inst) os_mutex_unlock(&shared_heap_list_lock); } e->shared_heap = NULL; -#if WASM_ENABLE_JIT != 0 #if UINTPTR_MAX == UINT64_MAX e->shared_heap_start_off.u64 = UINT64_MAX; + e->shared_heap_end_off.u64 = UINT64_MAX - 1; #else e->shared_heap_start_off.u32[0] = UINT32_MAX; + e->shared_heap_end_off.u32[0] = UINT32_MAX - 1; #endif e->shared_heap_base_addr_adj = NULL; -#endif } #endif /* end of WASM_ENABLE_INTERP != 0 */ #if WASM_ENABLE_AOT != 0 @@ -450,8 +551,10 @@ wasm_runtime_detach_shared_heap_internal(WASMModuleInstanceCommon *module_inst) e->shared_heap = NULL; #if UINTPTR_MAX == UINT64_MAX e->shared_heap_start_off.u64 = UINT64_MAX; + e->shared_heap_end_off.u64 = UINT64_MAX - 1; #else e->shared_heap_start_off.u32[0] = UINT32_MAX; + e->shared_heap_end_off.u32[0] = UINT32_MAX - 1; #endif e->shared_heap_base_addr_adj = NULL; } @@ -495,8 +598,7 @@ wasm_runtime_get_shared_heap(WASMModuleInstanceCommon *module_inst_comm) static bool is_app_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst, - bool is_memory64, uint64 app_offset, uint32 bytes, - WASMSharedHeap **target_heap) + bool is_memory64, uint64 app_offset, uint32 bytes) { WASMSharedHeap *heap = get_shared_heap(module_inst), *cur; uint64 shared_heap_start, shared_heap_end; @@ -509,6 +611,14 @@ is_app_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst, bytes = 1; } + shared_heap_start = + (uint64)get_last_used_shared_heap_start_offset(module_inst); + shared_heap_end = (uint64)get_last_used_shared_heap_end_offset(module_inst); + if (app_offset >= shared_heap_start + && app_offset <= shared_heap_end - bytes + 1) { + return true; + } + /* Early stop for app start address not in the shared heap(chain) at all */ shared_heap_start = is_memory64 ? heap->start_off_mem64 : heap->start_off_mem32; @@ -518,31 +628,27 @@ is_app_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst, goto fail; } - /* Find the exact shared heap that app addr is in */ - if (target_heap) { - for (cur = heap; cur; cur = cur->chain_next) { - shared_heap_start = - is_memory64 ? cur->start_off_mem64 : cur->start_off_mem32; - shared_heap_end = shared_heap_start - 1 + cur->size; - if (app_offset >= shared_heap_start - && app_offset <= shared_heap_end - bytes + 1) { - *target_heap = cur; - return true; - } + /* Find the exact shared heap that app addr is in, and update last used + * shared heap info in module inst extra */ + for (cur = heap; cur; cur = cur->chain_next) { + shared_heap_start = + is_memory64 ? cur->start_off_mem64 : cur->start_off_mem32; + shared_heap_end = shared_heap_start - 1 + cur->size; + if (app_offset >= shared_heap_start + && app_offset <= shared_heap_end - bytes + 1) { + update_last_used_shared_heap(module_inst, cur, is_memory64); + return true; } } return true; fail: - if (target_heap) - *target_heap = NULL; return false; } static bool is_native_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst, - uint8 *addr, uint32 bytes, - WASMSharedHeap **target_heap) + bool is_memory64, uint8 *addr, uint32 bytes) { WASMSharedHeap *cur, *heap = get_shared_heap(module_inst); uintptr_t base_addr, addr_int, end_addr; @@ -567,14 +673,11 @@ is_native_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst, if (end_addr > base_addr + cur->size) continue; - if (target_heap) - *target_heap = cur; + update_last_used_shared_heap(module_inst, cur, is_memory64); return true; } fail: - if (target_heap) - *target_heap = NULL; return false; } @@ -908,7 +1011,7 @@ wasm_runtime_validate_app_addr(WASMModuleInstanceCommon *module_inst_comm, #if WASM_ENABLE_SHARED_HEAP != 0 if (is_app_addr_in_shared_heap(module_inst_comm, memory_inst->is_memory64, - app_offset, size, NULL)) { + app_offset, size)) { return true; } #endif @@ -945,6 +1048,10 @@ wasm_runtime_validate_app_str_addr(WASMModuleInstanceCommon *module_inst_comm, WASMMemoryInstance *memory_inst; uint64 app_end_offset, max_linear_memory_size = MAX_LINEAR_MEMORY_SIZE; char *str, *str_end; +#if WASM_ENABLE_SHARED_HEAP != 0 + uintptr_t shared_heap_end_off; + char *shared_heap_base_addr_adj; +#endif bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode || module_inst_comm->module_type == Wasm_Module_AoT); @@ -959,14 +1066,14 @@ wasm_runtime_validate_app_str_addr(WASMModuleInstanceCommon *module_inst_comm, } #if WASM_ENABLE_SHARED_HEAP != 0 - WASMSharedHeap *shared_heap; if (is_app_addr_in_shared_heap(module_inst_comm, memory_inst->is_memory64, - app_str_offset, 1, &shared_heap)) { - str = (char *)shared_heap->base_addr - + (memory_inst->is_memory64 - ? (app_str_offset - shared_heap->start_off_mem64) - : (app_str_offset - shared_heap->start_off_mem32)); - str_end = (char *)shared_heap->base_addr + shared_heap->size; + app_str_offset, 1)) { + shared_heap_end_off = + get_last_used_shared_heap_end_offset(module_inst_comm); + shared_heap_base_addr_adj = + (char *)get_last_used_shared_heap_base_addr_adj(module_inst_comm); + str = shared_heap_base_addr_adj + app_str_offset; + str_end = shared_heap_base_addr_adj + shared_heap_end_off; } else #endif @@ -1031,8 +1138,8 @@ wasm_runtime_validate_native_addr(WASMModuleInstanceCommon *module_inst_comm, } #if WASM_ENABLE_SHARED_HEAP != 0 - if (is_native_addr_in_shared_heap(module_inst_comm, native_ptr, size, - NULL)) { + if (is_native_addr_in_shared_heap( + module_inst_comm, memory_inst->is_memory64, native_ptr, size)) { return true; } #endif @@ -1060,9 +1167,6 @@ wasm_runtime_addr_app_to_native(WASMModuleInstanceCommon *module_inst_comm, WASMMemoryInstance *memory_inst; uint8 *addr; bool bounds_checks; -#if WASM_ENABLE_SHARED_HEAP != 0 - WASMSharedHeap *shared_heap; -#endif bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode || module_inst_comm->module_type == Wasm_Module_AoT); @@ -1076,17 +1180,9 @@ wasm_runtime_addr_app_to_native(WASMModuleInstanceCommon *module_inst_comm, #if WASM_ENABLE_SHARED_HEAP != 0 if (is_app_addr_in_shared_heap(module_inst_comm, memory_inst->is_memory64, - app_offset, 1, &shared_heap)) { - uint64 shared_heap_start = 0; - - if (memory_inst && !memory_inst->is_memory64) { - shared_heap_start = shared_heap->start_off_mem32; - } - else if (memory_inst && memory_inst->is_memory64) { - shared_heap_start = shared_heap->start_off_mem64; - } - - return shared_heap->base_addr + app_offset - shared_heap_start; + app_offset, 1)) { + return get_last_used_shared_heap_base_addr_adj(module_inst_comm) + + app_offset; } #endif @@ -1135,19 +1231,9 @@ wasm_runtime_addr_native_to_app(WASMModuleInstanceCommon *module_inst_comm, } #if WASM_ENABLE_SHARED_HEAP != 0 - WASMSharedHeap *shared_heap; - if (is_native_addr_in_shared_heap(module_inst_comm, addr, 1, - &shared_heap)) { - uint64 shared_heap_start = 0; - - if (memory_inst && !memory_inst->is_memory64) { - shared_heap_start = shared_heap->start_off_mem32; - } - else if (memory_inst && memory_inst->is_memory64) { - shared_heap_start = shared_heap->start_off_mem64; - } - - return shared_heap_start + (addr - shared_heap->base_addr); + if (is_native_addr_in_shared_heap(module_inst_comm, + memory_inst->is_memory64, addr, 1)) { + return addr - get_last_used_shared_heap_base_addr_adj(module_inst_comm); } #endif @@ -1249,8 +1335,8 @@ wasm_check_app_addr_and_convert(WASMModuleInstance *module_inst, bool is_str, uint8 *native_addr; bool bounds_checks; #if WASM_ENABLE_SHARED_HEAP != 0 - WASMSharedHeap *shared_heap; - bool is_in_shared_heap = false; + uint8 *shared_heap_base_addr_adj = NULL; + uintptr_t shared_heap_end_off = 0; #endif bh_assert(app_buf_addr <= UINTPTR_MAX && app_buf_size <= UINTPTR_MAX); @@ -1263,12 +1349,12 @@ wasm_check_app_addr_and_convert(WASMModuleInstance *module_inst, bool is_str, #if WASM_ENABLE_SHARED_HEAP != 0 if (is_app_addr_in_shared_heap((WASMModuleInstanceCommon *)module_inst, memory_inst->is_memory64, app_buf_addr, - app_buf_size, &shared_heap)) { - native_addr = shared_heap->base_addr - + (memory_inst->is_memory64 - ? (app_buf_addr - shared_heap->start_off_mem64) - : (app_buf_addr - shared_heap->start_off_mem32)); - is_in_shared_heap = true; + app_buf_size)) { + shared_heap_base_addr_adj = get_last_used_shared_heap_base_addr_adj( + (WASMModuleInstanceCommon *)module_inst); + shared_heap_end_off = get_last_used_shared_heap_end_offset( + (WASMModuleInstanceCommon *)module_inst); + native_addr = shared_heap_base_addr_adj + app_buf_addr; } else #endif @@ -1287,12 +1373,12 @@ wasm_check_app_addr_and_convert(WASMModuleInstance *module_inst, bool is_str, } #if WASM_ENABLE_SHARED_HEAP != 0 - if (is_in_shared_heap) { + if (shared_heap_base_addr_adj) { const char *str, *str_end; /* The whole string must be in the linear memory */ str = (const char *)native_addr; - str_end = (const char *)shared_heap->base_addr + shared_heap->size; + str_end = (const char *)shared_heap_base_addr_adj + shared_heap_end_off; while (str < str_end && *str != '\0') str++; if (str == str_end) { diff --git a/core/iwasm/interpreter/wasm_interp_classic.c b/core/iwasm/interpreter/wasm_interp_classic.c index ab1486aa97..317392f1a9 100644 --- a/core/iwasm/interpreter/wasm_interp_classic.c +++ b/core/iwasm/interpreter/wasm_interp_classic.c @@ -1623,14 +1623,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, is_memory64 = memory->is_memory64; #endif #if WASM_ENABLE_SHARED_HEAP != 0 - WASMSharedHeap *shared_heap = module->e->shared_heap; - uint8 *shared_heap_base_addr = shared_heap ? shared_heap->base_addr : NULL; - uint64 shared_heap_start_off = - shared_heap ? get_shared_heap_start_off(shared_heap) : 0; - uint64 shared_heap_end_off = - shared_heap - ? (get_shared_heap_start_off(shared_heap) - 1 + shared_heap->size) - : 0; + WASMModuleInstanceExtra *e = module->e; #endif /* end of WASM_ENABLE_SHARED_HEAP != 0 */ #if WASM_ENABLE_MULTI_MEMORY != 0 uint32 memidx = 0; @@ -5780,13 +5773,15 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, CHECK_BULK_MEMORY_OVERFLOW(dst, len, mdst); #if WASM_ENABLE_SHARED_HEAP != 0 if (app_addr_in_shared_heap((uint64)dst, len)) - dlen = shared_heap_end_off - dst + 1; + dlen = + get_last_used_shared_heap_end_off() - dst + 1; #endif #else /* else of OS_ENABLE_HW_BOUND_CHECK */ #if WASM_ENABLE_SHARED_HEAP != 0 if (app_addr_in_shared_heap((uint64)dst, len)) { shared_heap_addr_app_to_native((uint64)dst, mdst); - dlen = shared_heap_end_off - dst + 1; + dlen = + get_last_used_shared_heap_end_off() - dst + 1; } else #endif diff --git a/core/iwasm/interpreter/wasm_interp_fast.c b/core/iwasm/interpreter/wasm_interp_fast.c index a62b6b2dec..c7c3efd6ac 100644 --- a/core/iwasm/interpreter/wasm_interp_fast.c +++ b/core/iwasm/interpreter/wasm_interp_fast.c @@ -1525,17 +1525,11 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, /* TODO: currently flowing two variables are only dummy for shared heap * boundary check, need to be updated when multi-memory or memory64 * proposals are to be implemented */ + WASMModuleInstanceExtra *e = module->e; bool is_memory64 = false; uint32 memidx = 0; - - WASMSharedHeap *shared_heap = module->e ? module->e->shared_heap : NULL; - uint8 *shared_heap_base_addr = shared_heap ? shared_heap->base_addr : NULL; - uint64 shared_heap_start_off = - shared_heap ? get_shared_heap_start_off(shared_heap) : 0; - uint64 shared_heap_end_off = - shared_heap - ? (get_shared_heap_start_off(shared_heap) - 1 + shared_heap->size) - : 0; + (void)is_memory64; + (void)memidx; /* #endif */ #endif /* end of WASM_ENABLE_SHARED_HEAP != 0 */ @@ -5085,7 +5079,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, CHECK_BULK_MEMORY_OVERFLOW(dst, len, mdst); #if WASM_ENABLE_SHARED_HEAP != 0 if (app_addr_in_shared_heap((uint64)dst, len)) - dlen = shared_heap_end_off - dst + 1; + dlen = (uint64)get_last_used_shared_heap_end_off() - dst + 1; #endif #else /* else of OS_ENABLE_HW_BOUND_CHECK */ #if WASM_ENABLE_SHARED_HEAP != 0 @@ -5102,7 +5096,8 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, #if WASM_ENABLE_SHARED_HEAP != 0 if (app_addr_in_shared_heap((uint64)dst, len)) { shared_heap_addr_app_to_native((uint64)dst, mdst); - dlen = shared_heap_end_off - dst + 1; + dlen = (uint64)get_last_used_shared_heap_end_off() + - dst + 1; } else #endif diff --git a/core/iwasm/interpreter/wasm_loader.h b/core/iwasm/interpreter/wasm_loader.h index 9171d71d37..7ada1a09ac 100644 --- a/core/iwasm/interpreter/wasm_loader.h +++ b/core/iwasm/interpreter/wasm_loader.h @@ -74,47 +74,74 @@ wasm_loader_find_block_addr(WASMExecEnv *exec_env, BlockAddr *block_addr_cache, uint8 **p_end_addr); #if WASM_ENABLE_SHARED_HEAP != 0 + #if WASM_ENABLE_MULTI_MEMORY != 0 /* Only enable shared heap for the default memory */ #define is_default_memory (memidx == 0) #else #define is_default_memory true #endif + #if WASM_ENABLE_MEMORY64 != 0 #define get_shared_heap_start_off(shared_heap) \ (is_memory64 ? shared_heap->start_off_mem64 : shared_heap->start_off_mem32) #else -#define get_shared_heap_start_off(shared_heap) (shared_heap->start_off_mem32) +#define get_shared_heap_start_off(shared_heap) shared_heap->start_off_mem32 #endif + +#if UINTPTR_MAX == UINT64_MAX +#define update_last_used_shared_heap(shared_heap) \ + do { \ + e->shared_heap_start_off.u64 = get_shared_heap_start_off(shared_heap); \ + e->shared_heap_end_off.u64 = \ + e->shared_heap_start_off.u64 - 1 + shared_heap->size; \ + e->shared_heap_base_addr_adj = \ + shared_heap->base_addr - e->shared_heap_start_off.u64; \ + } while (0) +#define get_last_used_shared_heap_start_off() e->shared_heap_start_off.u64 +#define get_last_used_shared_heap_end_off() e->shared_heap_end_off.u64 +#else +#define update_last_used_shared_heap(shared_heap) \ + do { \ + e->shared_heap_start_off.u32[0] = \ + (uint32)shared_heap->start_off_mem32; \ + e->shared_heap_end_off.u32[0] = \ + e->shared_heap_start_off.u32[0] - 1 + shared_heap->size; \ + e->shared_heap_base_addr_adj = \ + shared_heap->base_addr - e->shared_heap_start_off.u32[0]; \ + } while (0) +#define get_last_used_shared_heap_start_off() \ + (uint64) e->shared_heap_start_off.u32[0] +#define get_last_used_shared_heap_end_off() \ + (uint64) e->shared_heap_end_off.u32[0] +#endif /* end of UINTPTR_MAX == UINT64_MAX */ + /* Check whether the app addr in the last visited shared heap, if not, check the * shared heap chain to find which(if any) shared heap the app addr in, and * update the last visited shared heap info if found. */ #define app_addr_in_shared_heap(app_addr, bytes) \ - (shared_heap && is_default_memory && (app_addr) >= shared_heap_start_off \ - && (app_addr) <= shared_heap_end_off - bytes + 1) \ + (e->shared_heap && is_default_memory \ + && (app_addr) >= get_last_used_shared_heap_start_off() \ + && (app_addr) <= get_last_used_shared_heap_end_off() - bytes + 1) \ || ({ \ bool in_chain = false; \ WASMSharedHeap *cur; \ uint64 cur_shared_heap_start_off, cur_shared_heap_end_off; \ - for (cur = shared_heap; cur; cur = cur->chain_next) { \ + for (cur = e->shared_heap; cur; cur = cur->chain_next) { \ cur_shared_heap_start_off = get_shared_heap_start_off(cur); \ cur_shared_heap_end_off = \ cur_shared_heap_start_off - 1 + cur->size; \ if ((app_addr) >= cur_shared_heap_start_off \ && (app_addr) <= cur_shared_heap_end_off - bytes + 1) { \ - shared_heap_start_off = cur_shared_heap_start_off; \ - shared_heap_end_off = cur_shared_heap_end_off; \ - shared_heap_base_addr = cur->base_addr; \ + update_last_used_shared_heap(cur); \ in_chain = true; \ break; \ } \ } \ in_chain; \ }) - #define shared_heap_addr_app_to_native(app_addr, native_addr) \ - native_addr = shared_heap_base_addr + ((app_addr)-shared_heap_start_off) - + native_addr = e->shared_heap_base_addr_adj + app_addr #define CHECK_SHARED_HEAP_OVERFLOW(app_addr, bytes, native_addr) \ if (app_addr_in_shared_heap(app_addr, bytes)) \ shared_heap_addr_app_to_native(app_addr, native_addr); \ diff --git a/core/iwasm/interpreter/wasm_runtime.h b/core/iwasm/interpreter/wasm_runtime.h index b9d49ec229..0acb8de310 100644 --- a/core/iwasm/interpreter/wasm_runtime.h +++ b/core/iwasm/interpreter/wasm_runtime.h @@ -373,8 +373,6 @@ typedef struct WASMModuleInstanceExtra { #endif #if WASM_ENABLE_SHARED_HEAP != 0 - WASMSharedHeap *shared_heap; -#if WASM_ENABLE_JIT != 0 /* * Adjusted shared heap based addr to simple the calculation * in the aot code. The value is: @@ -382,7 +380,8 @@ typedef struct WASMModuleInstanceExtra { */ uint8 *shared_heap_base_addr_adj; MemBound shared_heap_start_off; -#endif + MemBound shared_heap_end_off; + WASMSharedHeap *shared_heap; #endif #if WASM_ENABLE_DEBUG_INTERP != 0 \ From f173354f2eafb9887050c5991f9efa097ccc0db4 Mon Sep 17 00:00:00 2001 From: TL Date: Fri, 21 Feb 2025 11:29:31 +0800 Subject: [PATCH 14/20] code refactor --- core/iwasm/interpreter/wasm_interp_fast.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/iwasm/interpreter/wasm_interp_fast.c b/core/iwasm/interpreter/wasm_interp_fast.c index c7c3efd6ac..7ebe6c9744 100644 --- a/core/iwasm/interpreter/wasm_interp_fast.c +++ b/core/iwasm/interpreter/wasm_interp_fast.c @@ -5079,7 +5079,8 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, CHECK_BULK_MEMORY_OVERFLOW(dst, len, mdst); #if WASM_ENABLE_SHARED_HEAP != 0 if (app_addr_in_shared_heap((uint64)dst, len)) - dlen = (uint64)get_last_used_shared_heap_end_off() - dst + 1; + dlen = (uint64)get_last_used_shared_heap_end_off() + - dst + 1; #endif #else /* else of OS_ENABLE_HW_BOUND_CHECK */ #if WASM_ENABLE_SHARED_HEAP != 0 From 948d981a5c42fa350dbde8092975b0df4223f11b Mon Sep 17 00:00:00 2001 From: TL Date: Fri, 21 Feb 2025 12:32:25 +0800 Subject: [PATCH 15/20] aot data struct modification --- core/iwasm/aot/aot_runtime.c | 2 ++ core/iwasm/aot/aot_runtime.h | 5 ++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/core/iwasm/aot/aot_runtime.c b/core/iwasm/aot/aot_runtime.c index 0f7b5d3d9a..d3cd27ee98 100644 --- a/core/iwasm/aot/aot_runtime.c +++ b/core/iwasm/aot/aot_runtime.c @@ -60,6 +60,8 @@ bh_static_assert(offsetof(AOTModuleInstanceExtra, stack_sizes) == 0); bh_static_assert(offsetof(AOTModuleInstanceExtra, shared_heap_base_addr_adj) == 8); bh_static_assert(offsetof(AOTModuleInstanceExtra, shared_heap_start_off) == 16); +bh_static_assert(offsetof(AOTModuleInstanceExtra, shared_heap_end_off) == 24); +bh_static_assert(offsetof(AOTModuleInstanceExtra, shared_heap) == 32); bh_static_assert(sizeof(CApiFuncImport) == sizeof(uintptr_t) * 3); diff --git a/core/iwasm/aot/aot_runtime.h b/core/iwasm/aot/aot_runtime.h index 297b2a5b5d..f841bfb6aa 100644 --- a/core/iwasm/aot/aot_runtime.h +++ b/core/iwasm/aot/aot_runtime.h @@ -125,6 +125,8 @@ typedef struct AOTModuleInstanceExtra { */ DefPointer(uint8 *, shared_heap_base_addr_adj); MemBound shared_heap_start_off; + MemBound shared_heap_end_off; + DefPointer(WASMSharedHeap *, shared_heap); WASMModuleInstanceExtraCommon common; @@ -142,9 +144,6 @@ typedef struct AOTModuleInstanceExtra { WASMModuleInstanceCommon **import_func_module_insts; #endif -#if WASM_ENABLE_SHARED_HEAP != 0 - WASMSharedHeap *shared_heap; -#endif } AOTModuleInstanceExtra; #if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) From 035f758734a5a14ff573f4cd4e8a192168c224da Mon Sep 17 00:00:00 2001 From: TL Date: Tue, 25 Feb 2025 15:28:36 +0800 Subject: [PATCH 16/20] code refactor --- core/iwasm/common/wasm_memory.c | 42 ++++------- core/iwasm/common/wasm_memory.h | 46 +++++++++++- core/iwasm/interpreter/wasm_interp_classic.c | 11 +-- core/iwasm/interpreter/wasm_interp_fast.c | 7 +- core/iwasm/interpreter/wasm_loader.h | 77 -------------------- 5 files changed, 64 insertions(+), 119 deletions(-) diff --git a/core/iwasm/common/wasm_memory.c b/core/iwasm/common/wasm_memory.c index 69801a2f7a..b9a42ceff5 100644 --- a/core/iwasm/common/wasm_memory.c +++ b/core/iwasm/common/wasm_memory.c @@ -596,7 +596,7 @@ wasm_runtime_get_shared_heap(WASMModuleInstanceCommon *module_inst_comm) return get_shared_heap(module_inst_comm); } -static bool +bool is_app_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst, bool is_memory64, uint64 app_offset, uint32 bytes) { @@ -641,7 +641,6 @@ is_app_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst, } } - return true; fail: return false; } @@ -1220,11 +1219,6 @@ wasm_runtime_addr_native_to_app(WASMModuleInstanceCommon *module_inst_comm, bounds_checks = is_bounds_checks_enabled(module_inst_comm); -#if WASM_ENABLE_SHARED_HEAP != 0 - /* If shared heap is enabled, bounds check is always needed */ - bounds_checks = true; -#endif - memory_inst = wasm_get_default_memory(module_inst); if (!memory_inst) { return 0; @@ -1350,33 +1344,14 @@ wasm_check_app_addr_and_convert(WASMModuleInstance *module_inst, bool is_str, if (is_app_addr_in_shared_heap((WASMModuleInstanceCommon *)module_inst, memory_inst->is_memory64, app_buf_addr, app_buf_size)) { + const char *str, *str_end; shared_heap_base_addr_adj = get_last_used_shared_heap_base_addr_adj( (WASMModuleInstanceCommon *)module_inst); shared_heap_end_off = get_last_used_shared_heap_end_offset( (WASMModuleInstanceCommon *)module_inst); native_addr = shared_heap_base_addr_adj + app_buf_addr; - } - else -#endif - { - native_addr = memory_inst->memory_data + (uintptr_t)app_buf_addr; - } - - bounds_checks = - is_bounds_checks_enabled((WASMModuleInstanceCommon *)module_inst); - - if (!bounds_checks) { - if (app_buf_addr == 0) { - native_addr = NULL; - } - goto success; - } -#if WASM_ENABLE_SHARED_HEAP != 0 - if (shared_heap_base_addr_adj) { - const char *str, *str_end; - - /* The whole string must be in the linear memory */ + /* The whole string must be in the shared heap */ str = (const char *)native_addr; str_end = (const char *)shared_heap_base_addr_adj + shared_heap_end_off; while (str < str_end && *str != '\0') @@ -1390,6 +1365,17 @@ wasm_check_app_addr_and_convert(WASMModuleInstance *module_inst, bool is_str, } #endif + native_addr = memory_inst->memory_data + (uintptr_t)app_buf_addr; + bounds_checks = + is_bounds_checks_enabled((WASMModuleInstanceCommon *)module_inst); + + if (!bounds_checks) { + if (app_buf_addr == 0) { + native_addr = NULL; + } + goto success; + } + /* No need to check the app_offset and buf_size if memory access boundary check with hardware trap is enabled */ #ifndef OS_ENABLE_HW_BOUND_CHECK diff --git a/core/iwasm/common/wasm_memory.h b/core/iwasm/common/wasm_memory.h index 88d0f3c5df..48bd2179fe 100644 --- a/core/iwasm/common/wasm_memory.h +++ b/core/iwasm/common/wasm_memory.h @@ -41,7 +41,51 @@ SET_LINEAR_MEMORY_SIZE(WASMMemoryInstance *memory, uint64 size) #define SET_LINEAR_MEMORY_SIZE(memory, size) memory->memory_data_size = size #endif +#if WASM_ENABLE_INTERP != 0 #if WASM_ENABLE_SHARED_HEAP != 0 + +#if WASM_ENABLE_MULTI_MEMORY != 0 +/* Only enable shared heap for the default memory */ +#define is_default_memory (memidx == 0) +#else +#define is_default_memory true +#endif + +#if UINTPTR_MAX == UINT64_MAX +#define get_shared_heap_end_off() module->e->shared_heap_end_off.u64 +#else +#define get_shared_heap_end_off() \ + (uint64)(module->e->shared_heap_end_off.u32[0]) +#endif + +#if WASM_ENABLE_MEMORY64 != 0 +#define shared_heap_is_memory64 is_memory64 +#else +#define shared_heap_is_memory64 false +#endif + +#define app_addr_in_shared_heap(app_addr, bytes) \ + (is_default_memory \ + && is_app_addr_in_shared_heap((WASMModuleInstanceCommon *)module, \ + shared_heap_is_memory64, (uint64)app_addr, \ + bytes)) +#define shared_heap_addr_app_to_native(app_addr, native_addr) \ + native_addr = module->e->shared_heap_base_addr_adj + app_addr +#define CHECK_SHARED_HEAP_OVERFLOW(app_addr, bytes, native_addr) \ + if (app_addr_in_shared_heap(app_addr, bytes)) \ + shared_heap_addr_app_to_native(app_addr, native_addr); \ + else + +#else /* else of WASM_ENABLE_SHARED_HEAP != 0 */ +#define CHECK_SHARED_HEAP_OVERFLOW(app_addr, bytes, native_addr) +#endif /* end of WASM_ENABLE_SHARED_HEAP != 0 */ +#endif /* end of WASM_ENABLE_INTERP != 0 */ + +#if WASM_ENABLE_SHARED_HEAP != 0 +bool +is_app_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst, + bool is_memory64, uint64 app_offset, uint32 bytes); + WASMSharedHeap * wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args); @@ -74,7 +118,7 @@ wasm_runtime_shared_heap_malloc(WASMModuleInstanceCommon *module_inst, void wasm_runtime_shared_heap_free(WASMModuleInstanceCommon *module_inst, uint64 ptr); -#endif +#endif /* end of WASM_ENABLE_SHARED_HEAP != 0 */ bool wasm_runtime_memory_init(mem_alloc_type_t mem_alloc_type, diff --git a/core/iwasm/interpreter/wasm_interp_classic.c b/core/iwasm/interpreter/wasm_interp_classic.c index 317392f1a9..9514e1e64f 100644 --- a/core/iwasm/interpreter/wasm_interp_classic.c +++ b/core/iwasm/interpreter/wasm_interp_classic.c @@ -1622,9 +1622,6 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, if (memory) is_memory64 = memory->is_memory64; #endif -#if WASM_ENABLE_SHARED_HEAP != 0 - WASMModuleInstanceExtra *e = module->e; -#endif /* end of WASM_ENABLE_SHARED_HEAP != 0 */ #if WASM_ENABLE_MULTI_MEMORY != 0 uint32 memidx = 0; uint32 memidx_cached = (uint32)-1; @@ -2399,7 +2396,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, else cur_func_type = cur_func->u.func->func_type; - /* clang-format off */ + /* clang-format off */ #if WASM_ENABLE_GC == 0 if (cur_type != cur_func_type) { wasm_set_exception(module, "indirect call type mismatch"); @@ -5773,15 +5770,13 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, CHECK_BULK_MEMORY_OVERFLOW(dst, len, mdst); #if WASM_ENABLE_SHARED_HEAP != 0 if (app_addr_in_shared_heap((uint64)dst, len)) - dlen = - get_last_used_shared_heap_end_off() - dst + 1; + dlen = get_shared_heap_end_off() - dst + 1; #endif #else /* else of OS_ENABLE_HW_BOUND_CHECK */ #if WASM_ENABLE_SHARED_HEAP != 0 if (app_addr_in_shared_heap((uint64)dst, len)) { shared_heap_addr_app_to_native((uint64)dst, mdst); - dlen = - get_last_used_shared_heap_end_off() - dst + 1; + dlen = get_shared_heap_end_off() - dst + 1; } else #endif diff --git a/core/iwasm/interpreter/wasm_interp_fast.c b/core/iwasm/interpreter/wasm_interp_fast.c index 7ebe6c9744..8538d376e1 100644 --- a/core/iwasm/interpreter/wasm_interp_fast.c +++ b/core/iwasm/interpreter/wasm_interp_fast.c @@ -1525,7 +1525,6 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, /* TODO: currently flowing two variables are only dummy for shared heap * boundary check, need to be updated when multi-memory or memory64 * proposals are to be implemented */ - WASMModuleInstanceExtra *e = module->e; bool is_memory64 = false; uint32 memidx = 0; (void)is_memory64; @@ -5079,8 +5078,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, CHECK_BULK_MEMORY_OVERFLOW(dst, len, mdst); #if WASM_ENABLE_SHARED_HEAP != 0 if (app_addr_in_shared_heap((uint64)dst, len)) - dlen = (uint64)get_last_used_shared_heap_end_off() - - dst + 1; + dlen = (uint64)get_shared_heap_end_off() - dst + 1; #endif #else /* else of OS_ENABLE_HW_BOUND_CHECK */ #if WASM_ENABLE_SHARED_HEAP != 0 @@ -5097,8 +5095,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, #if WASM_ENABLE_SHARED_HEAP != 0 if (app_addr_in_shared_heap((uint64)dst, len)) { shared_heap_addr_app_to_native((uint64)dst, mdst); - dlen = (uint64)get_last_used_shared_heap_end_off() - - dst + 1; + dlen = (uint64)get_shared_heap_end_off() - dst + 1; } else #endif diff --git a/core/iwasm/interpreter/wasm_loader.h b/core/iwasm/interpreter/wasm_loader.h index 7ada1a09ac..676770ee22 100644 --- a/core/iwasm/interpreter/wasm_loader.h +++ b/core/iwasm/interpreter/wasm_loader.h @@ -73,83 +73,6 @@ wasm_loader_find_block_addr(WASMExecEnv *exec_env, BlockAddr *block_addr_cache, uint8 block_type, uint8 **p_else_addr, uint8 **p_end_addr); -#if WASM_ENABLE_SHARED_HEAP != 0 - -#if WASM_ENABLE_MULTI_MEMORY != 0 -/* Only enable shared heap for the default memory */ -#define is_default_memory (memidx == 0) -#else -#define is_default_memory true -#endif - -#if WASM_ENABLE_MEMORY64 != 0 -#define get_shared_heap_start_off(shared_heap) \ - (is_memory64 ? shared_heap->start_off_mem64 : shared_heap->start_off_mem32) -#else -#define get_shared_heap_start_off(shared_heap) shared_heap->start_off_mem32 -#endif - -#if UINTPTR_MAX == UINT64_MAX -#define update_last_used_shared_heap(shared_heap) \ - do { \ - e->shared_heap_start_off.u64 = get_shared_heap_start_off(shared_heap); \ - e->shared_heap_end_off.u64 = \ - e->shared_heap_start_off.u64 - 1 + shared_heap->size; \ - e->shared_heap_base_addr_adj = \ - shared_heap->base_addr - e->shared_heap_start_off.u64; \ - } while (0) -#define get_last_used_shared_heap_start_off() e->shared_heap_start_off.u64 -#define get_last_used_shared_heap_end_off() e->shared_heap_end_off.u64 -#else -#define update_last_used_shared_heap(shared_heap) \ - do { \ - e->shared_heap_start_off.u32[0] = \ - (uint32)shared_heap->start_off_mem32; \ - e->shared_heap_end_off.u32[0] = \ - e->shared_heap_start_off.u32[0] - 1 + shared_heap->size; \ - e->shared_heap_base_addr_adj = \ - shared_heap->base_addr - e->shared_heap_start_off.u32[0]; \ - } while (0) -#define get_last_used_shared_heap_start_off() \ - (uint64) e->shared_heap_start_off.u32[0] -#define get_last_used_shared_heap_end_off() \ - (uint64) e->shared_heap_end_off.u32[0] -#endif /* end of UINTPTR_MAX == UINT64_MAX */ - -/* Check whether the app addr in the last visited shared heap, if not, check the - * shared heap chain to find which(if any) shared heap the app addr in, and - * update the last visited shared heap info if found. */ -#define app_addr_in_shared_heap(app_addr, bytes) \ - (e->shared_heap && is_default_memory \ - && (app_addr) >= get_last_used_shared_heap_start_off() \ - && (app_addr) <= get_last_used_shared_heap_end_off() - bytes + 1) \ - || ({ \ - bool in_chain = false; \ - WASMSharedHeap *cur; \ - uint64 cur_shared_heap_start_off, cur_shared_heap_end_off; \ - for (cur = e->shared_heap; cur; cur = cur->chain_next) { \ - cur_shared_heap_start_off = get_shared_heap_start_off(cur); \ - cur_shared_heap_end_off = \ - cur_shared_heap_start_off - 1 + cur->size; \ - if ((app_addr) >= cur_shared_heap_start_off \ - && (app_addr) <= cur_shared_heap_end_off - bytes + 1) { \ - update_last_used_shared_heap(cur); \ - in_chain = true; \ - break; \ - } \ - } \ - in_chain; \ - }) -#define shared_heap_addr_app_to_native(app_addr, native_addr) \ - native_addr = e->shared_heap_base_addr_adj + app_addr -#define CHECK_SHARED_HEAP_OVERFLOW(app_addr, bytes, native_addr) \ - if (app_addr_in_shared_heap(app_addr, bytes)) \ - shared_heap_addr_app_to_native(app_addr, native_addr); \ - else -#else -#define CHECK_SHARED_HEAP_OVERFLOW(app_addr, bytes, native_addr) -#endif /* end of WASM_ENABLE_SHARED_HEAP != 0 */ - #ifdef __cplusplus } #endif From a62d10400f2eed1b50b47e20bf622f1ca7533f72 Mon Sep 17 00:00:00 2001 From: TL Date: Tue, 25 Feb 2025 15:39:57 +0800 Subject: [PATCH 17/20] format --- core/iwasm/interpreter/wasm_interp_classic.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/iwasm/interpreter/wasm_interp_classic.c b/core/iwasm/interpreter/wasm_interp_classic.c index 9514e1e64f..ddf47aeee7 100644 --- a/core/iwasm/interpreter/wasm_interp_classic.c +++ b/core/iwasm/interpreter/wasm_interp_classic.c @@ -2396,7 +2396,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, else cur_func_type = cur_func->u.func->func_type; - /* clang-format off */ + /* clang-format off */ #if WASM_ENABLE_GC == 0 if (cur_type != cur_func_type) { wasm_set_exception(module, "indirect call type mismatch"); From c5a33e06cd423352085b27ad80072091f529182d Mon Sep 17 00:00:00 2001 From: TianlongLiang <111852609+TianlongLiang@users.noreply.github.com> Date: Thu, 27 Feb 2025 10:07:03 +0800 Subject: [PATCH 18/20] Unit test and sample update for shared heap enhancement (#4070) - preallocated shared heap and shared heap chain - update preallocated shared heap and shared heap chain sample and document - shared heap unit test and sample - update document and build script Co-authored-by: liang.he --- samples/shared-heap/CMakeLists.txt | 8 +- samples/shared-heap/README.md | 50 ++ .../shared-heap/images/shared_heap_chain.png | Bin 0 -> 111396 bytes samples/shared-heap/src/main.c | 2 +- samples/shared-heap/src/shared_heap_chain.c | 321 +++++++++ samples/shared-heap/wasm-apps/CMakeLists.txt | 4 +- samples/shared-heap/wasm-apps/test1.c | 7 + samples/shared-heap/wasm-apps/test2.c | 11 +- tests/unit/CMakeLists.txt | 16 +- tests/unit/shared-heap/shared_heap_test.cc | 612 ++++++++++++++++-- tests/unit/shared-heap/wasm-apps/test.c | 28 + .../shared-heap/wasm-apps/test_addr_conv.c | 14 + 12 files changed, 998 insertions(+), 75 deletions(-) create mode 100644 samples/shared-heap/README.md create mode 100644 samples/shared-heap/images/shared_heap_chain.png create mode 100644 samples/shared-heap/src/shared_heap_chain.c diff --git a/samples/shared-heap/CMakeLists.txt b/samples/shared-heap/CMakeLists.txt index 6346d077e7..5be99d3a22 100644 --- a/samples/shared-heap/CMakeLists.txt +++ b/samples/shared-heap/CMakeLists.txt @@ -83,17 +83,21 @@ target_link_libraries(vmlib ${LLVM_AVAILABLE_LIBS} ${UV_A_LIBS} -lm -ldl -lpthre include_directories(${CMAKE_CURRENT_LIST_DIR}/src) include (${SHARED_DIR}/utils/uncommon/shared_uncommon.cmake) +add_executable (shared_heap_chain_test src/shared_heap_chain.c ${UNCOMMON_SHARED_SOURCE}) add_executable (shared_heap_test src/main.c ${UNCOMMON_SHARED_SOURCE}) check_pie_supported() set_target_properties (shared_heap_test PROPERTIES POSITION_INDEPENDENT_CODE ON) if (APPLE) - target_link_libraries (shared_heap_test vmlib -lm -ldl -lpthread) + set (LIBS vmlib -lm -ldl -lpthread) else () - target_link_libraries (shared_heap_test vmlib -lm -ldl -lpthread -lrt) + set (LIBS vmlib -lm -ldl -lpthread -lrt) endif () +target_link_libraries (shared_heap_chain_test ${LIBS}) +target_link_libraries (shared_heap_test ${LIBS}) + add_subdirectory(wasm-apps) if (WAMR_BUILD_AOT EQUAL 1) diff --git a/samples/shared-heap/README.md b/samples/shared-heap/README.md new file mode 100644 index 0000000000..6e43ceab91 --- /dev/null +++ b/samples/shared-heap/README.md @@ -0,0 +1,50 @@ +# Shared heap Sample introduction + +This is a sample to show how to use the shared heap feature in WAMR. The shared heap feature allows multiple WASM instances to share the same memory space. This feature is useful when you want to run multiple WASM instances in the same process and share data between them. The sandbox nature of WASM is still maintained in the shared heap by WAMR. But the data management and correct data synchronization in shared heap is relied on the user's implementation. + +> Note: The shared heap feature is experimental feature, it should be used with caution. It's optional and only available when building WAMR with the CMake cache variable `WAMR_BUILD_SHARED_HEAP` set to 1. + +## Build and run the sample + +To build the shared heap used in multi thread sample and the shared heap chain sample with following commands: + +```bash +cmake -S . -B build +cmake --build build +``` + +For the shared heap sample, it demonstrates how to create a shared heap and use it shares data between two WASM instances, which would satisfy most of the use cases. Use the following commands to run the sample: + +```bash +cd build +./shared_heap_test +``` + +For the shared heap chain sample. It chains a pre-allocated heap and a normal shared heap to one chain(linked list) as a whole and attaches/detaches all together, and pass the WASM address directly between two WASM instances. Use the following commands to run the sample: + +```bash +cd build +./shared_heap_chain_test +``` + +## How to use shared heap + +The shared heap is an advanced feature in WAMR that gives the user flexibility to share data between multiple WASM instances(it will be the same address mapping for different WASM instance) or between WebAssembly and the host without incurring any copy overhead. The shared heap can be regarded as an extension of the WebAssembly linear memory. But it also heavily relies on the user's implementation to manage the shared data correctly. The following are some takeaway points to help the user use the shared heap correctly. + +### Create and manage shared heap + +You can create a shared heap by calling the `wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args)` API. And based on the `init_args`, you can create a shared heap in two ways: + +1. WAMR managed shared heap: when only `init_args.size` is given and `init_args.pre_allocated_addr` stays as NULL, WAMR will allocate a shared heap(not from the linear memory) with the given size. The shared heap will be managed by WAMR, the wasm app or host(WAMR users) can dynamically manage memory from it by calling `wasm_runtime_shared_heap_malloc()` and `wasm_runtime_shared_heap_free()` on demand. Only the memory allocated from the shared heap is valid and can be shared, not the unallocated part of shared heap memory. And it will be automatically freed when runtime is destroyed(when `wasm_runtime_destroy()` is called). + +2. Preallocated shared heap: the user can also use a pre-allocated memory(it can be allocated from the system heap, or is a static global buffer, the correctness of its accessibility and size needs to be ensured by the user) as a shared heap by giving `init_args.pre_allocated_addr` and `init_args.size`. This kind of shared heap serves as an area for data exchange, primarily between the host and WebAssembly. Any data within this area can be directly accessed by both sides (assuming the layout of the data structure is known). For instance, the host can store large structured variables in this space, allowing the WebAssembly application to operate on them without the need for copying. And the pre-allocated memory will relies on user to manage its life cycle. + +After creation, the shared heap can be attached to a WASM instance(an additional segment appended to the end of the linear memory) by calling `wasm_runtime_attach_shared_heap(wasm_module_inst_t module_inst, wasm_shared_heap_t shared_heap)`. And it can be detached by calling `wasm_runtime_detach_shared_heap(wasm_module_inst_t module_inst)`. So that the data sharing can only happen between the WASM instances that have the same shared heap attached, complete by user's choice. + +#### Shared heap chain + +Sometimes you may want to use multiple shared heaps to attach together as a chain(linked list) and to share data more flexibly. You can call `wasm_runtime_chain_shared_heaps(wasm_shared_heap_t head, wasm_shared_heap_t body)` to chain two shared heaps together. The shared heap list remains one continuous shared heap in wasm app's point of view. To create a shared heap chain, the shared heaps can't be currently attached to any WASM instance. + +> PS: At most one shared heap in shared heap list can be WAMR managed shared heap, the rest have to be the pre-allocated shared heap. + +![shared-heap-chain](./images/shared_heap_chain.png) diff --git a/samples/shared-heap/images/shared_heap_chain.png b/samples/shared-heap/images/shared_heap_chain.png new file mode 100644 index 0000000000000000000000000000000000000000..740a709d49b3bbb2a8d8423221e18e10953e789a GIT binary patch literal 111396 zcmeFZg;$hc^fx+)hyntlgf!CK-Q5i$ty0p$fOIG+HFSq`GfGN}NDM6?NP`SBgmm}& z4EX)M?_KNOb?+Z=U5n*<=6U8k=j^@D-k<&1`^-lz4aJ98q*x#j=%KQboDK+tK?ee% z%iX^Rd{U}PLjnAU2G&uO0aXo=uK^c#ZKTzuL7*=X?2A_z!1aT-N`_z%h$0O2hXz^M zwE}^z3zg-h^*qh~&S7}!+7M&z=pGZXcFjG-@o3;C+1lcn4R!e1JoDDWgM@1~%=a6~ zR#oAa`4Fs?UOhp;F}M5rSj;Ql2Mh*R#s>_Yw^`G?hsxXsUUmDAnkbd^_`eGfXo38W z^8Z|g^WUdK`|pZ+f;#r@e;0u-f}{xky8yj@`2TS9e4hx4(`4~g3)@dWa*^wJZ&^iwYdncRoSA(yCt}%-*dGm zv@*aAZ`=oYZHcROY*AWT+o<9u^~b8FOjxG%=_fxRB#`l+bhe$hpg5gtQSYG4=9zis zrQ>>1dZ*FNP8tz|%L8%A3CJO;%6me4-oKYEir42Gmcy!Dz0eew7R?`Wa+=VSEDrx- zON`Ox%{ojrZEcEnO!c!TO||!U@0y8@EBtSN(kDyw<~U@P?4G}JQ$0fp?qFhJe$06+ z++tN^7Z2;fed=TS4W`94Y_A~(k9R7g`4m$`L(o&mfEuDSuZ2@vEW5>2qL@>sZ_0vc z2L?X-`jQ@_)t`7*IP%$ijG*`xb@Qk^|M`>|IFmSTI8F7*pJi1dC+6Nl^~bJuTC!D8 z&3%K|$vEBIVfxMa81ga$93PWBuff%9S=oI(U-o9`w3->_)r495#B<9mT2}hC`H4m0 zKbyOk$bbXeE!VwfcY}S|#}01BC_PZKMaFevM(I(a0~bnC2SqdZdlFvRY^DjSpDdFE z_%UX^K0RbIZ>Mk8k%tX@&y;Ctp>J89^rX1RxTv~Rt;g7lSI;JYa&XkDcAnj;^ln8p zt}(y$=%I{EuiuKbx1euCML&aqKsEartzm(Z0vaSce&JUJJm8XO%7BSkns&e0HOQCK zG~w{{(kJDw$`b}Kqhj??vhx1d*kE1f^BC&k&2##S8dk&L3ZBAAr+srqI)^`AErTrE zos4Vh?Qu_b;KEbzA)4HT>&j-h&lOghsHpOZl`&kgnmtj|OMcvoN}N5Har-f*X)-Lq zHpcYPpfoPSbu(V?`3QG)U#7HE8Pm2ZUksx&F=pI*PV+I8k@FNH#OxC-|M%xnjyi=gf`>GBiT;UIdx5HD9O^c>$dNw85+iILcC;TVX=WyXI!sv z{W8C#evoWTF#Ynubr7@7Q@xG2DmKSj{D@E`F0C?}4G*QXS2QRu7Rc3dM9wfg7uMUT zkFFx@E;MDprjD!MZ|k%IL&CirsbA2w-&L@qh7np3G3#oT$?Ntu+L~5F^+L%vF68_T zoZ_!H&k5d4!v~o+j*(?Ql=piF`r>C1F|;W$gRnMV<?!x#%m<{|Ni+t1F|#y>Hfnl>w&VM27n+pi6<>6B}tnM02Y zErXg=wYrktlRKW8eN~~!o=?BzkhM-G`Z=AlO0GH2foqPQ)1F6{zGB+*a3Yg;^9o#; zGknVWd1zjzJyg7hQ^hI6Fp?gxU*GrI{W89Dgnx$RG*YPA%R z0sd{h_Rzbtc!Fw2M}exc?@m`T=$6^JX{yhty9ZTks-z{xs%hX zhhP~PWL6HdwoKyA;*sU;hiRundEB$u7LWW4p|UI0miw|CR#ilTs)%((+^(-W8aeJq zY?fJ~%{+yKJ3AlmdM2-pX~kz#^bX8xc!d^Bv%{d3g*t=ag}4#xHu4JBMQek4K>zH- zO*!PA^lKBzZ1Ie7Yf$-XzmsfP&d36eCp#z17|j96Q$MFtQpq)|u-u*Il!mC5FFl;h zaw@=b|`$VQKc*;$6aU0TIuRb-_VU0~;i>H6!WAgelByLyOKlu)& zcmm>V$T1_B65BdK5`su=h(laZ8Xu|XdbfFP39DJBuf!j2k|WZQe(p|Y21Di7b74AY zC`18T`e(aB!LO)n*psaX$-kAvAFWHRT~90F@|QD#YrhKD>nBI`jS~}9!4ehCckIU~ zUMRvs4-#XQsW6?p-^!C3Os@T5QL7q(BU(ex#NnIOZmQDnhs~7aKOJ0ptTc()7&MWj z85;Fj5DKE#vtQ)%9J;^#T7E4-{(VhwEqbr*%@4SzYe??&4@0Z$ABHUFUya7LyB)2Z z4h+8v#aY=2TB$^$D@s7fXAKKA^9O$Y*<5<5XP|b({j3Edf#mGBXm+Ll-pX0S{ydhA zf70b!Ynbb7BTNjlhln{rRjO(j(bJN+HF8fy*y71q913o5-FXinC{XY#gEl%tt3kS; z(iRGy?CivHMNZ=*YDlFI^XeR?Rq1Cs6BVMvG8tUaMv=kuUgk4CXW>?dxyad+N`vf& znvyNA(eNny{T3655Bvvkyd}N8Q)hb8BI)7SSAUuP81DI;YmLm{5!^E>dO(&~dVpS= zT0Ma8-~907dfRGb-)_{K>t98%_mCWzpW2H8H+tSyd|`$>??JiX$*!IRsCoIoT2kXH zm)}ltebFEMe3@6rY_QJ$O4GFYTP~%47#R20M&T%Gm78!bMPl;4$M4KVSD$h>HRHRh zWGofKP@xUK@y{9QWtj|YMdD-glWKj<2cqVD>vGPR5ZYtH45po~awdc3(!tl4+p~f}Al^T1-X)7K+t;eXazu3~!(&>zxLd^Sc zBRe4WIt$Y>i=LK#V=_dCJwsy+;!t!DCKzW?Y1bFepp!1cxHg^6V>U~*6pdM>?E*H% zO~ZL`bnu+AtNP7f)mjK+jwCITYv^bp9O^%~8oBtQqyXzTet;mQ{EFuZfgVWfPLCpDoli#=^-TFE>?dV5|s90IK#CpPIgtM>A+ zRT-M2M9o&p^9B}EKM|Do8?zA-KX1f1ox_p*^wBD`Rw3gw{lmGlo~+bUc0*{I!%p$= zlegD#JXWL`cf2g!EEZHJnHFJSt1V)&XD9n2j%)Ir8u04{aWBpD>Ms6kjI@N|kKiW+ z_$6$mU4uXUu1|A#2%AwW5YILX(^M)f;qZ^%?kcaQXc2eWG&bR7!JXWUAdc?|t-?U~ zEMZ}^U4Cjj8jF)JOLSfCk%lhVLK(%$5%#k9(V+yhA^O-uEJsSut!Jg?~OoQDa@X?~U1LL)T^K3o4bQy(v z<|3vtd|Qu4&QA>YL_F29dFux+u1cQ|G4zloF)f~|Zc5y{t2H`L0c+HkwJQBh45$^h zt2W)hOTRW6ac7||VbTTc?vLkxotn7}MVV=n3^c^NOerb9;(CIjlt8`CGa^a|N;b4- zLHUAlWqfILxM;YsAAUIb;K9enoL_L+^B<91#K6L++jy`a_*%(8P{}eV_QhL%s<6ze zgs{-3UgW4xs0BXB{68k5Z>k>xkKm(i$tGYXf5hq^XJ-BtZ)+LH6T-vj&s_(5_c=wd zbxl0;qH0bLk;T+t@q3w(QaKP(?y5H zev{g3_Rabr;rASo+AU(QP~DfVoOY9}-N(KOS*TF}M@xD@^M7P*_x>-p{y(OA|F2*A zVF9?eK3dfK0)L?0yh2Lsu-cKzdnO?9c0I8|x9NL$nA!NY)Rmxff@(2Q-;E6a_g4rA zp2%sk3g!hCbM}6jAGS0WYTv^_jVkr$B@w|1i=5ZZl?&5qzNmQlgdQc>$})b4eTDnx znb#k%P~1VFC-u%QI5-aV*^a5y&6dRPxwty!6nFluO-UPt3hRMCaBrtcj!K!#+*dgv z3DmcDwn4X&7y46&A zRSr=F-LVxJ+$G+8CRWvF^Q8dlK4<~`R@z@yFN1~QjgC2D8WZiT!u*xAkQO3eS>VRnP? zKqvP;%!#j_a#k|^p42STg=VQv2v_(08z!yM^(K=sS6cWwE>f3YgE%Gk+B3z4s0~TZ?0G|#Jm|2G#&=z0-4i)^Q1Vt;kv@* zfcSk-Yu(TJ%Ce^EpM>)s+%Dqo9UzY?mgJ{xevM%@UFc|`G69XeeMMl2Y{HFI(q3%O z7N@X#$AbIs0SPX*A#zYzS)@TP`^Luvu1bZVz-=H}n}L%ONm{LkqL+j2BIfPh0hy!S z7z{{UqL^c3W2jv_;!993xChO(#3;sOnqT|hAgQ-n{nWYfZQ-D3`@XBO&_@Y1Qz?yT z`vh?0Szpbq>U39SQI1k->B}odztsBJQ71#SVKR?gc#u!5MQcuy&5iID?f;FoYx|Sb z)x0Hnlxjvrla=oi0LT(#2U-4&wo`W_!@%wN9_IbE{i3r0|NP~{(JGGlfU%h(8+8z-R3ymgy{pP zUQd-)YmTXlRD=Hp<^MP6PP?Dxccq#55LlI)wzs4cQQ=feY6aL@Klx_Jz~BFei;QJD zHEYv=HsjFsgto`Z6WF+#SI3NQnaQK}BK^(t)b+QKi-12T0ZAX_w)Q)7fSsK#O6_`5 z;hzDx+`X|hIwP}l$Y_Q?Ewk-)6#!x%`pth%;K`hoMkr_g&tSKyp&ZPjpAuk>unI|d zV=t~wsk|tJ6rOT!G4dLdS>1-AaO|6a-p#5WJgkk1yy?Y%z@ zFg4>B#TE*-4V=3Ftl$Jhh~hM3nOZ1>o_|-OtVLM$Z&_N~M{?bSZfZ)In;`MPC_G29 zb&Ds~7iv-*MSS#g4C`Spiz%r}&8zqS^!9`CcJduq-q&$v^=s-t{-e{>YQO68h161) znzPV|Tlb&d{*c`+fEq*0O9PcFf)z1U>RuCxGn;1o?I&Vv2uVo%rwNIF?~egUorCHr zD2h=~5q1pb8+%vsJlmugqW4e5Cs?-%Ve-Ge&}5;du+uBD4DuNnV~K`+8PZIAJ^S)M zArBA8+%XSfx(O|R_;Vg3SJkhz;iL83!u%ET!~VANLxbx&qyDDKIdw)R|#W$NwIsx!s|=**ysHgI9ZEDiV2;Q}97$PEwC#>UeZi z2nICr;uhk!AlhC(_DgTb)B$1%xHBN~Y0S2yiMvDIgML=*PD12wX^!5+4Db59Y ztk&^qjW_9YJl-a4&`pX4lKLL3T@UyOYO_BK(_5^9y3?Ee;U9wNqu_D1k8Za!ox`nm z=Dh@8vv^U6pz_4LZ-U#a`seIfopaO?Cj3DSF}D7ZVn`g#!v{0GB?S`F@)8@8R6r zhg~=pcX_?%P6c^t>hs3!k@3hF^n)$P$3*M>*VV8DS;HGf1w?R*3h;|BoaPDv_#LNT zbO1j>TAsByiMCP+MviWpR;HzB{P#5Wt*C4O+BM&x(kISu>JMDR@opohnJSeHTOLkN zeY||%lRllBdB>)vhTLt#(!Sw`e~@ll*sN|kTl6+sp`*0xm)V*qHnIKbpx7fmO}LF0 z<3?>2w}PqIpY{lzk4{xlZfc-SsgGXk{Lu*aJGomZ+M@iQ+E{PrQV!(Cxp)ak?LA5F z1dF+Kl=T4lLDuM|Ss~kvPX~Q`dJ{;(`M3H7M~hEcd)@oEs-apD^9NM&0QaMl9*zB< zANg`Cy)S>pT2^>I)X!Wzu-vyW^PY3Z2k3p>vK8KcR=DUBNVyD*{*8`iUgF?gucsA@ z-Pe!xJwri>-kZJ;XwB=^nb9Y25gq8{>Nesg_!`xQ_70n;@lYH;S_5zK6wcL#1Ke~z z@u6s}*Iq%Q>($+V7V-~m1uC7IH7VpwI?<*Ku_p<~VejrHNjyWRl*p}}Nsjj-u`VFAbC?G$8pf_p&r{;@)=JrKOa!zb zdNa|Nu|RzTDCc}q{uMXhGc>Z{B-`64@24^i#XdD_l;+IvU_?$lv?9G(oK(oon{f7C zG)E6)k;4Eo+bZT&wh!*u%tALRlVes+c@Fp4qC)K$mC;fR`pqfne@6cg6b3JUafjkq zF3+}58d||V4Lsc5?>0T~ALmoKQgn88nvybRo&qz`Z;pL50RYaeU_dAT*toP)H*q!i>ui+{{blRfzLTs)je7b9xK95fo;MWRlzK>;HS{KNK3g*Y2J&JLy0R zc#_^P)`V70FQ^y>X}6bcV2)ct)$Dt;UJmoJ+lrcu4FJ7g<*RIU3z9*dw|kK3nBVy0 z9duaf{BU39=lO>@GX4!)@iQNkEtB{k`%SQ2qkM18E3_%1i~mIRO{gaJn*q%YQa_0S z$|v&9n9;OA`G;wxg1gV8@+GIUx1a-0$O)zv0BU;&M&oBw96xDdKi6sUv+w=R-&k$w z*>-&aall3z6t!vk3p(qDLvwhnQqu}CP>>{zhhz0p5zF&HFO6r2$I-4@ZKX`@De|~I zfSHb3D~)FW;OZ;DR=Pz|*q%GVA0+&BilVr9X){`cVFTSpX|G7k!B^LF!0j5P>?mO4 z-8^`p2L3C-q#K%M_-05N?eS2@%rQ_IYUHuXbgv(<-1<`hyaDjbA$KvDfrmuxhNOpj zhou)IC!d_`)d5fGpYh$K)K-}b{Dbuv3e}EB?{4UKD-$3=d2+HIjT$P3jpuuSoXo|= z3Nk|G*%gz;`r%T*HVsCz50kD!s`sYdMCV3iRHi){(&Nti6G?X|7drrdky`Jd;n5@2Gbk%%Ba@4aSMSu*by@BGS$@s<>cejHnR zrXBGB!6ddRCqxM^f!k%FQI^PKhBil^g?S?V_kq6=lU!CTuVGN`2=nPiPf zK71dO@wY2=5?ff`9XsT#`na}4z5^f=xD7=2nbv_j%E?ZuAKGiLmX+qIR8cmx_IfF1V5gc~9~Kbc=BIfg0F7 zo=BW)T+LQrh8zF#0CMf$`erum17AScv(EPR(_>FL3FwRl5N83a1`JrwA2;eSW#GiK z)nU@$>iKTl+Y@_h$I3=STPkB_2>&opynQaS5$@?QOK%QR=w`83P!agt%wt6o_KON> z=O2Skd!_TSfqiN*+j5O)HwB-n_*(g@f%f_w#b7qpIs-ck0p6lIYjLeX!0)kwad}4^3b86P2@;r_A&DFIM@!B;$2><-XhtR#vL9jCd z?0sTs>Jr}5S;2yc@;=Ycq#GuN^lUz64Ox1^Y2QUu6_oWjKBm>JJo0~8o z30=_-#c1fEdW=oQ37CABWhho?60-4xM+IoShd3) z_Ew%3;Vg5$Zo$j>gpY0>zFMu69GuA~xgW>P|MxR{BIoG-w{}T2;bNj84fuFLJuq#9 zW$!os3hGN*jSdBqGsr_xrWgCXq=-nWtJf;qkt|npOXI7=z?yf7X@Qa$KQqqq=S1}1 z;XsYZtc@fx%E6efXV7B9Y)`_?1Q03UbYW`Xhh&}a;bE}-q|)%+1<|b_;b#~0_yJF! zJjMN`1I;Q&seMnZbxjbpZ;m++_nwaaQjyDk?oe_%hX8g}&##5jNkv~ZwaeEK7cQ!w zhmTJ79*P6OEiEEdHo7nI^*HE3zcAlq*ZDBv?~C)UqL*eam6f_9I=PuAFXCmpk5@Oh7S6|JpCR%)39y3H-1`cwPA)e!P<$?sy2Y& zj@K$vRO|OG2-j9jeh#|pIU~TCt5>SicM-hxd$(r1$kf?xe1|xl$Ce~beuv%hwfMcC zb)GOIQc-`Rhm^mA{5S24c+*OZYTmJ4ayt+ZvhAQ(nh9rQK3jv_2dLwvtIIa`_Lu&G zSNR5;XQ8%c>pn{n!Er;;c5wjt$pbl$J!tkaM>+e%%`m(itxE+8+cCtPW##F}uIeDm zc!jNLV-lNos)WB=@uU1Ao&E`NS=KwkweX0qcvKQUXFMKxFlEQ?NPH>QG>`S5%Ti~Y zu)sHcuY$Sep}u+`+BClWcVmP2%pd3kLlZ1q@$W@Zql4t_^zN4$>KnbS0Ht3z+)1~Y z9vT|wj2XqyK6BhF&%5o|9(Pk0knlE2zh5->IX3>`y2g=J&vHtVqQNWpOWCk0vk_~N z2v?-`rfO0ShF7lR%I#fAN1){gStSr;H03GT=I?qO8vGy3TY^0D5ti`9m(T$;Q22u9 zfqGR0j`#8V_@gbRj)jH_k%QTK$OlQEc-4&V1FuevGcNb{QPqxF;=F!FpX0NWOKH)G z-!dK_@w4cbCDjy^fi7i%#6jwHkDqJ6MluU%p_~abUZSlHjznsv@f4-BJ+t_b0nQN( zNUJgf`YmRK9q*oqHzD?N0w|XL)WTkZD2ZAc5>vjO>p zm`Lt*;bngTFVXQsvivqq&g(q@F58ch%ri&~`~E#C@)>e3?O&r}m7RQ$gKm~GJs2o) zISFh`M==?0%nwBA?NA_a^V;eyr|!-ADV>!ZwEN{qK&ND2XYUwtYOY86pNqr%rtV>J z3l>|V`xhRZPxOHWyj+=}C~iz?_A4Q*c}aIwJ!c8%o+=-}2!zt~1x?R3ABU6jZ7Fqh z(9~LuJ?l%OE#Fi7G2h~)=)T^NAQhJ334PJ1{DI$8Heg$wLX;1WmusLfpz7x;H}C57 z^_4#2O`5#~_2I^t;XPSxMNNbRx#&Ae>WC1FJzxDfG1O4jh zJeu{=1``!eZM$o=5Ax?1(ZmAr@SLm`Tvek%h7#yxjNK@(;gY0f*5vvsQ1~p1gOJc05#|Mf)Y`*I&~F z42`saTouNmE3%U2+;hCH?mcG z85&5IIBmT^nTg;Y7;z5>A526WhDTB=Ch!20&!B4d)Ad!eBBMrj@p!4O&e?%2OlKdB z<(Y|T{i|h-vx7C>OZStzhQ`KY&glM0TzxRud#BkJ^wy4^U_c(2IGU6c-o zS2n6m7Ur``0v0$DHTTih&Z3Roc1B$Ks!j`Id5Z;}93JC+q7cO!=}&>KogH2mj8>_v zRF7AwKeqeXA|Wg)8Xp$M=)NTq`)>PGRi@C8S5bF6o1!!v;nx@+tN0MA_r>kaAtUEP zE0XG<0fBCm&gYbVyse^6I+;7$Sp73XKVxku3zLAaNHxXiY#c7_AN6sF`{g+vdEpVo zd0bLbVX&pI{ZyXB>Dj6BYEMt8ZzDl24I)?>Y`fjosh8`^yuJX93=X^y3xkpC7#SME z7G=w@iK|rJBG=Z`Hv}9~xT@@t6MB_)!c&NA)AmBcg-3>pDwE;o55~mXX59i71DybO zFaE$mSWUcOYMVvBFWx=^?zT3{@3o)^$?0pfn=K)7Uz?5SOXN^F2D00iNjc}GMj<03 zrw3`O1h~cO8AerMZGIyCFIB`|j_f{0@*yvYepP6vIq*>xOl39Wy+Kw5!T-XQ`2DNc zkczDuLYDAv=WU4A@%?mRuy<2MW1I8r31qb3UH*m2{qsL7tVXTYtQdFiDtV@xgCE`j zLiH^9s(#A*Cl|0RlO8;pnGcdo!@kBjh{{ikcsWc;`yOmc`whH1KXqO04kBkoC(2j` zyHJ!-n@znucC7xi9=vfQR!?m@iA{!RH653_JI{m$_Ep%ifl_vLFKMHX{272rSj;M7 z?*=BZCdK)myOsWWKkIgkTu*v~IMB}3_nW=@?xdHw*6xtVZl;Jknv`1#!HfGtC`t#N zfe_Q)apt%XYejW52b-aPzl9Cc!$dumM%Kuv%~B!NPPpj#%I+so+R#H<>_|@O^`vsa z62nNR++Qx_JC!DNhV{|tOjQ$OjzvR+?<%Jc;u+esOJk$@xEHN5>i#lpe{mWV>&s$o z9>sg=GUa zE!nx9jALgHMd`j}aXVVR5qi^`US~|tHIONcUTMlNsOC;9*nt>H2zuqudFUtn#NYlG4mJ0~XvV?p)@;*hfqEtfhv%v=+fChK41EV@mp1IXA$+2X5LLy#@^*Q+gM z$QHNKyjshjdJmRPFD^XJDO-Jx-$xKJ;}<<61W^}Bs2M9!lD&%u$`YeuA_Bpl=OYPw zhbvz~>|LL_jDGQ!_cm%3ZmhMJ0ou-zkf``6wV$>>%I_`uvbLVTfBMw&qx2={HC>;d z5>HhG9OSUM-bi5P@^X1=OFHaIgkvxQo4A}~2_G}%OlV+zq|G)}DlDjQZ$+hVx!HqC zr7I@_o$it=o?bIsZDMJ8+5WTh_n6I$Fl<^cvg1ujnV}&K9qv*xg=(=zu+m%iqglz^ zOWZ5JzqMkMxqdZo{j6QVn-i6!EV@OJecWH)?qZ31A0+yh8p@~BZTvYLuhcayBWRq~ z>gws&;!5ZHh%GE)hb}DJQ{nmLgAPSm$Smsu@OLiE35;wHhR6Qe*dIM^C)$TU^}33|GY;BVO!!OjJ`- zQ=?-xoHE1c?NnhMfag{uc>prc3PE^O0Eqnyfx=K`UFtm?a&pqu+ zQiMNA_gBtdM$Mf5-UUJDd0f*J?`1TwP{s0!gzpKR^ui@3gg&z`o!{YWY2UFd$4cB< zo-58*y9Ag0txL=++=jtfzN}#kLG_8QjYW%LwA?1PZ7Ks|(oR2lQY;?b#=-)%bmU2D zOEh&X--_qc@9*Eq>8FH#*LCNjp0n8113TsSjWF%9_?tUn;evK5Q4QbB3VyZ}JM6nD zPO~0c9HL|@*T+QFP@@=MPP0PdXpuPGp!CFA8%(R4g2PBxE?t1vhR3qx8JATXN;I_7 z?+YYwtG~XE&5)+Lw6EctY7MyNrno#$td<*hVQTOmBhfaq1nw32wB53H(Sd9p{R(z;?J-~+wS4vZ7%YtCRg!( zZJmXuH6$pC8@L2S$*YckA}9=qL-Yj&P%-v60C-A4G1fYb*Gh{rr0X)MET~!TN8&P} zE_t{8f~F>BIIXKvyGMcY3k;$zr=V076YA%nu&_9T*A+< zIu)(@gmhyJ)OOxQk?`xwEN6>Pt6FVcirQE%oOB0ap@9NzpI6SasW+r>7;8FVGir~T zQ6$-*{08;2s>h*9b#!a)^!GqTk*?G*_4kbn-QjPkyYgvy717lL)w}Y&z@NKo_G_j~PNH8IAI3Ks6OL)~EV^k!!8P`Nz8sBIs+4zo^qoIHGxcrx znj>$~!&)>uC)vNcPA<3KJJY0};p^abPKf~u40jPEk7{Q5!d`J{d_x(>)u`o!0uX3C zG#-S{u1#s>v)SaRQ`hS!|88Si;ml&Zi)Ieca;ZVit5L}uk+T|+?|gVfHD1ekZ~Zt) zNgsOk455!VZ@+zzgBYfunifm%WTZ4r&q#ktChoQN$Cl?$gM|B&)p&ZWvG1n|Ov(f{ zD^J;R=f9GE*6Q#j2;7?XQ%c7mBjrCEm7FZG+Q=ptQLswfEFnk7CC(7IPTHCakgKtY ziR*Cuey_%VbeN9H0vCtVtf|bfEkkYplkEEG(mqc>hf2Dbdz+r})@;2Ai)Nb0G(Qpce5%Ip~^-5_A_8UjUkK>}N9alvf{g)+eiH3SK z#5Kr&>>r*iEER^psyZOJO%O76y! zf8JGE6uXb*4Bj;QMd$0g`-gvX&f~=qr2Jw6n#KA42{Zv+7l6dZaM@-U7 z!Ab*YEMZOLtE5i0i)_{5gRJG#O;*{)0t z>(C2Q$b#3QvZXAMdadNp;FtjC-C>er51+m95H9`izOCS#YS2em`}KV8u*jrWD2^?& zy4~`e7i!1)I%YoIv0;-vsd>H9XH`HFW~_2Y>JKWZcX?Ci--&C~@T&R1umt+z=N&$W zH|A^Q+Fdl{2A{pWe8!I0RhdJ~#>{c!Nd-4$Bn_Z)yhYJCm7%^zf9Vf_xPJCf&!x*U zXP$!IbNTa2SYGLh(RM$#Wf_(3SNz!* zwwPv8N#Upiih3sMy=bc?++&^O3l)aEqrZO!6;#K`u19CQQxT;{FhBN&E#W z!v*0A289uYrOr7+uc|RXO5}NjK!6DhN9SroN=0QzZ5$bwLNaNF!{5z59QdxzpJaO;~gO-!aUH7kzD6hkcG%|Q~1vK)D*K@_SHFAnVudba%(pxCVZlEuvaqRh)z+_CumYl z4yn_+mFmsmnO$7mpa)scR@@RyP{f+dBL3NwmTQkqfO%CM)|@5evX`j_mKxHpPGU)p z3#6hJKy^-n+03TGA#1dVzra*t^SAk-cSA`P0ogw7_dA1pjl zkTKb9KMo)G>tG=s#rI=&*0UeAu>!w-pL^#5f-Q3vs3hrpfFanM!`{tdfF6+DQI2I8 zV)LmVaK(%L9YrZBdrz)R_ncAWYe3e44HkaN9ZCs?4Yu875x&CaN4Fzckdpsyetxzq z>4$jskU~P)i{a|N_kPR=i6&NE&`Yel@znWX^ES^|5qF2G0W>8sOTI&^4XapEN;WL& zD%3%~jvR3qOZMGm1mfi8aNwVY0%cRnw?7HGuzw=heck}fV;}EsC(s(S=4kE~a1WpG z+%M0D~78{DGDJnq)Nc;^M;gAoYazTQtDbJ^8z&3>gTacmB#HsW%}%jrMks>2+$9_j`?Y+QMMQsI!6bG zL`5S6!uqD6HZV9>`jT|CZ+Ry!p)XUY=t4AvVFpx$YLoal1Q2{{3&tKLXPSgb!k}HH zx8bvfbn4HJzYI=Kt9TWJ-1NgLy&)(6%j{A0_WI=rgUwHAAfe|=vkW~8H#*soGR_>-3Q7K&I2`An0 z^_7p})^cJ%;1MOeCP)p{fk2QY>OrXmBbatpL!)N9RlpBwgCPgk>)nLr*`7lDfPhB9 zn(x>8s3||i3keR6ZEnWIz<3p01r34FAG59^i!Wv4UU+(WjrBmbTnFVCzEGmeUN!qc zToD6hYG{ERz%~^PMyW;V&3Fgv=4Rrfs%5P=368DTHK1=ozE~e^gkF$}zq_}>`6BJ? zfK2*5N#RDhkk|ShLYA|~0Joo~yksi#Elbd2eUB`OO}D?mUzcE+n=4p$QZq*)%-;eF z^lA-K?f9`R26zWG+Mnl?Kz?dzV^coxQItj{!I8JM$_04Q-FTNk^n}te;?b(PutZ_= zw}dl5K!S4{RAXRZ^jm~K<^LV+od3D&ZdinG>&eN<=*FZokW$ijTfFQI3WG2Mz|R#7 z)<=K>IdJSy!1H4Wz#_Q1MVLG2Nog|gzNfl0`IO94{#g8(=tQ@i4aG%ziAhk4hM8bH zW;=2FJpSeK;JF`=6AI^?c{cVA<*eIZ;;crjbBC`B>>PgUrxNy7*-~$n@Jxq%WbWz0 z$EA}Dh>?6>umR6a-2HR)w!!UuG9KtVk?ZI<@ERoAYd_ZV!uaumZqn5R+w&?O=-KYQ z9zwU$_&a)CJU2V5=7l2EV?Vz=*5A#O;b9^{p9aZk*WOXa#RVwO$cW?;nuh0&-r9BR zgG`!jSmQOvT2HlPQbba_KR-%X8<f zVN42ga(9p1l?3Z6j&_4!KnKfGeS~$xyYscH{~;j!kcFV(+%2Dy zo+hWuql3i`3oGU2yyh4HQ<1dXyY{$Dt_~)m>Q5>c=x^H<@tV7!Y+-9RyH4kKi#cf# zaGuH%aGEY;ldm=fHqJwjD;x^rLz9aaC3dMf?<4d{e+~w0GpWjFmG%V6A0l^vWXtl} z70yB1-?05#a|^t}PP|2`4~2}btoLpxM5VLze1LrPHaW4;yZ$Q2AS9G}krY$7%o_Ow z#&r6L64e(LtSiAd#5(-VLw4JSu|Ak;{N8P+DwhkjEd|~ODd2oeZ=;K&6&6VCQKgjY z)ZWAK{7EW!)hYg5y}E@Fm-93AY@x(G;1D%>-P^!VGnP6JsfF&Y&F_1#n3k0sK!ohO z-<)v=P?l1Xal4l3O_iST%Z6zd?LM~A{rOCC6cHU+U@qV0p}M955aqFl@}ZpsxVoM` z;^M8SM60^>Vjkdr<$R7Qa)JB3)nSm#0-{Rqk;`*PsedbJ?#zpNHJ7HhyDW7MV+q-k z0rdSmxZ<9o!XtmD9Y5dI@g?`XypTYVAZ`FXx0gBz8+P9uRVk{mgh}xhL*l(6i8P9G z$_?UA?c66#vm#1M+K6?{I|cnKIJvZpmx|6M*qV?Ara2rES0>8Ek~@6>JyX&TMK83A zne^K_pcHX?DgJPEQolY7^HO`#Ud?U71qee^g0hn@uhH_q16wafSr-AbFhh19Vq0C# z`8(1gbDe?Ha8gk!;`8L$hrrJH_w$|Q@iwooB!(5H;+@Ua>wXr@gPXN%0`fLzJg{-PETCtc|U*8vF+$rnArS$ z$#c~`?#S(d1fc}rfYhFllyHijdGsu1;qe#VgFJ1YVlUPnX z&A4)|J;t!wRS7dTU5*}ZJ@5OA9%9qO7a7V_lh=Gf zdT&#YfxHb>hbuEJC;e5ao0ZI-reRoPofsG@i$fR|WWxO7iwv_-lP4)5tC7^1KK#5Z zKqJ>}O-I!#$H~*;ewa!ChZ*~z5b;osQUb#T%}oFHU~W&ey<(~93r@HT$1%7Uk33i3vEkF(t0{+d1cB8y`dR%6CbRBGL6h+*9F$c4SLrDcnn32iobq^jRm9)AipJ8VBs_#^KKw}8J zDKF3YvP7W|lZ4%}c^uzo8puvsCiPhro_gxZuaL+cB1h)uH4xVx`@t1C(2apK-d^rh zE~ooxX=y*}H_yNGFCZqH?PF$QD*NHcIH+*jLbc16mMeJazE1R0!Ur_5mK_TIVHj_Q zx|&}lG>WqeU+BGOi)WyTv{YUZhPG>r^iQ;+8dt0PVv~VxNP1O4gS}agM6^+=|wzFSk5<-4)C>GSgwtsl^{1oV$u8CUSPPhMQL($4$=Dn!+m zVYUW-LPA(zVv4t)FDwq#bG)(Do^YLYPH=C(2}roCj8FVdF5PzP_qY#d!KrNg?H0gc z#$;(Qa88r`oaC{KF4faEJmr6xK!jw5#b1oTVCEa+CI6vUYN8?Yz*MZ-aX})B?yd+> zRnYsr)|bQCaQ7{_N{J5eOBopc7t)jqt>v4VkpZQDe|u?BfE$IyAVmKQDG*}G4{Om{Jj13xk1?^jw7w)LZv;KtGhY}VK}E*%k%7? ze~_ZULFxK>^Yo70js6r?W{4MAqw8P$lZkD@gncqXyB9y+=Sf@zz3!t*TCJeXvyUx5 zbecOO=-%bw;Lxzl6*M9p)=V%W^CVd>kImVhudFoTZ2J22xG>;~k+6Y2g7`VYZrA>V zbmjQ`{P)we)Hzv}EA`K!TjkF+=N_UMkN=e6%k_rgYE#2MHgLHR?bhqRH14I3<@&5jvH;=D4 zB#u2hu6>>7xz=Y{>$C03a+srK&)50t2=M0O-=ZD=sg+&eVsg1a7@GFhQJ8C_SH6LC z*QpMYiktDWz9{S`>4qZo6A+6z^RJVoRySD^@OH?W|h+^CyOxdQ6b1`f}8ffF(zd2h6$UBOH&a>%ia( z5K+uR_XIJ>NlM|X!5!(d2`1Oyh7I=)p~gUkEgc(F!Y=&vmu!<*q*p3f5YcGgiHf!y zW%Tf(z0-{bkc)j{z9a4vYI@xnZUXCi_fvX@!)JsLMNKVI8`2!!i}mo$g=>ETOWWZV zdBIZ~!AB3#Kb7h=MNAPtnLaCc?Ymq?<95bEfVnN3BFJdm8>hE`I87{*e{sabIwDRM0g?8g3j4AD!B5 zbuK41f@(|(e{<_IdrWxU?&@rZEQIm(YhOxXPkKV%9K9%ILNa0h`cxfd_ru{|!^Voq zbq22lHP#1{bzProqzF#0Ypw0*sHTWcCN2qHc&D zUb&%q($A?^KKY&x8jVL#xlV`1s%p|bAJ`Vx|8Ac5{rLVov1;D(s@$G_Q< z6_~M)Iz)@d2fV9pU{Z};^;oqEk~TJQCnDSY8N;X>JXw+=tI27ac1q^dEy5`1f{o|i zIH@S=6#9Cj<=7*4o<>o6X-&22J=jodxNFlHRfpNO7&nUQXhAg*q`KOi4n zHI-mCwN^uroGOm11OaLsS2!g4tEGT5g%sEkMjBSTY-?%eQad?SOL)cddry^a1xkr{ zJH1Nmu7d`!nhSrgt9hAiKGGNawj0!IoTU1Z3&~kX<)5(3x0*R48Y=~$s}!|3L1wgX zz`mR;gga>oE7ydF#Si%vat_UP{FvKNOsjO$hyrXBPDLWV_>JLL0$`Gti9f)`gm8)#KcUx zbRz5pwxoGk7?h->K;=ESQv2|*BX%BfLML)uyXKBrlRPcfUjZTTzFL@Ve&qYDQXjId z!D3a;5B)aRUNMc&h*)&quhRGPlIrE7t{bPtpbEyEi$Ox+?bUm{gXB!O>6uhhWWyH+ zrC847FCnK}3xi9iMGF$xyDz8W^Z&bWZE&KJ;ll`3x~#b-XPmcnUV2FQnqjs51q%ix z&a%10e2asF9i6pW>-*E0>)0;(&i;X(R#ra;&*U$sI@?6bRQ{pQzFhY{^tyW2k}A}9 z+B}d3GpoJ{Nu=YlJ|-{0sNQfSH%FQeMs-a#pe!}pL7a!i?J*mjhiHQR^oqE!sjf?d z_)dAThBW6Hcwu8Flk7tnpC)$ATP1j73#Fb)GSY0p-luEupst$AkVg>5{Yq{hwhf~O zzOPIorUKz1J2mQjs>5ApSyT@;9N%`A)5MCR@izlnz-`8rbb-MgE3YuYoyzp^rIxlX zPGZSP7806bJ?`&5#JI0rXbr8I$e#FNt^ag0oGt8)pb5JFiqHDS^cC?keSR1>mBRd- ziV~R8M#DzhNJJ`Se(*{AT-0hMM}zy8}b&%h;UR5b5nsKRpvcD;-% z#;gw`-=Y~+zmS}s9T?8YkWb_)rkd^76B1C416hh1v`O4w;#&D+G(I#&F^`8%w zE+O2x>t7}R4JR^Q#%Rmrm#>M0OA_5`oS&OYrE1*G_F}tLj4$kv4~2ONdyMcmxeL}F zhb)RC#9yQ)USL6bFLzg_pv}ggeLv$C)#8ch@#&*?x`#Nv^Sw8iC0#XPHm8`uw$P>f zyD9jtB~xQ(ZwxDS%WT1fcB^{&F*kyyCv$|0a{bLB9_&Q}eN)WV75TB$X0TU|gBS0` zQK$Ux7gw&j)$Zc8OHsM5Lgr29e>A6abxcH((HGwscLTGCocJ!P> zC%yX)J>Iv~MoXU}pF&QUCO4i))bhB3eInmimQPxz#V4~Qj|+phD+!}?)IVpG!uDni zO%-^BH?mK~R)QjZ%;r~jla9=VB`mk%@{z;ItI zyKf?WhevC8d>hNSj2~RV^P6{Se}O?oKl)w|x7pn1!{+_tS_?iaVS6BNOaqe$kIT?&qv(SKo&w)OnekJe$;Xcy^+?A{sXvk<_PQ_IeKrnHm70_#Wuj)}HAfwE zNZzD*LHIkPatCwu&1)i7+m|=GN5=Au;%COo>`b*9?K4@1G85pR=j(C4m-IyCvMGU? zr}4B6U4gKFM-X8KnDEK`8JNObgx?%FdR`;Fj^V~~5(ACg=A)XSpvAYy_jz1eY$`{7 zM=CWvM~RvC`f5eeZ*8jm&NGB@w|v;W({G-JwL3!qXalHBNh-xjbn5YE>b)|m=KIh{ z=7()Od%O{T#}#{sQl~6i50%FkD^({rW*ce6wA{|)PFRmWuQ}Ir<;RT8c0dY7&x- zvHD%9p-ym#tgD6@#05bBq`uZ|+Lp)APJvS)v6Q{LF z;R>Fd_`}mQS7@i~14{I7S3+$PaYUdOybf%I! zs_}h$r4bwUgALujE;He{5s3V}%_s9%VvNAZC|f?!S(c7`6|pp2;qr^!N8=r$!DX@T zdPbu6PmR^$%dUfy_#kd5k|BP`ezTVzQIMCSk2?b@c)5QQ8q ztZ88G)#C~Yqwx3b`#G5Q9mib8prUr2o1CZ9` zo2+}Rh(bFLE#k}9)x3{y)p-o7n)}e!0!Bz4FuMW=M@+-V`!;1-z zICA7j(}%eB7icg_?oJK<3|B-XIE*E2o?2|kMw!nWl12PE=U>C@QSUSDt7=xSCHI8i z*x~aVBBf*^+clAeTn^U!?i8SlV72&5f58WxX=S(C;->~sJlxt+2XdJEWi@$SU7)X9 zEPsW#v~L;=9N+-rR;}6x*Nq?ms2m=X3E65}bDi^iKdYuyA&3`lnD=7nc3(V`&^zK6 zD>BnJfZ1T zaHobLt>W22xJeKrI=_+U4T}KY@o@{lkc?B3O|)T=Bf~d6)hUf>a&#<6pk+Z;5KKUt9czB(UNlTtB<8|0V%jFNgL0)G z6;Toib?zL6Vp6WVZ5{S-8!0JbaIM~j?EMcN6fH!%#~iDX@`K#pwp8HkXIFl#99FTI znjKYy4KnNn*T6x}S{w57RB;kJn4xSBcq>26fYIyZV!s5&P$M!u1Sx%AZW2#|8XDdyvJUVP4{duFDIj=Fijg{t4;~7vU-CG^Xf6UMN#3-R&%bQF=3WZS)t+ z(zI-Y)O~&x@A&qBl5V6Qx;-p+S7rd?r`H#WIn|MzC?6|`>(^uiLm%f8;&$&v3Pe&SNOFO6{2P#?jv-2Y^Cde-L_f$bxj$%O_F^2 zE*>cqu^h;&hv(1vxFNMp$LOG|ZyV8-&75&^l|+rqCA8aKlzphEa~A?og^-y;<@XaH)-rGasS%i~ zD;-#jP(w(7Nz>~x%rDm(62rW_G8NR+dJAhX*;&c_%#AqPrl2|$^dx>m>+mfC&-L2z zn|o*q{jHbH7@)TGcx->c=#F$SXS|cJVFYH;Mo-f&UjDczvOwQ-do3%p$<`+|Ca6*a z8sz{ZEZ#yj-%}zcT`m{jx&-c8(;WWP>IYuo5v3O!z@T1_x9F8r^-fG~;Wq2-Oiy|6 zJ=w&$=^jM?`(q(bh3X&7jKZb!3Y3XXJzrFVNf-taGAs6i!+s-vK-ZjCp8<;QbM>Tp zAzT62$|#~$eVS+y4(UGF!Lh%E;<#EavbUI)FSc^oVld!}=Cq3Au|MRHG5p~aO> z{Ajj3>owoOymQ=w*Ey~FE|5U>1zIIoM0;&cp2JkO4nBG+?)yh+dma+$T)9p4NY94c z^-8a{9bvk#o%(I+Z?ixLNZulHM;{&(PFkh|xwXXa!je*IB{*UpfP%^=+|SBbizw8H z&|dsJ6#j~awzX8Cyg?O)jJ_!7@9&1WzLjY(=S9){?hNUhO9nJL`dc2aJ+A?eezxut zO@(JQsW!I&_|Whkf@*+Dfwu%)LjOwhX_0S#fDBJ1%0xF`p}}dCh|S-2@h%@> z>MOg@BCq|wc9k|m<|tz!#kIwg)9rc2IRhJ}670==p0j-|x|padfqkrm!X9)Sl9%_E zfl5$2p%4zAEAH+woTWR|sc^FRjG)r2Fv{qUACsMWPX4U`_&V_g;wT}g`c{gP3)kj? ze}D6}wuTiE7N`@BG!4SfZf};Bh0-A zRpk!DouT&@pRd}d$6>0nNJA-hV>Yxrmtl6w@IbqXlVt7OHM>t;)YN=F8|Gp+$!+Dv zeeXiZCSyH4noNxSW{eWr+Kvp-qpYW%Pzl$`rnXPrAZ%*sXXn&@18pU3{dlp)rpcDz z>kl{XFthbz&E@LT){esQdSkwP^Eu#$z%X(x6Qo@87nijz9~?h@$`+lIii4W)lPyd zUfWbzk?S;u=L%(l8cI-Dg&DMFk57jSj)xHps@NZCK{NQ#(faMp zFsAgt;s>->6hN62pn8=aCu|@&?rf}0)Mu3K&vX91F=TR@Q2J9}}7H0$`Cq~=l zr=x-{KzcQrx_om_a=Uv()!}&D^$UC>`8e>rSZA65e3qLAi!8D5!Yb22EtVEq^k{N5 z38K=VL}6%Ztl1dpejz8M#IiT@kdtf+s$u11RTP-y6=v^9iyk<~l6W1?_Y->uPK9XM zS!bQ?P>-0~_WH5nmu+^|aEWK^&FgtHnr$)4w~nWut^xuDfY9!N+3h0aU=#Me;COgsMcp)PKOFR9nDJ&90qJsa`S}=UFlx zxie{_2&9$~1s&aooH=u~7o1@##X;yX%O-u7h@$kCdv4*t5jDCeewr|lFPf>q*(|4K zSBn5OhVjlWXSTgE4!dfuf;7zD1Zz)nw5=@)$jIfU%T01X_1e=2U*c5+eMTS_!DSxH z4f?Gt1G-ij-sa`BX zA9csLURCs6Gau(&*4+G-B?1J4&4jLq<%m;Q9Y^`B_t#A0rxxO%ePYbL^K4+Z&p|{a zP3_6q)~Uao}SNOcv7HId&{Nt_JHnudn$?-_~2b{{f~&V{Xl znX1{r{=~^t3y#Cpmd3X18OoY)z4N&4`at@N%R!R#oy`KmimjIUij&2GCnrtJl3Sw} z*PNKAi~f9LE!VrB5rZ+6j_^+c?zS|*-~7RqFrvu|4Y)bDLnwrU%&$fRd)12+fiOB| z^L6wp{ji6dbL@8Bk`SpXn8yhTHA>u5x0awF)YhuG*IAz?+;f1w(=Yl z2rRFu}hLbG2-*8e9^2b^_8vN?$%Z@q@}}%JF%^!yn@+SP6mr6r%aS_Njr>T<$lhp=7FIXMeTcLvAmE6ewpEbbnW> zEVMX~A0IWp3cja!T_8~+v!?tjQw--Bc)_!)XCbX`-28z#nEG1Gqz_O)Wsl=F?TCA> z-REIP1xgY~@8JZf;s6D{CdOi^Z-V5yV zaomz(@`Q*Uev`l0(?ib&c4W8mWPl&bvTMSEZ+q^dFZvCH7iEzX?7Blsb!o&!;#FXX>?=V+7A?-YE< zek`9NpwGmwd1e{EF2p%UwU&k9B~{yfsa#-bwZmud)bX#N$isWc#ZzsREn&RGD(aPd z3GBk<=#$?X$E|gLB42L5Vg1ltYF6N7;u-=hAyN$Ex0kDR02`Zu{uL-&nI6$nH>xtz zGU66KnRw306yvRrX~Wv~5&f)#_z#va$JsOP310RRYvcAm%G|fC-Gs{Y1}S4rilwJ6!s49|{ykkMk)VsZ=Adpl&#iPf>Vg;Rk*BW9$>RpdE%90Te=16lVGQ;sv`Xz|s}iMTQL z;FxHi-}8spLlJjk7_PCdKjdNRHrngPV=ZO^MWJ#sa*umToVD(BQP%g$<^CBABQ&)C(Y>JV4S4{wF^9@Z?sniv`Rnj=R^@3ROllE_J(*X3iWyP%i3 zFCOWGe>!_ZE1e{sDA|hePlQ)n&ikYQhBgHXOp ze|IESY1wmPBcQcCmdl>^F=~)NqD22qD(16uQP+*#<>!%82P-S&0!$WS&FP0pu)fbT z(T~T=TBa~QKS>xCqWxX@VwcK(w2LH&VQGYmv!e3(tk?ykS!$?J%xT|{#c-nYNr9o4 zYm#{ncezx&7-eDot&qG8dMKTLDw7f*vx-fX4uO6NN)#H86!W0K8qtA(ScmYg99_V` zD(HN*t>55cQ2|G4lu0)-(1H*OWKb_c35?~^?6ebg_oRiPrXqreBvc#92 z>o&b_NzpTclm~FWJe#l>7S-bCwpI(ukt5CN54lB2JHcm7x7M9iiDe;)h8;ce8mV$# z{HS%lX>O+5>M|TZQ&j+TOvwz6JJFnY;529WUq)<{Xy+(W98F+yWiTtjFZxyB!kB1r zY`uh$rTkHW` zd5~N)2BuHs9}Y~9Qc53bRTC}YMD&2E(|2Tbt33MdPx_blK#mmZ&Zd+8a}NoI9Z|h1 z%BBFt!9Q6b6AMILl~xf{#xpr0dTnql^LuauJjE|#77QLqu()px^{5~f7%&+~qTua_ zRntBZek6!g0YNZhS&{1lx=GzSa6X1~KtFh+&?c3m@$?UES)&GDiN2KTOS~||N7_{; zs~cO?rsw3mDKspXae=m(G|hmNMb)QRbk*AH=&192EgY%8+W^$W-Px9&loDT3Rw|Ht## zK1;qV*Yl_knG4GxeRot{cv;jWDBQLCW9MktxR;khFElK@PS^eRaQO*|U+^Z`jLia( zXK0OTvggHd$jL+p;`<=FzpIXBH)2CTU7ks`W-5H^uMgV-o?c4fGgq?l(=F9fEXO8w z_fUtSC&C-hu1$!S@VH^ugm3UvhlhWIE1Yd~SMFyW>&_){ox(U`!;@p zCkRoZCIy5RkFWmFuEvm~QRy;bb6qh>@a5*z@Fo0L;1=-~m8r*-j#>#>&RIxT*M=KW z-o^C#PCIwvGvz?#dkm*_uF>wcctWrfv!jI1w_`?-65kUnGoaRq-x#TI6#&wmhnsJB z@}#K~b_1BnJ=pbM0;w6pd`Zk`6Yb)tr5K$@G1{S>vQO z6foej;pY-obtq(e_;B8Px*X31q(=BoBf$TNq71sY9-4(k41 z^+ZOSn?Rt*X5AUi7gjG^kUR6gELlmDt^(8yrVm-9tRHFvfxni(B=v$Lti0zVE*^L5 zSUu&b%N2*EitloG2|6C7YNC?|11P7KtAB)5-Po-oRt?(`b3>D1eg!Iiy*T->$Q709(mk+Q{MvR=10$I=(!K=s#9#Y#nXGxb}*eMZTjWx zZ+7|9k!Ull*7_-wiQVJ8Q~s4By5cZcdmc|K3h{owF@QH74tGy?m8FOmE*K< zkVOHM`;+xlJzTchPd|L59YyeQU%%d3#$?>Dyar-Sly8;9C{q&Qq#|15+DAE4{nmXw zsF4c38R14r{7$&+5=@fnfUS1BGWODV6i>r??n?0+4(0r4Pmkz^Zmo;@U2#4>$aD_C z1oVYm4y>HoJ<5Dj$JulMq$NhK`wX|hB~tDVHk#W@>8q7r_B4#8OBO~Otrxx#a$9wZ z?M!4g{Dp!VZ;fa*ixIwsNPl96Y893HISoHPbvfJ`B&U!6w45>F08EwzN{vqe$VNkg zKo`h$*L#x1C5W$~N2JEmtd6$N^loi@I+c8oJZlZ61H|%xi;&wDmZ}VxRofd)`eH9m zKS0y^)2UO@J=q&4N+IF}hXR1vGZ9*cMC^%WO5i>kTjOWy8jjogdr$volM|I(U#v}< zT7KnbqCOX>N%%MF<;hF7_4vqBj=22|&C3cYvR*2E8!MTAC@aG6x}u~B8KZYJir?lU z8kvUPb{h%_UFiU_k5NJ*!C;b*%m%C0>v+!+J?RaywytQOGq88*t2>V?yzH5>>EnI7 zeF2A*TNXaUX{(3)%6MRuU}Gq8Yf{Pd%5<=3?eE`rfN}Adeo1W}$W02Qxl9!lr=uGm zOzd8k=%sq2C^06ibv;KbIZ>Mm2kC~A)k;yFRoSK7jr*Y0sOov6d2)1w<2+x6QbIJ2 zFEPp9B&Qz$_7jg?iXeZ6T1U2C(A>}*Vo(F>5x2{uyJ;KAOwL0jaG_IM(0_dfc7Is_ z6Hz{?69fdYiXNyLOtv;AfH8i%8?&q1Az#VL)1cuDCx>3NKD# zn6{3neV~N8bliE$ter!AebD<@z|*17QWXq$ zd#(EOzN$6qNSl!sYRnwf@xNM~QULWy!L6Dya(H!JGK5%Us+2NfSedHH`Y2U3A52@y zhXAA{TxX)n3~rquD|@5e%N8)V zj!v^)xFY;`cMeFum4QM#I)95w+%TgYPp73PT?@1>+jM;1@*LDSo$Byw0C!2aW5PEd zZd{KV8(pT4WTh6s;G6GQ_}!kR);{WpV`v+1Pg*WUXEH z`}OPPc;x#>zcvg@ey7fzBo(^5l3XKzHL+L@UTm9wC2{FEkU&SH&Y-VyS44cG>#Cxs zt74LUB1=);u=e7zy$8OAce@i)VH~Z!kHrezz~**35!uVD(MEj7T~f+DQzW3LoJt~^ z-~H~h1b{dG*6gWos>&nnYU^!oAM=~m+bAyst!J0cHO1lEiqRw@aTXO?Ia)jWVVJINdU}6Eq;_qjUknL+XMMmmN0>1do(*1Kf>opNt%k?Im3VjXblhs zkSA?Fl+`)y7zl+4?D^Nv-1HIO+OHIFT&I11leoF$cTbe((K5XyP+T2|c`Ak_aZ z7Rs3&f4!sQv zYM7_{D+BruasueNF{WoK?p|VVRmO^8izx4^#b7j?=rL{^|IOY>ufU z3u{V>#9At;jZ*%aQApOp0E&XidDwyJPG;^$3u!_*2Wi!G7TZ!eRwLs_S7_T8T~cq* z@E0I>v=kIo^4U1c!z|_I*tl}9|1Mnu>yIyPbj$Js|8O0<91Y!V-rbE&{`NK7HYGnI zh?iZHvkpeiLW+<_iC|$z3l)U&SODhAiSX|#T?SDCm&)T*KjPX*VtK91?QI`jVnw!b z&I=LAw@^{SK1CNYP3!>y!q}LLqiSdHHul5#{5cgB`T=O&z`i<)erF&<1!)IeB=v%a z{8BvhbzQ^6s)(=RudJQ=yLbvm*FI4Bw*{Bdh~Fs@YV5SM3Q2t_^fE`SR$^_k_;-2e z-{}fsOY4Ct%+x3QPfkzCR=eh(l4IMy>bsL?{l{zLFC9O_OI{?$Wm#DCy4ZR7^0_szQ!nkRHoXem{>g+| zbnX0v!|Wh>`KrGep)22!@4W&xxNtx*6f5*YLcZnh1+5@*^Vspr$$7pn325IbY2>SH z4;L_3Y4QmVeSCy272~>ekK#%Y4?6=_W)%4MNV%-P&#~!aEIRy5?6!v*<7>7U)u>Yqp1Ra|LO$uo>9k9{ahe5jfKOO@YU6Ue<%NV&GQ9}G^%aY z$3IlMgB2oTFC3>6UkGllYe8VD&& z+i6$lwTTV-o7&boX8B=-zg=$pD%`RcB`Aaa(D9cYkJ1yEzwUaGM2tk)0eyEaQJgtY7=HU)$`g7@FsO`mPUe>9#oW7%=?2BV-%i zIrYHbsuHgcOpLOtGHQCMrG=Oa9^bV(FTT*FauPtym?L(^!U`nhz3v z%&-DH>ez@1!(f3APx0#DpwjfI@kb0w*XFNSR2uzgwP%GltIlyJ9=HxumY8g`gq;W7 zuhK-ElGUu)fq_;QmVe%tYh>~q{J{4h`QU|5cE)y!s&9;^`r+rK7;=4NK~jnJhc0(~H~NM(zW};Dr-1X+S$y z%~_EM=ftFSz5tC+@L~X)At|lTo*_fh93M;ETZV3B?|E;p7(ef+oo452S~uy@k?#;C zK`7=vl%A8M;iK6wnXm5$GE}t%{optD`oWH!g+E&DLtG4OOwFL67161}%z42nzHry-p_20$c2LUH9d#3OakIiMqK39E; z@Y;A$D5ZXMb0qTC9rxVvimPv$*?FM(Jnm@drPuGCdk>C=-mIo>=20qkPE?QwK${X5 z-TtgNUq5p#njKuXogqoM94h3EIuO34Nu;um(4Ai4Oq5)y?oIfCQr!dVlz!|~62n20 zLikY6VP=|SjAKfY5v1t(g9Z;pfF+K&W2r_S>8Vv7dRM=mzK=UU7`HR(!~Hp!V&g-m z6GPRq(d{~6qyT_1c;DXvgT@o_g`{4!IgT;+4MGV_L$u~H7cwcJ&5Q@YGll^{fMO&r zxd2nL?L4C&FC2DG9%$8mj^V~w&q=Sdw6tUZY7K(gE$6u*!Vf?WqMTFt+5qOVZC-1v zDI(%6IP`U<=21oPvW?sMb!~ppz6b^RzO4&f1HQaKynkS`SGQMQ6xh~~KlZGbW4T#U zSZ8g?8I&F$Tj@Rf;phB9FWs$!FJ`)i!Mpf$q8J`^|3e?dIt!H4jItJX#4u11VZ367 z;LaA^S}c3AUbr8L@R(BtMl2V!zwm?0QiaYrXgLe>>*xX%vadh+pZ9c+LLVWfyXUEj zG=HmyV(=xish0;FTehDEd6R>*0I%>l|E&Ep&IJ?PyQh*Kn>o{0Lc1U~1K2U7eqz`8 ztgKy9*YvL%HV^b%a5&GWt(4jCRyS}P&wP7S>Ke;tWe&rwwLstBS#~oizR+AxJ9uBA zZ%n%jGMTz7jeFOk|JLaD%t>pf_t99R8W$#$8f!pB%<{62??B-|LjL7!#)EF1Yf0vW z{vsB}ZEt;4GBL($80qm|^RE-@En3&V*{~cIuB>d39u=i2m~U`na*ogs-`0h=V_p3V z^#~8A;J262`qaVmW`LLWvY2c-hA{`AYgOB4MIsBUTtu*}J^SwI9Rk4B03W~iih-Xv zBd+i{QG_Sl7}vj+Gf4aurGfELWYwseZb?w24>|I{#DwV@KDhx8b{OkhhvSjrI0p?*4L0B4IIUm5rT%ci3s$S@_>@!tR|DG4>~1-*?Ie4hQ(1pXhckxrpy zk^2@@ZFzt1K_+;uRQxH0++Xw^dJDMMgBAFFNP_@otBfwO}`{4SQ-6Wy4MQ-nxbF5Lc?r{%-UpV3qnFW3uWA4k9mtHurjH6`Y&K z{5D$(lxRFMoX;U5Kg1>Dct0rI1bH`_t4KqW26yc2I&j+LfCMw?6t^%XDO-G)!d0@4K-H3phH=kU1qjXsoZZ|k z4}l2S#?==KPbxQxagBBXzEa72$lwdvfg$%#U3Y8)Hk?1|{4ZdL1h29;^|4IChYv@9 ztG5jjy8Ju=8$1WFvKZx^RO@8)Lv?<%zb1FEvx~L$yGQFG%{_DkzY_3l$OpS+T6J~6 z%^JP0Djcp{Sap5@m`Ef=bry>yi~x_Ec%Ry^Yba95W+bu-6d=zrxZ{y^VCXrTbLyo6 z)B|+gTl3Df*y9}bC)^kkn}d`u@8rtNLvTe-?DU`2>(05WQ4OsELLf0nZFiTOK&1&rFwVw?xM_LabYgX8Ywm@U(pbxhT zBNv_j1oYUi>Yun%P8KE543yFkOc)Z!gByj)|5qk@@5M%QF#7oNYDbKtd#Fp3fYT$g zzjht8cw3R|`qtB?CdS6^*#-wy91{i9W#V&XJ#hSM>ROnZZu1`B%QFe-RiW-Z_zkMh ze9z!VRAb4rvC4kO*+%c|7Zio`6jK}1HU@TpPCBSHxiFxlwENJh+Im9C^BI4;jZ*3N zW4RAkDegt!Oy>2E6aXLS-oh_V2e%8-WqAMQ8<2I%Z8PX%cDF>axSv}iySFPOb(Ktb zGN!%5k26b=`7%kCU25#70-WpU?}h9|atmQK`G%K`jsW*qsD^GnHvi{PH()9;7HIpi z7w!Jet6a{wc4m^^MnR^^fz(*A31V%=dm|09$a}6%ylm2e6_Zic{HQWr`B%t*=@nig z&{#z~ZDQ_SevXxOHlU!}n~m@1nW|zt^#DQKq^p<%E`GlkhOguUWjm2Gs3aJvM<}7$ z&I5)pzn zF!6kDh<3H{C5&AnrHdxm{?-TiF}F@|5&Hf)IwP_cwzD~uvbOb^D#u2WkdKvj60|%v zgO?3ku0QQqho!Gp7%Vf9?A87c*Bsw9;0fbrWjEdWVBZuf_{|yIZl9+kx66aIicKtU zX$*TgxakEh(}IR@n)vvhg#B%RnMe7&ows;izROk)H{H^S2lEni8~0=h2ak7~q@%{Y zL$o%E?}p$UE35s>57GZdT)Ru2rCfQk6g0g&Ss&1kaI)5Bo9D0hxcsWYmYOek4pt8xj*LPooR*Nk`Sioh<0=<$2+}bJC%C zd5$Y8ENnH*8WNQLCKG6?1A93pb@;-Z{2S3c{yGs%WRAbEUib?c+zOzTjq<^`PhHj zK0^J^{KwY@TqL!a04oRV;aNH(_&;A9gPqX(v!J=E9CIvbZgpQRWCZDdc76S#@Hf1#BMAGoTKP_9^h!aoh66yt5kFeE()&U@>43#rePL&3JTnDr^C-QlCP_+` zbqntJkbKZH^k%c}R4%k8BNZ%r12+8t4m*|r<})-hK1_Gqu~2aYLlo$!I0B;xmLHsa zPGIyZ+S(1?PQ$-ww4+g@U9=;j8gJ59w%j<^4SbF}f5hU+gk3=H7%IxcteQ<)$A{yt zyLeu>obT)A)kd;Rg0=kXx1Ij4BHx=*p*a}$M2bpWQF2++14Q&_hF{cnji{kfGp%=R zyN6VHZBe+I*f08%$gP(u+&XcVikE%-(B1N|u~f(~Hj!O-YAC&)c}J_4Gp zS=`h6e_YYB71kwYYuK{s`X)!hh3Oe&(c14g;g=R^o?aX7+Vb*yFN#0U)jz6xnv0?1 zQK-h=4S1WLiS%#R4P$I+nL$WUMS?@c*Qa>n`Q3cd{Cy(tLC&4)y3w^C;sEKDnueE} zm0wMwb$%xfmBUHpQ-i`zCJXh+nIBGUaZyNR|23j zLUw zbIGeaQB_mmX61%v)x1p1+ZPiYYP0j^LWAVV=E_7Rm`&tA1yU=$0gqR@^$zNqV$9mr za?vmNkzc=-7XC3+G(X>XTMU&Ig(QCJIoVL&Bn4aBtpy?q2MToUB}JYd$bd)*2I!rCw=Wvnr}(Z; zT_b%Mwq};d18cPqQZ4}gSQMeArska4h?Z`PHbp&gpHeR)9Z$B0Ccy3p6BT_GkA)T? z`tQd3)t@hInKowR03&FhK*3zQ4?{@Z?J<~Rnghz>T5qZEC!{Q@q4SoUq!Mi|c z1&U#VKeMwvZ^J5!Q7IGmqX6Y0NAQ9PoDb9wJ#YNqmhC&K)EBqQ?uNibYj&;In%{bL zT>Ji=c~q0*|9)o1NLx7E2uuY<3N`v)P*Ld@9D&uD4fTi6+WG#4BCdea5dQRJL^u9h z_n+zM-iDuFMIvL^A(19Sa&HJ#ru(`4f<9hgKCTH~Vy*eB8Nf z%=WImvr}u|!y^PevdAj26HxPBCU97Au7)VvSxO7F8rig$rI z5F@k^g+u{x?^sVNfe5F4fxLdyUMGMQmCEOFAyGb<;s=60!2f!0{X6^kT}98dka$Xf!GdL*MBL(-T}V%AC7kM%WhxCkr-{qinU4s=^{w9 z4Z!$!{~h_NUX!AtCd;KFf`v*KUM#kYe*o67n>5~m#7P=1E-tYC{^opH)arLUcHrYC zL02+spOQ4u2ZVt9xU($WZA%fTkE@3P!XUFb$Vi`Qc~- zAqUsjAgE8AD)G&mDD{n#-qFsB#X8xR1tAc0wUJ;kDd6m6+*=EvD2z)Zs(_N+=U|p4 zToG_sC+JL>6`uASL?K?$1*o>^M>DS0pHXFdo3h6+uKtyI7Rz)HaDUG0-GZcoazK1M zXVT6uwpmHt+tmSe2r_sZ5qNT3Fe6mP;4wG#PeI=SY%vfnb+YF#>xj@ z$z71}RFu;K@=G-1`NGjq2}p8{H}iD9*Ie`U->UNU_)opRkX(u+NPp|`e?wV33osiP zj4V2;?~pCL=}h|hUwuP2hkU6*+<{Px+jO5ZL0}oLAm%NWnUERtkZ;w^Fb65j1&60p z(~rKNq_m%hZE0B*)BRck{yZGy`#Iia8IX6C6BS+iPzaE+@7gc77=cLR6@?m9_}t(V0{``6o*xa1#u{ zNGA~lj^~2VHL$YfL;9n4r{b0rQ@7u*4*pwJ4U=};g}WQ=Nx&IYgvUl*Vy?zi){Wj) zMF5}|^Zz65y~CRPwr)`rg&#^$v4K>jNbewB1f`4g4kEn-gwU&sVyD;8q=in97CI;@ zy#@k=E}am159O?&dw=_T&bjxy_c`xhBq49!w$_?+%rVE%p4tZz%m@hwQuPm?M$$R1 zTzxoWUFytsUf_Cb8JrcohyDUP&P0MaGEJI${*^Wu3a{?YAV-iw{97im<wA{E*I@4WvNH6FQ=L>CTMcVehqO=Alvt zb#Qk8|AV5cEX;eDeRio8*c^_wM)Wa`6{yk70n^My-M7bWIzJ*$dK2ledf*ODKY81f z+;R`;tfcJc`GjCg7#J({iCi#~Z@AGpA|%V@?WgQXr94DSUu5Kt10QpMf?DAVOXb`4 zdChX4fNvtW0dp0iGyOT&6F7*4Xr_UEU}rZv2)qr%uoP*{T*I+0GrPeu48_cXhe9rG z-8BzOG9&b2-T0@xHi6w;l67A6m1Ttxd6S~NkKt85nJ@V-0%xnU46+2z&c4HxsV9Ph zwW+lYm#46|ON!sS#$U1f9!ElV$#vGz>ab}1Q`xIn(AQP*M8^92J#Auz_I_t8AQ%GF zHi)Obgr&}SWgXwRpSc-7NNsqYhtV&rvxehdB(xD2wzC6u1Yi$YnHP-+yL4Q_zV$J+ zc%)^w77}qDQJX?FZOer~74Y4!kr_K8-`R+@RwpY+)w8GD?|w-V@v&EfQyF2EoJN08g)ccN*`VUbf3oPXB|lH0jcw}qIK|2cN@mVd)8p~m~b zh*PSMPYL+A&hphbO-tB|2$QuDW>fizqF`{?If9$j6 z?8i$alR<+H!>F|C_3PI`@<7;>?((-0WPn-jN`0Z&6i92kbo%yh#?x^Y{w3#SW?UbF zAS(j?hx_lYGeX2kq||RDjNsdK9E3y>l5P<9{kf&?lFtV0l?5|VBrqKWg)6`erEo@JNtF3}~i(_d-y zdP!QrVK7OGskdzS*KwacJ?=f{Lzzr{yDY+W0}(&Pf4KmkzCrZD<-)11=$}`Y98dT6 z=f(Ykzqjc0G-y?MdO=|t=x>UX= z#~FVQwO-_OZ);-?Lto&dgocHc)2lH#0z5ze6^iW@og*dH22EeBK#>|W{y(&_v1t=J z-S=AI7tk^RG;53j2u<*9^aU;q2a)FpAHHKfecUE%&~V*5xt~l3f$N`}UeEgHS52Xy z59ba#X>|fh63Qm^OvnIU3 zgg$*l{(SbrAoxDv^!{wm{&;&Gj6_5vV0-<_9;bp2IbpvN4<0Qk9HA_SI zfao3XpI<(Gamn$&UzgM8`1BK^|M`6X{NMl2i}XK#>AxnI`)99Ty{bTfR{XTZK{=S> z$_+j~mknSj2f6}UJ37=764wHbmzj5g%C{S1L?f7N|HB}akVW73KZOsNB5vOE<_29y zstJOhV`5k#m^Jomobeh7#AL}xBryGudckX6^*B+CHysd`v{ICZ2>z_ww4ULO0V8v) z(VV&}vouWrcTok^m2}Y2*aHwbu2YPFN#;I5Sp9`Uy4vD1-~>0uk9NAP6w|+PxI5PxHBX>3NCZuN5M+1#hm0(ur>nU+&a5FYze9cRRd8_D(mF`S zKxu{4y2$nAON`1?%`QWmG~uHCU_5>PuDl0q{@x$!4R5Om5-V<2-M71u5Y)ZVO@fIC zVBm+(H;4gPFyk7R*u$zkf*2lvYyM0q{&D>mBu?LWXL`D``_F|4HdDPoe0~8?)&==h z-1qnQQQ!~=9a~$!e>&Pba@m->yH!u=jxkVS-%%2fH{ z;N0E#4A~Z{h8w$z3$uk|vW$$3hM_J?UpT7wj`jiuroWF+(;wFRb5k;k(_~DyBc=y8 zeSxmC{GN!vT32#|({o0T_2a<91>W|J&y2Su<__!ODe{4j>ln=H{izolRW+sO;d|(j zOT!+`-V6xUx)5&|a(0glRY91lnZ7ojYU97ZNvesH+=+T>58QfmvfK=36Zr1g4VUf~ zZ)LYwyytZ$k+JE&Pg6L2Rq9I{p#sTK#%pYR3qU_y<&Gl?kEHZS>89AGTcg;sfT<&s zxw$MMNhXvTI$J&$aygOl4J7U&mrbj!x_CD|=ff z5MhGki;pNbLxgW;I?Jm2YpQLrDtJJFlKwP&UT#;|1<2ka-_v^9+C{cE|3Hcy*9hfPF9TZOZ zGc-2WQ>48(c+9_Qb^+v6ytKDge3}D*nqYsyzZBdl_iMRLD_*i`B=d#o)*b=~FGuTV zpT(myVii(U)xuh(b*42GkuUnAzxiGV@_3-bE9+p};R$dxd4SaTX-ZRIAR5&NSI_F` zJ-9xY{Gq8w6zALSGmqO%>ov;35L2U|-@pg%L#;m3euMbG99 zgdM?PR#Jbbr}$kk9QZD>PCXgOo6Q04je^}M6qmax-3L#6IR&dmtj?oQ1`(U;bE>xn z`N(mMo4~n(&}Dq3tK1Rw`onde`=AKZ1B7bqrc(1+=|)OiFK20_h7eo{7^J)(0J|1- zNiX`$XE5uLatJ4OYB27unbO(sN2H-zNQMSgZ&A=^cSQ?WUG=4Y9P9yZfH?s*aAAMD z38mmW9$TBhHAWLqr|)X4&}}WOYr{m3G{oxze7(}^Va5vg_Lc^0Zvyp`*8~lrBfLi% z3oozx=ER6wN|$-18bQklz&dZm=jm8B>)ls|w9X2erG$|pTt?&NuBhep7js+wa(5%Y z2=LW8gemf~C}dn6G;Q7os?C&jl(!|{(j8OM%NU00wqALYYYzN3l}lHE=Vb4@2<=&^ z0wV!c660rdpH8amC_sZ}^JIC;wL%l0mh>x1nBn2bH*e01I6cuu?=D73@|ctvy=rWW zY3n(xqJ9u;J?6OQXzukX-$>GZiEoOHwBxY!=X}&rz%n_ZyYB4j!;9>1y9i6W(x=lU zj^$az`A9$s%P+NQ%xzsjrs~@Oj(8&33wM@)-(Aj{QUO6;$ z$0Bxize(|Nnr=R_XYXrp?!(Q4HX-xzjK5__>!RyoZ=BI3iNZ~$NSl<`Ih>T6pw}#g zP$dPeg6T=(C7j(ie|19lMJOqEz`$~*90e5&Hvsu3qEBcfXsel}ZhHemXAY0N=(0yp zCsUK8XxlP;i*5j&)W+8y>G}Q5*;vl|l)AcH`jA*lEkL3>fPS+o`T8|>|6pF}C$q_X z5pfnriSxFd)qFVd1mL-Azu92Ts@C#40WMh_&0YP>P z;%a}6)t+b{9sN|{Y;?tFf4>kRDH3i)OzIxhs4Z=CNGlRPKovv?Tdv&NVg|F`8vS%Q z!MzB~bK<#I`+r&}0V(86Biur-_RNZN?W!E**gYo;Oj}EF=Tq^5sY~i&#m`LW^|5hz z7IT~U<&mJ*Ny*79Exi=JH}!A2U7H5vd!@$UGo^AAG=&s$Yr@*b8yFQ*WDTV6rojHcC{FD2H&KHAD=GFZD(b1^HM9Bxx&2Aczabf1+xlYM~z8?F@8-ky7 zIJ8;B9zm=+3YOaz1=oE0g*s*y_Of0W}tvDiRW$b- zxaZW|RcDiT^jrL*ANZ=(_Z;XvU)$$fVugL|_QhFe;z&%Zpt38rJ@lr?x~hV)ELHuP znOq_GA~r>u)M_d{=+#*4QR%PO-qoW6`GbYF=lk5~4Su-#K=f{z8Qprkzqi+t{$H>hP$!6+DdJl*d@dQiQA!nhg%%>g zNJ~K>ACjmdFIQ-~SlN*6G=E39t1B(!`P(o$A-PPF&`@m2gm6FV(%Rvhk8|P{0em5G zXgZH;R+y$qaN2<2J8&+Q*`6Bj?*nZIKBTkX->B+=VD|npAtky`H#a};t7ERw?F~1~ z-&yo-`nk|S>r|CLdqZz`WkO6Yac6z7-jd3G0NzfG{G3>!isnvusrOnS#u|fv;6yt5{DnxLB_7%>6m4{#d`6qRH=ChBsQTvBLB(aRBbNMdXK7}spew{jcF zZFsYY)bi?E&^pn+bPbxsAX^79K{Dk8brsj&NR0va+g^+evudm4%TQ!SA!8&4?z5A^ z>3xO=+uo%}PuiyP-WnnkQt@0d8BSgJ*b_ndI{jN@ktKddTF?H+0u!Ia(0frIX}7IP zU0R94_ejS<8K}A9BQzTl_`Z-8;BH1bL+#3Z%KB`oHMBRi!zN<52GxSz{50dZT7dZW zj>HKw_$B2c{TI8v?to0y`3g9EHc$6f0qb{-#OYN`UAJ&m(De4kH;2W=W66u%CSMyE z%X~^jttmhLIRW1Z4&3;QdN70sz zYbQooFWno%QUe)W9Hf`NuT+Jd;D`3)ErKZIDe?l zs5*Y8JyG@8q_}^JITtalw5v!eMm8MxR7qm;J;N=?*&<@p(Bi?|$v%>7`TGNN#ofW- z^_mkV{#xj@;%aKyK!4>Zi5T%YOb4d7Xbf-tv#lI$GIXz=vCO_Y1raXnkE-Xyy+njG zVxot~{MHTM=j|r<(MHNrF*0&5qwz-Yu*HUUAkk+O1{R7RGAYvai~UX2pdaKJ4Sj-P zV)T;IcT54W|6WGlc`5w5HC(2U+n}=JoE3;e1-`x82?FESOWc2^GCHWR%zdxm4cSwZ z)+CPL-0iG06Q2?`ttaZ`;dA55;uo__@h(>A)r*;0S%NlCuh47ybT-`TZ24()hc7b; z)D+gKv#s_&lnX-9{z5sSl+o#%7)A4mW7+nuWPHcnA=>Dr3VZJy5n4tWf4c{)raPN_ zzRDE^2_46AKIN6US$5g@dgn2l5gEpqpQ^F#FNqbMP}kg+`VC79C|OEp5ZBlY`X<## zX~jJcGl31;!)kXNk2btZR7BC-e@fi(vxZ89R05f8d6*&vzlX6$Ls{KhR zVg)C%aW4@miE}&>ttKN?wf?htG~{zVQPFBiz%|Dr-GAh;TZgoMO@LQxX$QPd3 zm!Gicn;wzd=-v>inAhVkzbH~0^Mc5*%Arm?j(OrEt(#04E|GzcLK2m&YFQTvEvn!v zv)n3}@WPyA*Qb`Pb~IINiurZj)-LK?k@VW{s#wK#JQX{_dkU?}{(CVpP$NCl`{Yqm z{v!%U2@3HtHoXM>j|ul5#|=Dp##ML1uMe?9V9MPq$tEGzo<_Hd4jr>P?vC6Nbr$M0 z6v^pj8wHLYkszT}ntH$&Z}c8?SMbY4kvz;&sdp-z5}@!cQ+C4$=XO@R`(5WbZg!f8 z$sbf_8p!t2M=7R(Qe5Iy?s#*(LNh)t{i3L?B=v>9<{bT<|zq7$r)!dgM$R^wq^22Xi5nTx~mE zeUSK4EU=*na&i{G;g4RG^kXty`#BFeVeo}*KR$wlinHk#nUx-mwDM*=+7g-v_yv>b z=09taC(D?6aKlfM(1IwKz0sW>`a;%hF&vg3*y?>dkRsxM_PO9*Y9o71Gc~b7) zdRmcB{YKAs1WmmR&YDZ&eRb?cey=29>zBT|?wCk>?##9A?7oN{s}C3yzKCSRucb7t z)gqVYH5~;dRxG_k<7#jh+Z#T%1WxdnR$xArnUyGL?t64Fl``&3|K0;9K=clG3Oc;F zJINgm0K<15Z}tP1Kf-oD`V^QZN`E2Rj<%V=Warh$@Dr9{p;NG!NV9QV);5ym{hfX) zLNwy^Z?Kf9?cey-zxDxkcggznhX{LCp!qkXrActPJKg?2{_A13^LVYbb9W?xYEkVm z0{bZAUQ=XJWdr{zr<2rf3(OX3+;A8QvVZ<5RslO+D?wtsEd40z2R z?tj#?pXqB{EzDbO{rHBnt#&`iFsrwzavhar_f^blD?V3Gx#QZk`3=J=zugr7t@U}W zJT;Nwy7?En7rMnak1Oon1u2Y&i*Ku45vjc{EqTqWaIWP@JI9+{BU!6zR*vhN=(!IFB(4~TLQy)fW2EXj<&3JkWLc0h9_oHPTJ^Ey*{noqQ8ta;qtyHW&he^Sl}EcPV?y{MG5z5B zh0U#Uk6Z4>qx4*3p7^f=0c71TxVv^ve!NG`4ZR)RU2~4(I`A#(GE@jP^&S_BsNw_n zV7EFHiu# zb23cFr&tBGjRP~cN?P0n1J<;QkP~V)wM*_v!nL>XSeHJRM!s(kRHWY?;RAWDan;hg zn+fxmrK&bhJKczd!m$^yU$qI7{Rq8V@<4taZ=J7a>{X6PL)Mo^3P80b*L@LV>^xC# z8S}xaJLMtC7XA?lC9TEy9ZN;3Xn+$Kg@M^=((n|hj#%kP*xYcQu7I<_NMv}7Yxy{| zSj5isMcbFE8{D%+8wg$c5Mp4YtsYzqS~=QC)t8j}GL@}n^E@_2wq(_wintzL;xLB4 zHbjQFb{e}!0=)sH&XYP;m)c_(*$NPSVI?LF3oo_hYbj=)NO+=R|GCPhQ&Q@(D41uA zWDavO$nfs`Q5%SuSOCT(LJmAeUD#&&wvRCp*Uz0pKWA z6cufJ^A(nNe5|ThV-((BAvqsey^2XP!B=u?*5;Tr1j+;3I4cKky*z_Qf%}W3n^GQF zG-SVW;Ks%(M%{+d80m03S34`ZaKux&+bme^i+BL$pHf`W(5|_>8pY-9czxhbL{Pg& z#PZt2F7BwyChUxuaOiGWFZLW;j_kMIZibBX*EJKBhK0`m)McVcfnG;jW|uQhxNPJS za$S%fMAHsT@V9p)iI|6w-*W8AnszlV+hepJtH>8`pd}jV-Y%H95kJ^p4B9XW)7s>4 z1~9ZCdZ%MV4e3B=tKZE1hJNKkou%0X+t@L_mIXQfp=g{5x&F*D}_h^qo^J zo#*yX=Xh=>EQQ0&YM1g-{3xrKL|Bp}d|8=H%P4W&4RmZR_B*bUvSHkiO&#p8?Su_? zVAUP3Ey>*L!@RB5FoNpD^-gDPVs>X^3ngc_7NO|K+fmsaq83(BzdUA}<)k|uGgadr zjr(W3BAXO8Nf!3ei%)c66nFdXnS^bV4eAa@G#qp!ky3oxj(21#_?8^vPD{RoUC=@D zsK$O0^zG<>Dk33I4K8Y|Wj6CBRX|7rEc8y`qU8fair8)R3Tmz|S6+4BcE6z{u<3d! zCFJaQ(B$(?vp!6C=)CVCB3PX$*Md;QOFzXl!8Mh4-iA@I^*fvlD}r?U4v&}KjzuYn zT={Txhz@ONuKe531S3z+_tfl>d+PxORn1!2kFC*ZX-SsUmilI#@#bcT4VaI#B!%k5 z8@|1xvhE8x2{NiNTrom&sTsfxT;=7V&+Nf=qT7MmQzoD3Up$7Db_ryh6A&HjdpX|= zGu^fhmmJg;q^T^@zG3mJXwmt`$6;fmva#`aBTL;dpbJIt_s!&T@b=dqIand*rcPO#24mOcV`AJOfB zHy`8Fs7Kv_iQA5oVS(0QZCxFU+^ciw(Wo->U*RIh!nJ)y@fgL~?+;JnM_?kieh!NH z2lv{2mgGb9C{Bca=uII;PPsDOjFiqn$%tvBNZbKBX>rnil;?2BBRKI-JDi;9%F+OK zPJAU(JqjaCOBsSEPPym}8zDF%MXR1q zMbh2#9^(8gHBx~S72*F_r!};durS0pd4%6>8av!IEFras_!;Q5i<4zh3}ry7+(~st zl6=N&87r0s8Ud+-qEOT0Lmzdalc19^t4H)+?B?|*Z1JOYwhbm#Zk?O~YKh6%g((pr2WBH>$tR=}>~n1nu*-+_?$@2$CJiwu1KZnmj0I6c&_zp9Zcz-M4+CF#5M zdh6g=ak$p{t?qB5on=F{J$L|Zh{xIu{pDso;@ZHYAOYg>*Y1aV972L#3So%8wIZ`I zhbw$$QZ!rBcT2m%-F23>Z&i@cukNi+tHSzoJ0^wSQZWiyd!8P8atLg;EjAi5;?@2` zxm1lYBh=k2)i!8#v1RPMx&Qd5yE53_^WQ>0hhC9!*s?k``#fJWJR`U)H$q9=n+aB84 z^ZYV=a`_IsayhN~O!uh+)30+&I=z90{O8DsaU{E{%>ide z{8Xz9m3Eej1q|rM1!KI@`!Jr=%@i?UVQ!yL)lWF1C1XF>`PwL$bUPeGB=57cvvWATH#IJfD5JV5<^mdJV%1yPiUBb!iXl6Y`J-u%X_s5bo?E4PS%hww5Z>2~z( zH|D~3!n)(d%>z^G?t_Ah6~o|>_I_%&B|LO_q!907{&?{#y;O1b*7A63hhq*LkB}P~ z4|tmioW2E}m!EE8z=A%pJ|xj+6losVs~5?rACeCEld0`=hlF?kZ~pZ;_CM3PXKxe5 zToQ~8exNNc887(sL$A`@8w9`5eVuRw&lgw5{5E=X2ds8ei1q9{y!JPAbB*f0 z)*5b`iU%F1>n@o%Nt1m2In~sh<+Kl*Ggu`5(6q+eWI0VKCO!Qdl(Sij&nK&m++bHZf z{_WfY`JiRilcKRL1?r*Q@j@Qy8ZUCR$<9JAbzkY=Q^T)rg}NICN4^7D&0_Sdn(43g z-nT3WXwTG)%V!!7dJ{YpTU-YH_O* zyN zvY_`<4UR3XGx;pfhtp&~$AHN=F~Do<5b85CZ&3#%edzSw$2aCHqde!l(Ruq7?qCW| zy61}V)ln0&85<98W-XW>rg05Pi?sM#=SZ(OxtMpZYX`#_R`FAk z6?!)S!KZfDHd`(x#BKVNj_3TweAG1ijM%db@y!DpOb`=GJeupSH?RZBn=E%)dHRQc}ET0Ll9L_LZR(A{|{7`GxmaUe|BV^MhCO45u zCeLc$#{xHT6b;Uyo7bfIxRffzT zfwbaM905JLM7p@pRK6q{_lc&jb>_XSGDt`Y70(7s73OBY{EVJ+Bex@@O21AHGIH0g z#VmfS?ZXv=fMr_QC%SRI94c9SPV-LVug8i14CW3#Ahhrc!9Pb6kQFySrd*3C5<`Gr zQRpSAHD$#X)+p@{swBW(g<_=x4w8iZ5#=)-5uLAHXeEZt6%(12fsc%+)Y@LPO8 zH+6xf2}-_<@6U8KT?9ciDs#wrWM`~zZ46CVn>HX}-JSIg3%>XrQ{`B0<>G+B$&!cd zU2ympOPmb&X$da}$XN~iCLOV1Gn0_iyo|6B>Z@Fa9$fh-&fNbtBhs61Xv4Qw_#Arr z3%1s0vw}tW%1|cear<_>dXliJZ_wbwafnWP^a!ys#nHHLX?m_)?!30RjH2V4CH#+T z_;rM7eqQcZn(LwS62=uBf{+cqNLJNHSj`9_+phMS&m%Cl;ieG(8&`jX?Ebmjy>AJ_ ztEcI0{cjK`gaKIqTIWz5CmShrgHoNVUF{fjp`p`Od5j&O`(M(L!0>YUnK`FE~@ z=E=S+F1L+-8kCezcM7qF6M})1!rG$>fVYYBL&K!5t|rIP5Yx2Q28Tmf8=KYPi zSijws#JOLb>GR{g^#O0yF2#wt6g?H;Hth`Kgt{oI152JsN@9gy-dQTBma`LZ3l>?NJFXCL|Vje|rAR`X*) zt)n2umN_2D0@gc^@$%ey8xW5DB}urUYh2NUDq@wi{R^XFLL-lq@B->Sb_XZ9F)$dA zrJKmBFM~kpT4K|B zRVU+d(@6EUvC_BiF`0o)u^Y-Y&U5jL5NySm4UMg!mC}5fh5|+BgIL91RbF0B^!A@M zait~qzint`*l62tD(j?O44ntHrBx@Zl0sM|6N! zMV&7mZAW42MFW0>_-}V%QEg1>e9ODJg>PW=1k%!!5Fd2 z4kBk00!nkAF8gT1zOWFWux)Fpc%`^5h&shD-F#64)z=ffmh5Wr+Dt)a96Ks$Pmf3+ zvHa6dq!vsE887GzY(ki3@9*&d9DvKTYIB3G4bqqSmuh3Neww7_TL1m5 zC0f)9B|5Vn%vSdWO|E@;DOSJOYaT**%Mqpn4(f8&OWoW^zC?HfC?W`bLxECEeEm6g zjU!Xw?&HKm<5?|3KXm{qC>y^cG>`*=&6e790%MeRERztA zG3gFS+o|x2`nd*KMT;Xn!}s|kloL4|we9GYjm=A&k;I)}Zwk0Y>g4w-nB^N7*Ds+5 zU&qk?cHntHQ2;|Ouw*o=2^>AAq>*?eW4Dvg4zI_EO_>KFqLF#^wxMsww~osu=3oKK zJ@pTJ?#mIbn`vq}UI|E%T9{=#i}AHFJ*f6rDi7Ch9|&iJj1B`{L_?UHKQ5dRqrn01X-VrPyLXd2Qnw=aqbKKK$9@pf#Ee8i0U$}7n0NW8|h5wsW2u+fN(MzQSZd=vq^$Jr(?9a)_q=8 zed1H7`bUvH?3&?4`++%NTfOUCjC;cuu_&*# zc&wfxXzaU7L)+hBg7;fS)#T?c?wPJK!!^19_9Pf)UztNpV*ebC%kQ4Sx*VK$v`Lcm z@}vPUhWLu-l=$w-0+W=)z0Ox}+6y8Zp}l)Y5*t3vdNw25bcdyf=F}vZUGv>KXf`?$ zSgf1^qFBsh8HeK{A2mlUQGcXI&b&3Y!qmon4LM$5H-coaF)gB#_Wk}kJhb_txvDve zrWq>jvC&;(kMh*M*dHjfwT&q|zHfwdx^i{nU$;U3G~R2rC(1awkvG#Cm0(lBY+oo% zKhq>&2h9vP@9<=LxoT(I(c2Ockz-x`7{Ab`5}XebZdHKoyxJD0p$$;4TPy)(BKG8tWfdU{&hE)`|>d=$*%N9}^Iw;#rhc&S4j2ys}ec{8~n z-a?i)v{_QvLVVXEt@#mBAIVh)wU8yPW~~0ikl)i)RkN91Zd`7z=4Fw22k+YJQ7t*a z++BX;zp#Trj?v?>Z)Q)Usliip;m4CM0-!;}RGk5YR{m^%3d&3&eKx}wH6zkB<>=hw!cJ6IO9n)0ipEkK2rh!Y*4ZE3X|nJK0f|EKhLg@FpE=-Ll|XgsyaTD z@!e8d8LQ#D!D~{YBA-1XX*uV=@7iLgT?()cAlptr7G4;u9vd-Fvv-6q3e6d^eyj4M z3o~s9)DAg1%>Q&#L^%|&`eF#EwJX7rh3I*E_iBVkx42i>w18jXWAHvx5dqC+q697_ zX>^jFS_B@Yn#Xtqi7iHBI=SYh`p4C=8g?MUFDgc_^lNOIxyr9DyiB}|@_K1tliqN> zrA9mnqSt!4y~goMA-2J77DgylQ_@NO_VjZP@~F`-FkzqXG8I-YZ<0si+5i`Qem`-r z-k}E3tP+-18m@&jN2($<;x4XYhZn26Q|g#h;|fKw7jcyfZoX2>Bfp)4wp}5-s}42a z1t-H8yaL90(9_QJ2bHC}j|_H4c)8oLqDwL(VG+(X^@Cf|H{5&z)CM!c-ptkk-G;wy zY~VBjj>T{$a9;kTiWY>dD*w+VXeuhI#lOX5Z13dOdS8}VM9`pfiOMTfGrI;Io3$Aa z1AK6EW1~cc*uJm?%vn*??2nCEUw*h6s7Wa90o+oooW!x;!NH;XNU8w2*sSG9E%C1k z_%MTF7aB2w`kmpG6;N|Ml1a`#U$~fUHuYkx3YflUL3F5>s{}xLtNRTuR+C>Y+XVc( zK1`BbzOW?b#Id_7TdeXb|4$mH@w<69LN%@^UhWm>HL#JUC0J7QHLMP5ef^%Vu^c7@ zO48ve71Q-#Y0jVd3aaR;VJ;FX=?Y|pS0Z{X;VoWuZ{scDh;2|EKQ}TyT23|Mb=4oFS!uT-^8RjDeGy$i^T?)7F`LsDLj@d3y&u1@<#AMp; zK8^&DqL8fhlG7rlFw7m`N0ICrHh#%+a^wsXTT^&GmhUuUcXxIKEnws0e#qF-ptZ)=u6ZQn_Q|U6vbWYHO;z`#QCV%I)xF45iJk`r-eATMzY3ye`6M|#cCc4Siv7fM^;92 z%WFq+V#Un{@^mA}7f~>HuR1J{4YYj5p55O(kJG@E$+Cq%`mqj{pR^5ZWL1#RZ*v16bX*gr`H! z^F?(V?Q3Lg!fMfXUy?b~Hw}TqZ>?Mx%GAk?%knKB?1VPeOjJ{u`|sCTbtP#YZ11a7 z2aSD8qNd22?=a->1u*N(hl>CT5{F8s8Eu0y3a5?if(fXkw8u&2kc@-)R}Io4QJzl% z3FDo>B?b!OkX1J0x)jQi%|^-phAgkNm|^*6j_*=v0k11gpeG(-STS$Y32XKJAx`nc z;})2BtO)cK3T(e4Gcc@)pg}!S$Y5+fn6Uu28^7TR%7IuFlHvV1u-S&05C4uf3uEgPSVSBCf(#N2ZMj)oX zstfeqF^)qO8xrFt;K$D%xGT~=#_xHPzR3-3wp*GpSNjL?z&U>fTJDt>-yD5Z>tH8s zwTApQjv@?Kihxtd%KizpAe~OGcDCS|AwacKe?o2eeG?!-jFIwh!&+6(Z>lztocP_Y zbgkf}_$cFqW-{>3?nYq}mwM7H*$lLd@^O3X+HjK5&F)zNYtQPGlL2qkIY^?|D!(-_ zABfFR)PUi9>e?K?RVc=}jFl)qfsSWq1!viH%GlyF%Jg#3kEK8^yEibXlH<#kP07j#x-_SGxVn!gl+PAIQQrBhoZoBC~$ligUdX;Am#AF2+elV+B< z87jKzNiLcU#$rY^kM`mFqrT@xHRpP44lpbx@)m^&c3uZt;)ZoL8DyM*LF-z4E9D3c z%iCR@5Zg`ud>x@)g9Bj_?!%WJ)$;{ajk}3xHa%LwI4h~wLLc*~{nL8zW8kKkDKwR?9vF$!M~?-AsjdNU-4ECOu4|el-@8jwQZAWHS6prtnjITt&)tIg};0S^z^G~KeGkR1LQhF09k>j_1etQ zOKkItJlVd-C|&j#v|`$X;sLRVRU5vAj?_?e6j@s6rKNNDC)XmAayeb(mPk&2qab(L zj{VOM)@n~H1CR8mB^>tl+(4HQ9lCf8RMyXnU}XXi%V>nN-E*JWtTEQSTSkV|8WBSQ z>{H@3cwyhN>z@7oIvx&(?_3mjK6G~^1pEp>p9*w@#6w0r6OFtjyx)?DhX&ksF!$|( zIgQ9y^;G&w)gt0#gUqzCklmCcIMufnc=>X)=mdpoC(?i4Edk@42I&Tn;q2af6gtVT zn%DN|7ANkbpfS1wp8AcLl*%|OY^>p=QJ6zt;u(?)7|QXF*|?u zhtsQQ+2-kO3g+#~B_<(X%JLxP=1p9vXfpbR(O{Z7Uc|p&MjEG{Q6Bb)-rh9P11l&| zU4;&HUv}KI-!<<3%wSCaWD%L$)VX6<>VoqkMCK6$zRZhzzsyQkKsfj9tazPlz%Qw6 zW>W(9q8R9Ac?yLqDHSnLtzzx%a*1EXYZj7z_Wtc^+7PNkuSx4L`|oZCni6zIh#nU0 zB0$`6mwxY2;uY+UUC&%)Ew7}**35H}AatWD>AT*S{(6N*j7|YYs!trxI1K0XYIjUZ zcpAZ($MOcSojW&pw0gjbVHy{bicvV4k~hy6elERva%jwJTKz0WN;)7JQ$8-bdIoa6 z&T|aT?!X)Ja9m0}%lqqXLQxyKQr?P`zOM)RJ|aN45g~Ho9DO;14Mv1~u-|;gFMn+X zS>Lb~kiXg;?jI1)IZdmwBmok!OX8=6PfL}nd=8If(9iR&vAi~U0o!$hd(_(a$BE-1 zTMu5&cics>YotU}Qy8_(n{P!@Ci}tyZ;nYbj1QtP&VL~fW>jDz*R}lqC{R2iFibKZ z*InOm07B8zJp$;ysqEm7{=>7>+Snpowz+cfqr#5ra~AKKQk%*+$l+m%kX4xvaa#iTg6`C7~VLLs$tm zjU#E35xVPO40S#z9LOoZgzYan_;Gh?WnrV-q2Ah|-{EQg9T!Q_ee@H;=z(oJJ}toC zC?WIo-Zh>m#UK{je@+O`n}?O}sOQe>w{Mr1!EIa4IDtp&lI!VOAYf$a`nxhstJc7R zy4rq$00Xn#q?5{dYkws_EzVMVB$^|t9YiN85#RY>nqO=?lK2|B&6wu-Ox*C^3T7Hj zoo_bTRR@q<3E_|Bs}ME2LII?KJ((%)xQ+LQU4Mxs4sF&QYgVZrXdN6-7Py8!fhTdl5f1b;)_MhjSP!4k;Rq-A1 z%~n5tNZ>8@`*#3d$u{|5UY+*CY7dC**~*~U6USGBG5P3(z&j#D^ngDB4916EJJbGJ zswgz5(z{TnbAPUro*hKN-;A^u-2DP+59)LfJJriWRfBX;=^bAw#V|T1;1@1=uzLYK+ZSV{ zhriJOwC#WK_LgB$yF$OBL;(vVq{ARYx;sVb zaDV|BO1g$_7&vR=FP`gM=e#)o=eo|kB4+mNeeZkU_gd={i$+%v?I-1Q#T5Iw@OME5 zQwh{igtg)v5WX0p+@b(FU)w*u8oaOf8sifjK zUgZR8ert>uQ2l_S@asJGvoUVuqN(k9&ODat_2nZj z!Hw7zf#5-Edi5oK*QE!@-ulpUWfh-AxJ#zC%D=?Z4E!f(4 zIDM*Q%k=T0NHyU)QjQ`RO&^3R2cPwTO)Pz1h8F%6iWVusNF9}GYE`(X%ezc>f22q_&c z-YFI(g%F}vCz=gD1Z6%gQ;g-YX5N@rjnd>DiTrn=A>=jbt-ynRxm#*(yf@o=`ph7N zvKq;gVtTvx7$kB}ekw&9&=mZeBfykf*q!UR8u1%l(b>gZtPr*IyDI$O6jg#7v`EY$ z&QL(|_IH74XsW`TT<8ss*i{x4(^gQXrAYswG)BMp6vkstp;GPi+hS+=OK=#JM!{#S z7yTnycG6)g@BfaaJ7kgl&U7a^FKuDn1e{ZuI5k@7-F5TH=eUEfoZ$%h1p)uB2vpiW z!ZivwQZ}1Xm&aMp4@{nC(MIMn=Pr)S<4H&Y2apHrFG*7K;;wr*! zYlP{y7Zg2w1`(if@7xBL``8^wOYK!`$Qe|p6|+*DGngbgyG=&ur4%B8XLst7(EwpH z5Ev1Ho$7mkQap&hZ?H*x3=`0u+dXk>M>7 zFo(IN0FxqEemhv{-rA$DT{@HW5u2`JnkkhOLGP&sfO$IaR`lkvbbst%h3g1UJmJBcs_W;OI(Q5i;Y*fpA1w~|KZ5{L7KmI%(uHbWm1!^N?_c^Jb{p) z{u$&lj-*Ef_doxO^xDwZr*tIWARZ|4tbxZ96YCjA!B-$-Im>dW$+8yB7k&Ushcd|5W;c6H zS2y_a&w0OhBnVZUrs-FKJ-+tVx$_=;z!rNOCNdT!>HvB3l{;qlyXMUae*ddd%EVA4 z#QH2I3#qUHx-_if61A%U`3w?uNrrSg z^U)IExKDO%Bh}zDCirwS=lg3y9v`1%KjE9%C>)XY7ExEo!L`ou6e5H~g2WTM2*ca$ zA&+lQY@kQ?-ex{PF6CJ-zkW&v!n)EUGoLCbjFjO>(A8FMdPk$8i3Daxug{1FI6~&-+Os7dKPqZZ3xNrpHoL@ z(S7M=iU*8R!I@kDW zLe+Jnpm@>z+PEN78QwwX)Pvk?@fVEp(c(x2xG^{(g6wLE#?eihIXcJXCrCV%im0Ix zKe*Z>r5}Ot2Z#k^R-U~yuB$gjREtMsT)SXoQ6f;n;kV|3K;{T~qmiQ#jYl{n1hpy| zx@>m(ar9D3PmeCv@oheMl4egG=&|3Z#g%Z&se^E7ar z<;#CZXeiH+c@hy$`byAeA}30B_U}J*Fqtt7P^9N_Z&?B;%Jc{|I-1mO#Fdqm$*8J1 z45=aUeShaA%F?QyIs`$E`(R%K4VQra}(jW`alsm#Vao zHIWVS*H#2m`^(^fz2F<{t|oSI%_^4%5ee+yGymyi7x((2?g!xB9*uTT>x?9F(Vc@o z-b5h`dgWAk8)RGm3xmNAu&5cEA$J6M{eR~{)3qcBIO-xI*LHY{=-5{fp1J-Ia!)_s zPwAU~6&M(JTZd~1nGxdJ5@bhpkTRr{4~nN|p!OWylZJ~&ryt}166{8Kpfp;i!U2MW z{ys31g#i<9IE0zsT;hnb?6Y}_Ow^#iK7Gqc+aG1TA@mFfEZpGe!r1@X{p~Z;h^~LQ zkyFVI^~%tqjOmyo*Y>k}kBq{ySxTQq9A6Q%xp- z*82DKT#im}yVIy80x6)&M(!6quS24uv44&p(*3l7V)I*wni8J4qiGS;{y&t-p8}AF z6<+_R@%z!-qtkRpFiQ~+bS$jE;#NoObzd{%w*GYGPY%>zQRaI@((XsfhXe$_WIx?I z(dJa&5MzRffct;^cVP&+gpf2k5=A(KirEqM;?V*fM26r6zz9FVMT4);n`V$64#`^R({r9F) z|GlYfNYw|y5P|+82NT@-=mw-7-2m~zULaC(m$t{3y~#qZrNQa%{3tG@Z3YTM*pbW> z{vI^@-{%Q_B^ZtQ{Qo%AT}DE$Z|ml3BY{zS*`{V$m~x@Cgk?wBc%xGDzS!XE!Z@d78x&nQCnRpeh{sYdJQ@}Kb`JiLjQKTfc^GLAxus_x@ z+q;p5%d_L2CA$-nc0rdAi*|t$(5a>8=|IaUOIm49%(Vp5pVu zZQw)lv702{6J#&8O1}viE|`kP<~io9gQSv3g6`^{d) zLGq;G_{sB=xaitYCpDgC7Mz;e1=2=(2UUrbnHA$B&oe8PB69~)QA-xK#J_lah3|yTm*K)^j{!mL2ZxCb z`DxTOE=^3<$g}pB?4UkFXPBD8r@jY<$Zf>ZMlYo-wk7gD)P({!;0k1$@Vr1QT4DC# zMYc*H&=9dEn0QAvO_xYUpRC*|Z6W)fjWV?S$K69w_-(KBhbN+9g16|QHvHui_!?v1 zlxD?6{>>|9-iUJ@2Q@qW;X#EA;G~ZM*kki2UVmRQ^36grQq zUY@+v#xYc040o+ttRGKux@1aw-fVAN1@kc;pGB{i7BTsPLAwAYMl7EyY;kDPqfCtQ_6NQsr=ltC zt5D2(lCIqWlkMK5pu_r%3`{n%tNTj3>n`|m$d)2(J=RO6?xj(($wlo?aYd^$%GS_B zLKFcDQ0%D-jmtvSN?zOw(RciMvQ!l2da^A|6$Bx+@RP*Lm6Lr;P5mQf}(({;8lMot} zdHXOZ#NL&!v737^vT>FFc`z)32X`p86N{o|Jt+ysz8N+uwt(;z8B4Df_Y$!i8M!a0 z4e(wFk$$k$z7+;B5$D)57E8r+_BSV2#I^!9@F zSbww!stC$cs(pP6j#KPm$c6}W+k3Z=YrRny=AuVJ@5)-@RAqVhR|5qV!qyMD0o2B= z5%0l10InR*Y5U4+_jeymoAr@y+JZ)z;Yxw5Fy`5nj;NZqywlM2-q;wZv2Y+D^M!&C zFc0pzKxIzl&chQhn{rV~o`+R}t3xH({^bvsijGJmQ}?1c4Fxq0my+OPYg31eNrm$! z?PYvcO|HsS1M~1l1u-j*!`p{_FdTmSJ4U~rH^U#Lg5%$3524^un%{S~EV<&Z=2338 zch#n*m~3Z=ZdsH6Q|0Qa+!TKPHA-o?V(C3EW%={pW?oYW*Kspe}$)2fk zCliqTvX{W*d`sojDqs^#5!D{hmy&xTHJvwojg+E+f9XfGuQ=CRwM=bO3q0GN={tcH z7x&eP_mFRu;q~t=I3HH)zI|DGw?HSipW_*5ndx~Knj$!>u-}F?=QqZHq`kE$Qb5+h z{%yTlO~Q4n&I!K@HtEjAznNX@-(Ug2RhjF{un}n$ zz~7pR5+wABx%^TH$Ts5b`ax{>gdr{?lNx(dskz; zy)q>5Ag5mUpWUIcRg4M|ZtzYIP$rdxVzOeNI(C@9GsA~X zE0#N%8+Cj1@#dvd@*l0ni|aw^1rp(ZLj zf9l>=OY6%CU`J*NOxN!|xb+Ixx)AnKMd!e}Tko({^{X*E(al%G^r=^7Qn(DOV_M&n z_E^_kWLX&j9s!rwrE=wh%|Iq$d*<8NV*Ni{=0lruuuOO?MB#adh)#X_Wtb!Ov+PuS z!~q^tGoB4%?xdaTP1N}Oe3MCSxW1}Qw~B$ix@>F7FJd=*diG%-Ez3Qi)BNORe&sM@ z9wW|eif2g>^s#J*RpVVplF+bmd(yI3aKNN(>A`4!+K3YD(2k~ubNMG;sd3!|HXJ-V zPc05ARjTZxHJfL6KOW%i9#jNc0Gk%xX|S{R7`W;xdEcq6T}r)MM_~F>(QR&HoNpSJ zEaY6764ElOcRW+C{GBD!hk0WuAd!%a{N9&Cz75;bfYXbpJ2{UIR1v5V!=11P%A()Z zLwJFvc!4h<&Kin!RFn*_#-sQY9_#!P&-WS08Ny)DmL9Ih3Qo%DXI(sP0zgT~;J(>= zI+T}t$zz3*4}SHlgy6Ng2D%V^WcSS~)W{YxAkFa&LeHms`u6 zLFW7uDk~`WE)FkQ4i8;6oi?KP1DSn8wAToNZ0ZM+#WL}!3xh?SFGdz8niC|!w8I#7 zw3#l>GKXGoJBmn6W~$?6Yy_^9C9kaK(R$~Gdg)GEmYm#et-|EmKkzTl-51P`_3jc6 zV|MlaB|?s!T7LAF^tf0kbk-n^`Pb^Irjw3BhqOf!?2i=l;+(|-TKmUiX zqoULwZ9gM_cHHu(a5i;!p^Udo=#3Z%XX>e3UZlh%AK?FyW4qDYzqvEU9`wsj!6(!C zffm)w-gljO0_(aWmeWh*_;_?DtNJ?rT%_iG*+W!?5DCc{8q!^;pMrVX_nwo_fP`LL z^Cz1y4IHSW2eb)%8?R|m&4rjGxmp-IWPg1~H+v5g`izcafINByne;HsF%)kHlq>lGQh5xXjA46%zI&|_?^BcjDGT!tsG72P!p{1RK4`$V_PaZ@v3`XS8UnWd+YHlTZ=GPY~P^a=DNi+CBSP2tY2XkbzdCZGN;kBPe*f)-?4jC~~1JluP5T43h2Z)FK|ryX$*0P$;Os z<(}!5SC43K5#|)+%|WCCUT9UILKa8d+R0a?K;@MOOn{Vq5?oC%*UmaVs_-2+u{gtt zuCd+KJw`BEliYU;Q9WMao<_3oN|mB~9zY}U^$E#DmjBS5H9T(Z7Rd4F0uN0!EKGr;~X z4rNhYa*W_va8lp~>f0UVA9LdQ53^FyWv#AoQh0OR^Uj(XM)xERdL(XtH*85i$=VzG z{3WZ*uV=Q1+ZQ5P#-RObGS#z&pAKj4W|Q7Ad;@t|EFM2MFG_KCw4A~f#wPokpp(1> z=HSum%2lAzbN9lmrkk5}20+FKQJzRcs@xR$?oE4CglJVoEEZ6T)@)pBvdnlnlb7fE z%&cIoRNLpJ5CrmnVK3HF&bwyR<-3SnN~q64A`A3!45yb;Y#J21q}2(fgNu25?<-xG zsOyu@73H;YAD2oX0){M0uez|ktU`Ud!kkCy*s zn1^7GDXsHvG!7acW^#6SmAivLH>7;?5~W~Hj{Q`dJNJ@wpZ)0QIgKKZs#(FGI;`)O zt!$sZG)*W6XsW!bd}y(}{Fh}D*j+`N^pty<&G$_@Rc+D+Ie%+>T0wEL)t;1mH<-w` zlodl923$D4?)6!8EX&!1#}x#NkYy^_3|5-on+3*%%&cp4)?@YCpiqui_9^V{xPD5 z_XMs$s4HJsEWvIQgJ&2Q-H5#b$k+A9Qo3 z3m-gp$?N`XJk4CZ6bCnSFN{^gs4yvQ^AZ~G)h*D-5Y39vg&QU*(p+1`Qzgww4t1x0^!rdt*L|Xgq+K)|7`w5?AJ|8-@%$$5BauQhR5@a;F|-kBigB!B zhg`#&TnjL0J)eIZN`H6NiW}$rIKrQQFe27&{}sefNMN+Pc_`5D{4?lrxDgm?r2bw9 zqqOGy8k45}UxX>*2VPvj=(-}{kzZs)Csre6@~u&Mk1t1O-nu|RlGuk8Nh5)r8^``) zb3sHu^3BbvMD*k4s;a8w`>$>QU_qB1ROB}Stb=x*2PZAUuJ3+vIaUOEg`ihX$rDSS z&Y_F`UzJbIhC7v*Ci^v7b;7eC1!2!>G`i!c(n)JItmbgMx#D& zkcd}6-JoSA6mNW(sA^&t!LQ34h7`B`ZbosABI%857+cn*>(|uv?nC>$PQV$}NVt z0Nr~+roXkQX+RwLtBV&j2Fhj18awuON(iV8rUgCgKd8enx(l#*tQmf_=|=qi#(k)* zcKQ1VG`}k~TK-(_nkK`WXI>LS0sWk}ZH0)SfejY4!azdO^GtVC=mh^glIc8W<9H1t zUxw@z-K#b@Zq(9 zuE*fLf76c(H|<>7fG&3fyag@VPoVp{&Z>pPxbRQhvO4olSMS{wA%#>tUmsC=mKGo9 z{E5F8?zK#p>vGLCF=DN3Fvw^34NOOuEaacJm@aMV2 z_UAUjb@>H__`zFcsk^h9FY6}#Em)QZmem~M2?z@Qk-0DNggNQP`}^_|xaZ;O_s zb$80+<(I%_U}t(MW=g6C+3mzy`#`*#6riz7I1P;9J8*%{k4tQKWf3{{OrAo2&fRQJETCG5BA!cZaHe7G%%5&SR*#skvRgb>nd0c2STpz)_a4 z+5I$g0;{~*>nb0`eZ&|3jmW!cr6j|_3~XN4PFFW^N|5(P;l)S5e~UfH-HOkpt;Zq;EvxO?}7RNv~$qkF| zKdo04LXM?g{Q3Dr_~dpdS-x{O1@{aVgxB>t=jw(7ON#sZ|EIxOJ9#4&-<)Z=u9Dfz zbQ=!q=l7=o2t1>;E(0gt7FhRU^S;lwlP3YiHyn)^qkE?{&VYj3L~kRSEf*K%l5jrO zuu(~K>v{LtEl_KLQ?|LQ0Jt*p8cZxF`Ov*3@_kBY)Nz&u9=e{`$H6LeOrKrq3eI&~ zLd%k~Ix`{81NrqAt2XbV#v;m+8k$3#42rC=HfuZD3m@uJPXvrRA9!Oof--1T69DXd zw-Y}M0qivakHK4HI{8L23`&dfsbk+aah{pzMkleN$?pZp-odPR37*@1R>0qV52nb! zgVlAh_N*5&8Q&~|qm2uspKFp1(jM$ZDKa2_V6cvTqLyws5akLKL^+t)dZeBU*UH8N3 z=P9du3olbM!k`_SX2^>qU8$Q@igu^R*Q+;6sE27)!$}gC%Z4z!|o=(6+Qq!X~XFgr>6Lma9R6mV=? z$h`w&CZXpE%LY|p#66CScUnbQ2evk&bVNDDi0AFCyS{$7!e9cL>lDylD_EXkE7+P% z>GJ@$?Au^&{hmny8)Mx&o=SL5K5@6gs5SEU?di7%mk(NlR!qF9whs_iZl~a#PsIb~ zzdzgO&P$uF@6N7uR6W-WU*C3l4{;Cfl@IV~-Z&Kg3Js2caJ}5$i{ifPZxu=+U-@(-w zz$*c~O4$b$Vi})GIoSZeI}$MkCwR6O&)IipyF3iJreEIkYkVTJS*MwT1G`yNqh0Rx z<8*F6xOXB)&5PId7_QAJK=0d{F(z zbL+}2gvnv-)9&`{i9TKV?=s7zo!$W&-`SiI%{NME9tJfA<7$o)Ih1dJ@MQMz6Xp%> z4JC)%J52?JAOXkSugt{$zv|QrYb2>Bcl$W~+a9E!Y57bzZl)KZdMhft+xSx4@xvRJl-A2hzI%`KjK)3@b6HrQcWCiddBad zKL&(Eo9%w?dE-px<9uk@>xF$0rIOmt*Z9zqQa02R9Le-l4GsQQ_P#bUFZ%U@? z)kXwlik;czn~?*zk;e*J-ZQ2VioDkf17CE^Yy_fE0hF~2r=0nkC*e{ z*8BA$o550ZV%fqzCg;uFP0MKLeb(~v*pNaDzVt4QErIx8%E-H-m-T&m-X_(wQui~n zuA3{F>*(Ak_;sS_)K{I+;A#?{`hymF zrd}*?hpuz}t=`a~%?^Jf2BIOcQkhD3U?(=EHP)(-dUF#f4Q|B9@<8zHMi%Zfn|fj@ z<`e;)ORM2tVut_ssm`RnTckBlv0xFjA62L%Lmk8f%ei(J!V4o9{c8~iKZ!?tLi~i0 z*Z&*u0X9|b>StLO*fs;=1ssQuXhZP!WxY3$E*#(kCtE&PBBlvPECl#Rtq&UnN8s6Z zM5scJ{ObXv+v7iuXamUaH@YGGoGdsqlvNh=YQukkh=}Ov7V-~x3BMDg+-F0uF|d`> zgotFGP{zl{=ZG!@|MpE=0hc^(@7Q6A*&~h!{7pO>Twh|+OG-*&9uoaWaR1^}Qj*B~ zb*}E9BU?q*^A((iX%K1oL?Bk&zquF@{)6u_X{o0YjZO{A0l!TPJ+}RDfDDV*8uOVi zF0BY!4Wd*E>v5A5?oQ5^jjt#{y%C1ic*0&@4U!xSUX}ifo_7bgI${)ryHoNKFP4)C zZOQs>v^tv!Q#3?v12qwF^Rq?!9r2VkYm0kH-JQ4aZ4T21#X|$8alF&^jQ!X1c{0M+Zd7TKOLChGOmZ=3l zWlgv#fI5s>d%?+bgFm6xT{Eh4l?NSrDgnjPde=MIEyb0(dTz(v=}#D35<{ER2g@n1l@bLVPJ>+`)rRhNfzlF}0^xqg zTEHfs-#ft|v2p{>x4KcWd8kHR);BIYYV3D)6SL*E7*fQy?b}*M@MzN4l{ZOsS&Wd= zs5dUdPmfTHGA#r}-SK0{QXh&ZSGU1$1_~ps0318w39+7i+}0Ku4P~Agm{xy6DHw(y z@Oa^BwxptRT5#?@GPPf%)LuF6+VW_8i3QZH-s22KBaC?#^a)$w>Uwpo+_;lC7H+H&IW^W$;*k4W&yM1C? zG{QHq+HZ?iv=A@3a{f@rwLSmJ&(rU>U+Ui3l60J1)1B%s3(uWs4YTQYg-hQZ?y9j~ zoY@n@H=!sFAqS$ER=nWZT0X5Ac38!dUSomWWABFhWt=Vu=oR@xrF~?zYnw)Z-Slc< z{*?BqR@0Fk6Y7b&Y;U-d%6y{>Nz)E-HC2lTS;r5HkX9;?DKv-rTmEQeeOAO^XiZlJ zyE_8l#q90ngt2LvAcRA@1)EL{2G_`}bDa^*jET6^qRV59N>n_~ec>g6QE6C?H* zNMV(5f7J*~Z>i5O!mKB)0AgIW()xCWR!*6}E(vAc$Fa0%b|=&zex7c8VDzzri1 z^R(53oa5l{+h%S(=&TeQe08XNtoaVcIfxc>N5?pRKb-X9y>Zt+`SI@4-j*G_WP&`) zc9Zh$CQJmas z(X56Z&3AE>1;S@_c+CeC0q$pJ4z?+jt>=LK1lb!B*X>5Guehpsa%UQpqe3@D><5tL z=U5-dShks+v4vK7#<3`#yzPloBFhD*EFHUYt>Qd#y_KRZ&g=>&HNty$&W-(KgFAUA z$seeeTRj@E7!3a|D8-51GoQIII&C5r-oLpfu(g*iG5dv8UEv?2=eKgsUJq3~&y*WU zOwe#)sd@zSgs6uq?A!xH4>5Kv3%72!Bq4)$_FQ{;8QF~bdn}jp08UlfT+%T9@Io(> z))*M|wTf}GI%azx%stZE^PGIa?*s6fC}DRlnfcTZL)Rlx-`I-^*1njI?V=c?&$7ny zBs+rjjBr4Cd&{Y$V#}B~o4tLa#1yR(n^uHI{w1!uN#}^P7_{M(>nE2`eR0K;!Q8_o ztkApq2BO*E?7(LUP3|_yI9T^T%^(W$kg@LjrF8z`Eg(&Ynt}BpLW^);p9$UGBZ)R2 z_4dI&vEAI$Ts&;>-t%gj-X#%E^jTfWtx`0b$Dqau(~e2-?^Uwz-7mw{G`3pStOd6U zWlyhoIDez*6w^&I+`->@`(pMc%8M4E-ny-f#iWCAM4~rUzOZhWVX$$q3ii-%jV7d3 zb#&aXyRlU;dwj((F)?u}HD?mhv7Cq$Lk-$%G)@caswHKNZw=6%@*4a;i?7vMb{4HZ zbkkaY3N3;aI}$7C(i9Uuv_|E~wnQ{5IPZazgbX-+Fq8#fJKb<&3u6`WhT2oM@;)ge ze&%2-oH=a+WMkrO6#9D|P6y*1sT>Oz*sjX0l3#`9S)8Z*7|xynI5-s13_&L@BW&?d z@hoaS^y`=F+K82y#Pe;8H?53*TpyR?NQ4+lJ55wHxb=jnSSapISxHG0j?;Fnbo_~V z^hGYKQ))IRTz31%Cux*${GJ5cMr9qy$duW95hAA%K7A4#6>4`zk7eMi@to6>np0=9 ziBIWlpq8hF*-LWh?R23j?}-yN9(qw_u1lz8EPjv4t2d6gAFP85*sflt5IXg4gEN6R zQsYnlVR&o6=`T>c&04dr*bzpF^+UrRsqB$bmcE=3o-Legln3ZWt@Z0==u$m%9doqJ zc3$=$NJs~S;_G>K*jC9H+BMzaTtbkk&N$2a+q)Zg%nY zF3U*XVk7t3|D=HtJ?!dJ_p;`%%JGY0xAjSk+}pK?Ns5`PBvvU8{>UZfJv?qqo%E$u z08Jog*=PRfay}+OF`*aJ{gKNqrm=Olr?s_P)3WVjBg383J-(cGh}J!mjf0QrFQ4t= zyn9*$%WXe)UyZ*mB*TkOaLZ>w3#x4svt96v@--8|QCh6;oNBH+M zk}8Y<_G5}$ssdXK>gsixGg+T}XV_uE1;5%T1Wf$5(@%WPr?!E^C(zG>$(gfC1GbjC(8fxeG8BrJL3o6NUP zEv*#sbxKRA-x#VVvT<6+|2S{VIj5^`NxDydxTl;kg+&8Mf(D)(*FhgGDgC*$b)Xle zKgN@pZ|hoiXCnF}@%7Nm#|qurdziO7sY_OLsQ_IhZ@N$4JyHL~c%CB}pktkcBX#!Q@N2`%Ac6suF!QbH`K0+5%^=$trW6FXpA> zgg~0;#vB+vPyBqRm8w@ucD1;9?1T#RBD?(IqCNKGF%f8c=K~d-Lrqwufk;o7({_cS zH7T4yrs$R$>9Q5CSI#@cC^Pm=^wX;dIl<+fdD3wMd*ECmz2`2lX+8oB)7;`=s$v2f zPOk!(w>LND)WRkFrT!k$w{!;DIHhJ(q@A-QDMDJy%%Y1*z$I55X_=yWvNUxm23(-R zU?prx@{x}KnS5&@L zPxsJ0TU-M%n~qu0YG6gQWtXf|@yemH6k^pOPF5qzqBbjuuzG^uQE`-f3|BOY(XW32 zAY?EJQv;Xm1S@Q@7Q1#OgK4Mkosh|Vb7y1cgmiQcG05xUo!)%;VyT4RuY3(B~JTgv6@tYHEw z#87Pdz=AmKoJKS>017n?IL8^a$13ETE*(=z1h1=|(jR-PD^g1`V-2t5ubkDHXwtdY z63KEMK%^qsdPG@y)Ud$3Ms(;OdcQ|AIYk578%?3-)kx2J55@}UWs_Z@M-^|Gxhcd5 zYRN`ru_Mg0>VLd2i}V^rUd6Zf9m5a-IgiYBWLFMlGwrZ~s+cmuKAtK!+>5M9QSa

`Lcru75oB+swyxO=|RJiyw^u9Pd2)V6Agog zmQUr7N%0O)dO1MsnxK2A5wz%J&>7%a1uH`3qJh09C&dfTcDht$B^KXMhb*2|{T6F? z+B-V!d(qHsToX~+y`#bu$`ZVo0`;wYjfc3?+VV2zN=KxOaJ`#-q1O*hq(SQy#${;Z z+y2t0AeBu1JQO&54+oIN(A}{Sn*Zgr06il4*@tp2aDxUY%AB9?oH^(57cbRjl!?MV zb%|b*jl9_sb@IilGuj`ii4Svr#;K9|c3tq|tie4utKlWNPkszsbx+wY2@7OV^qJtR z#SsTf2B2S?DAF>HI9b{f^FrYxNRg-eGE^{uSBlti0q=iH_xG1LXOmJx$D-k z8zI0}i9s7(4I&~rbB(je!rQA72sWR7`6XrRR?)0W5ZxyshaQ?LNq30C2IMQCKHMG> zrNVSz^za|Jxj`L;dd(qPm8Oe62M;)Ymzr0UExlFWEu^qJ^b;0#>VYp9>2?W~rX%f# z5HVy5FMip$GxE1QSj_1I5nPyD286a0bRaA&jL^K~76*SLjf$YJr5}dA z?l+;#2z_Jwg&-?2ufvoGFawHxbNRSTgF6tKqCj&f14acfNZB(l9|1dU5?qmC0UTMY z@otnC(IyJ3u10a8#!ky52tuRY)FYwRSx2)dZUkQ6I^6Y4+4;oFpoA?_tTd8o*qXQk zp(^ShhI7+HU~CA@3(VW`{QK5eyJ~l4fyWD3k8IO!VLrS`ateof%WJ3{b-tyc(W$&` z`i-BHzlv5}WMq4TzpTWjKy}q>5b6&CQBf5eMY7Mbfh^Zz%}Q|mv%hbJ8d-Dl zCz7xN)S(JA`(uKxFh5TlL9OoWdBitAJ>io6p6=aeJq#Hj@;axU|gD;(vKVQWBVvl&+oV|ZXKASvAUl8@Mf2QUN?hSbm`|*8zdk=R6 zZS?wc@1E%9Fn(J_#JTLKF2n6?Hc~I2*10VS6;R_vA++TUPg<Ms zpBTwvbK;Cf3TIwvMLtkrcs!YZqnKlepSd$EFfc@;vsT5haK=s!OetqQF9xxP@opy> zT^6b3dF)^UJ<$-3|4y#CWH%1kHsw=`7*0W-sq$ov>M^~Hb&yIZ4mgs*5vKo?OB6|k@#y~2x0(`!M8@3b6jdx;^Um@^;Y1Yi9p7nE3M4JBfGchc&n~9L3kqTM{;ds3!*XS+W1% z-0^HPG8t;lyG&hrrDHYV^0ef+kQ_6xXVD{ib1C~NtLZ$-XqQD8-?8V1SI~5N!He@> zH7bjh@K)BH6?nAYAw*$HG|v68OQ9%JKBrx7fZc?AQ6; zjiju@yg`l1!5#wVhe?js%Qpp7%M!8Hr1QmM{f^>_s$c0{BQx);MG5+tMpi+2B!=Ir zbo;9mIK0Hzlw1c zli=6TescweJ5M6zwQ<{e*Qe2}X?YcE?AEQEOrWe(o8`Z|%aKdoJfkOx?huPWo$)58 z4R6)k>Q0uXsCld_tdjcAoE)xWBx6E*TxoLnr}bC@TLDA5+Jj0fy%t-JeW?!qogN~` zU??p{tjf!~=Z z?#62`ii&5i(^z=6k1Bk1-s`~Fu=A&Ngu2OuM_aEnCn?b^h?o^mZdTT0GJY??C9;Ic zjq$SvsZX6Y5v^l=*_OA`9M|1udD-1?(5b8|HA7g~t>HyoIeW+p(zfE6`I&hjB=j13 z>~*G$LGgwmU9+ z&m#H#7(VAcrIq&Z2rH{EA|Wp#Z;n1EEn4oNK(i=hwg}~j?YrpE^7D~i8eED88fdSr zmqzhC@=&QNWb={E-o!V*Fq=g!n0xLs%itAAn(uCy$_%AsrBAv}VkJz59*X-8#;;av z(}g&nxmL zM;U=zUL{c1uxp3KRR>n{SBj?x%h}ZT@HaxTM+p9UpEYk7C{&c{esb`M=a6WjkX(j? zL&=ie15M>mPJS(vRfIS;x1H6SO6KYA#rel?FmJswp1ZkuYMkIJ_8}>my3OUS$Yl9c zg(A|yEg1KmbwxA*euxe@JXI~~{jSCM2e+|!c{nXtB`W3r>o-5q#Pfu}!?u@RN_Sr(%^>`=AT^25m_)9U?L2{Y@)483m zpYi77`xFtkZyiI0$dmnS8gwDMV%x5%FGP1NEZ>@Yr6b|HM}y0Z)w}}cYvlGsE8~{y zLTKO~d5ptQO`zwQXuRCO$DiZYMbz^~OLMDbgwuwk7ScCTOCiLE20Xy;xq^^qefzIV z1~`r3aMtZ7^40lPWTOUPPj%XEvX6n*-x`~pR2dA4(MPyX$W-M*lVQdA$<@?N<=SP*JmVPWvg!}*2sX$3m zFv-)UtB`DFAkMp5Ms3jtuQ<=`XHkVE_PlT4)+c8WA!SJT62MR|OV~Wa^HpI0HN>!& z`k>DC>Luw&F=m;$9Bd$KboVhko=4r9UanEue0c7F%9zJ#l3i0Rf;Zm=i+k^G5Su)T z%J4>oRUGcKu7ojN3IT5gM*5qly(=uvrXMpm`{EnOlwT|&9+E)(1tgCN4-W5&xD3b- zUdhUuldtSyVI1zJsW7{D_!Aw-@cBHB{m)`PuY*skB6#BhePe(f(|xp9=1GBFDl~{0 zEeixKTx$MnwQYhMSft@^1XWnA3R;vwQq4YuO;}jO`mjZ$?c;5YOY$CEGft(mbey|> zDaTUJJa-)L8)*|PR7>T1gE?lFpS2^2KHT?rV^Xr9d6O%(xaYFbk;s_)GZ}O1^p~c= z&00fBB`AGJ72h^*4CkkXa8#{vuG%{M14{upF{;Cjy z2R(+YPPe9yYgEp~PPoYLFRrJmnS5?BR#36PIVxM87X4Ky^j5h0#HPWjzU`>=pX)JH zp}?K3zAi#Rf?;?Y#gG@Xn;>in_HwbDdWx#)9Q;2!gTO)Fx;HjBgk2}n_tEj6?<@07 zIaNjsncI07hWN=C4GHmQS2x~K|GD)nieR2nOA7pt-W=#4`_g@J$dubda)K)zjm{-a zuRe7YpUCI`y_xTA>q7;#f!i9NzdrG<0Fm?lXVv3(O^-LH?FU}anXP3N94i1U&p}

s(FcNuP zWYATbhDN|R!0(`%d!aj{Y7^8L#IjqW)lsdM-x!eCkx%&E{s7C32mN=&TgBl!(F&0PK~_N&#&i0V8jtFr32)}Bs<%XfJ{ zQ-zvS-_ocH7@^!H{`ABdi}ONpRv7?aFtCWC^p4YrM#$+M&`nbnanj#UvK^M`@EQEg zGp%@FKri!j&bXjb$#0s$U{+mT;rpqq@}?gydaaX-gcDW?A#JOofaCi)*LR;DRNQ+Z zKZN66+ZQ?&*%CuV+cat8-EJA5)pw7E=I*nHXO%W>vOW$dQk_mcSUnGWk7jmA>}>#C z`LT{jV&M?tHx_!w4=`83Ts||Rk|7~S;`1ijw7kt$omHyy3hoe`Nkc@@urzj2;Ltz{ zI+FB5&-Xqx7NH`pc~zPfZw3H-+bm~3@|P>^?n<;`3B#Y(b}pR*dgfb_;u@co@owM< z5CnM(XC0GV8ZxYvxliV4S(b*o5h_k$ud7cRmRCCUHZYLE@`(zU;nTNBuk2BRLFs4l(KHH%=(b zZ0R@ZuEj5RrFLEUd4>y{ym(vV`WD9aVp3%pgY{ufI|1i)sd@}m*cIUKCU0cCIn2U3 znEv7F)-(EiGKKN~BI>Q9qUzrFVd)S-6hs;XL>ftH5TrY#OG;Wwaww&w2kGwa?oztD zLpq1<`0aT<-}S!phs(8eoipdmIeXvtbziZzo(cv}cc%4$HtrcG*)-+{wKQ!PlqK^H zTd7`h^(8E70pWx1$BeSOWy6!nL-A8xAfh3dBL4d`b~G5Ia&RB~9ybFCxyaaWvEY<~xi)2o&+kb9k_ zaOCNC(j%OH$R_AAvuz#&Ro_CsukeD75dkZ&9CED?m!x+Tzzv3hY%{9=9w)4=<_`B`W6ip zdSNj3*u}*Dd^=QsM+;+=c=;{vmN|vgGbsv>GXf131yl@?fi!~c@?!t2Om#hGY9sMy z5f6~o82);Pp1R}lY|{gr?45uE695Z}P*H*ikeRJ?m?(R-`YRX5d-0XCIVX0Mj6wjq z>VeFGxAF6N5ke0-}x@!^8AWs$OhzgZobz=1+3|n)y}rj>b1X_&}Uy_!1XX|-G#$*4XqZc zfLBY>cmg|^DwTn2`eXpVeSfg#jQN-+f3g75M?TZ_bcQ)R|5_zcQ)0d3b7s&){&WYj z2e6~AiShjYp!ra$daM6Z6EJ(=9PpRW-Q`dPp!p9kjVA&%l0vK zBea&Nw-HtwUEV4m1;qYZ!-4$jB|1^$H(2bcr9#i{^k0j z#r8o!0iLIca{3m|I2$|#KA?Yp{(@V*1KdItXuz2*)rZ-`Emh7Z88#(>k!RmJU2Edz zF7W6jfwG%@e%Zu}edYi{K`XJHfLjue{aIaoDahlnA?yE5*KB@+4N#YrErOYkfOGX; zuE?wVB>cspi&;!MaUyoWBE?u>Z3noeEc|qL+H0b+ZPujB^Dmgvx=XVYfCKvMumQ^r z=>*_>4-kpLi`~IU0RCzqC~D9P#`QGY+|6NEH!ut~LFld;5?JC}^hG zb+ke2M2PDS9Qoe1@d4#0#@{OcRWM1DeQA2jDQ65s(myyX=Y(?r3W;Z~2p zzWfKA5AAw-;4D|UEg}>Kl}n_WItb-T)C11u_4@FX2}f9(;VKXdSArO|!75R`)*ZVs z2aOsljMKXay>}4`m;`Q0K#yr|@oD$8ze#I>KSv~I5sqCMdQF9 zIPc##G&=3x6`zNU?0B;PuYWcKgGmcmV!I~X4oEsW|1I2&3ht4(h54HkspD(Ujy&Af z_THaqaf!%lWQeh}8oBl6L~t-HIMb_fls1TZHWd!sw7Q%l_p%u;D+K{WM) z%(Wvli*weV9j@)Fc#w5p9JbN`|>cc{XQ*J!Y(`T1e=4X}9V-kWd zj@W>j$dr}TT?h&y&0>Iqnq45mo z_ofr+^fXWE^PIueYI5mkTrzkpUp!~0MIdMIb>(l99jB;CfQV){NO-J(jxHQruvzC4 z_w9Jt?8hkUKd#_*?1-|_|CRa2--B69GO7aiD0FZ0f{j)ao9?czcStr81QroN%cC!v zKNyUlyRIFGkAa|y9CL@{IW;-`sME(dgN~uHF+n0H#n;98uE2c;p!e3|q-0w2O6E)o zX0pWtQDy^9sbOQ*x$R{!=0eapudE9%?J@&{sC~G5yMg^d!=T8$tVL^-4jjx`*#2-lBwc^3qJk02}mS@zM5}MbO$Bg>=(`$#&xxmzeVEQ+) z7dU8Q1PD{_yvXF2Vu^SKJhm+d+as~Z#(|9tNRNAFGNaI%(C!GV0vVoeDnNbi8xu$C zI%UFZv9U5A${}Pwv3^v4{?{e9O3d0##t3FyA|;xum6;%%x9@_qQLZ(@)qbl?`tQOt z+cPRB{qbA{hI$7S!4#k>I*qgG2Yx{2}GAu@0NLZZq=@4qxBCiH90jeFmFkdG(#A(d}fR&mUOrg{!ooBEfC@*FZKiwJj$-M_&S6g$7aqn8ebd?XIVEt84ipp&a% z^4wyU{>R-XlS7zEEjKI=wc8zrHppm?KlYrDMs18nTTVe&qsnxo##xL#_Lhy{@Ulk% z%iISh1+B&cLj1$Ii1+qhd(2Wg*-6In-0f#9`L(j_67gFLDUY=aDa|AN(~$tNFx3Su z8ApFE`I5vx8o4h5zj{^(hgE?6Z-m%cIw!TjM~q$pfwkk#D0paU58Rv=)X36c!Zi4G zjN(-Yt9ZbDp@8?2l_gl8^??Wh#IH{x?Xh)Q^LmjS`uLNlf3a~@4%$8B0E zM88^&IvjFj&E@0|6vpIl6SHh}K#8>m5^x2lgt>%O%N~KJ8<-kn_fm}wpxcE{MP&7U zry!MlyOBHiw^MIp`68Pc;MSmX|MJiMu4-YqjI)d%^NNg*4zP9cOM&6!Us`h1r$qcf zU5-~w6fEI;W$(VqrV=%#)I=Rbqx6@}^!C_lI{=4s1ULC=xtZbkMZhabw}jyHI&om{ zD`1w6fvC@Vrw zmixdDJZ8r#J{ei&m-U0HXW$xARCRX+VSl{cM>)A1Q+umQ0!G(Z`;VdH3G<~*Cr}J> zHQ)uEn4f$>n~#vQ)WQ24q|c28)gYEX&Y+5++)|7wz{TUj8s_Yrh`qu9XNqy5gK9eX za3oC-h0W=&HHWV5o$i_Q&(uiV*)F^QJl|k-|L#H9ev^X>#(DaG9NntkTyX5H>{Apm(hMJqNRGI(V<8c&!?&gho<9%#qfpnPZ<_ z@wqq3u`Pk6^-8S9M(az=HE8Nc};dDR5KjLEaZ-49OKJcvu+rr6b4Q$-VL9Zk( zw3h$QJI@%W?Le1C91lJN2fT>>4bZ^U*A4Rh&T&gISN;_c8+(;3Xm(>Cf(eN4arw)= zmG;@w2K6TgX$IWmrR$-9`hQw;mE33$Q++BMHMi6DDvtr|9mn|C0DHU;BOuk<9L z+TY9)$nTYbRx`)nR8lKS^wxJmkqzYg!VO~s(eHKSo~ObD19{=vB0T{WyC3-G&8mzDNB=O+DhJ36^%N+>2!Jn6W9 zMQ4e96@Go2*k1Q>PgFT^ud7DHy5-)ctO`UVV3EHc4KXf;7{>sYCM}P%l>Ep#uVMOl ze!DopRnhomd2pG#zed|sWt!z#BcIHrUZUS>9*ugzll5Z(-u3PXu|1x`4L_?uqP~3sr<$a6ur;|O!MQk!wtFW~cuZCrO zmYd1mRYrAHt&%YV*tLbjbL}+xysTES6(QW`@?R(0^Ik#!es4> z!}zUM4utDrT3@y*{tWaveK~{k*~*2o_L-@T_qPyh)K&nHC@ zO&KBPm@ZeXQf2X`H<7bS+Lv9f=KhO1g(lEEst_w%e#(LOM4^I?RlAX$xACU`W-Cl% z;C_4Nt%K1Zq+aUB4d+1KlHb#21EyyHag{E0*#A*GyW^eZ{EE49(|a4Zg_rdv^93;8 zn#mOvioDdj8M7H1;BCj<{M_uRm7a@ zjr&2q|0ok@?+=DgWJ7V_=;Qk8L+59@jJ{}w>DO48qtYj)&j#}DhPFT5WGY&Cug#8z z4LMPel8SW;J{fFTWXOi$=u35l>=b1Kn)3LFjV1|NkV%e|te1&+Z(l_JUhrrdq*9F4^eDh}~*D@ugYcJrf_zL-4RnM+J>+dDx!= za9#Xh1a`=%d+&1X{iOc(lD@vyO?2f><%sSmiJDPQj0_mMO#6qtN1B;2mvn;+s2=mV zi-1iW-HDZqpZKBXwAp>w=c^{e({JLSkEjuZ=QaG+OWYfb44AB4&LoD9)4y0#Je3}qT=w-_6jEtO?|!n)S!=kuxJ&iP7ysF+UwFGRF&6=t0knWf!g&c} z)S%a!Ws?EBEHY%}jRle1tD7SPW@#SJeyCA=#u)e%qMGSbd+O$DzXSNoD@ZEtN!~e6YbaV zscTU-A`RpHuPpM(<%uK@m$RP>dS$YRcjQuYg1R)<60kWz6af(QJp?AVy{z9OvhMb& zg1~%nLAPEWK&*yjGE;du#!0MVO>AbYr-8gc;mEB5pM;JOLI2CeD31gNPmJt)LhAv! zU*Gs=+Y?G$Yy5`=G9cM1?3i2pRy;hJo_Lz6Od3oZz67jNz3uB$+XFEMogn8CJ_v*l z%^Z;$WjgF>ij`L$4;E39UP=(79ue>?=>BSSPsfY>cZmHrSvg60p}DS+56{_I+Rphg zZ*?fvjLkiE{R?fIuFFG&enq034O3$!tcCq$m5WC%7p17n-5X_!e5K|l7gs+JxlRT* zV>;J(Q&IJ0!wo+~%<16YU(D@tF%j*VGQ$lj_dVJ-Z{FbGy%9rKvz}M(LFW}6%js0k zXe)bcQvwd;KMM?ro|6$5v(G8Eg-7*}+Wq?jovWtLpI&=A021O9@}|X%zdt@u5%PNY zx`n1Vj5@qm`qmcW3=Vpt3Wm!OmoRHYXu~rpRO%g-I=AzjyYp!w`@Jb$fi5rp4;1D$ z48M*gPlCCMiQ-eWxuIQhUNH>nHhG2_rHjBKHQVL~S3%iJc2h_5$qXOQfOp$Z`@>2D z0|R?RU0q$T4^6bqBt1Ra<*naspUik?DRBl9Le~j;pm*64?z7Fea7TIRf=R2lxx+dm zuosYIK-yzeE98y_uCIl=bFHB}!(=sdlCBP`zZRJFMD-qBKZlq5`?}>RT6Oq=v+yfi z9j$seTNoKjws_>H1o^%RACABxgc4A!bIT=8MCK_`6_RNJVqh`id+nOTDR2tiu+$GU zH2o8iTP%8c-jI7YHJatiU;oSS#~7SP!AS1d(!KXnBZ91Rk<{?p#DV+f=cP@> z&~y6lvgagkRNtM;;^DogkTUMxj)Z{3L;>grrH?y{g_#`IOPZe7PI3HZ#3X0Yb&%T> z$HO^&dUfkJ@oYI!c}f|QRe4J5v1FuYcw!rlHwPPyJO0eFxo>_oU&xEu-Ou^5L2hyk^;=aFg|jF#IRA}c&>d4ckYvV)KidVvH5mYQm}uplA% zz4zd5h^0`<#<3P8J7+ zR&H@}HtV3-;+Qbk3c_9KW*wp9uqX5I?##p;;J zUT?>zLTcRxRpnZFd^+u%I@!c>#VOmF65cXGw!DA`4y6&}GwSPyW!l}@6c7{2N+AWl z7+H5-yD^IlbX$e>%QC_<-+-&DLxj$4F`KK?37&Wk*AkFnstBci5C=(!)WazP>I8KU z&&2+%`!r&@ydHe|x^vS`u=GG6I+u8Gs5Zmy+Ww`Tl3*tqaD13w*fSo#0 z>X8=Y`)#$OV7DkQLAOE-*gH4m;nBybD~RA#K|JooZ8l2ZtyY*p>!Jd7%cn-LkK)-0 zjL-Yx|NAwSj+_tgXh5KHiPc=0@L0B7cJ~ViufrBotX^|KZ-_Znw#Mk4ThTIjl-F_4 zcawOXh8&Mt86WR2#gwxo3wY(ep^`FD@*PYTPUmzpoX@@XJl!mAa=(;cYWaKW?s_XP zv}btg4x4aLEO{H2784sAyE{7NB94z(I+9LPwc~Rk=W9g#vsAzJy>0FErnT9Uf2mM^ zO`E6R`LA<-C&E`a*a1<4&oSTv2Zps%y0qUHBNyHsiY6@Y0SbHZ%8v% zHspKf=vCNO`v~1as+Xs2qrQ{6^^S@XQ=1NKyD@_2R_Uk)tcLlzwblCzH98<_stA6i zl6^$HR=x#Nv5fJLmprH-*ZV#4U&bHNE;Zy69<;D!uZZc=_sy04GeO@B+MJlP#l7xt!n4Ak-HGPE>=CN zuYo(E6q&9><7Fu^feV)_-C-v;3p7&p+s*=AL}9Unm3T3Sy7SU&2(T!$&8a0A= zc!YTyBMXS(oq=Tm;A2b`Uym(ZOA5cikB4@g(@QrwYCAmX?qvrYZma+MRw8Cx zw#zM1n{sL1Om`;KMK4A2if{aEJeQDG$}fS&yb*B zpw2+sDIpKlZN%CA-*4FovUp9phwjI+?5w5Y+49NukT@m40h!bPbxj!s`jUYo5(T}W z)Ky*l@AeNj##%4NWHCV8ouSlHEs?-5B;N{t`!iMy<}aGrzDyX?X;G)1?aqpTJiE;~ zx%2&K{&Kgsn1F~a5L4$D9xJ;ZpU-yFYqYL5V zys%nS27EEzsc^a>g&#c4d?GG?tZru*O5@-b|G~42Y;S(vvUZ|U{e*fq{v8;Fmn*oP zlf7C=5GT_&A-1LmcRbSUDJgA^+h@}GIy#lEstSW8mmm@zD11-DF9N@9pl<>wU!=3? zLrp-GbGB<`L&f)VtK437TiZA!s9_wXRBGGpC2aC^zV?tacD+Y^#6Qaoe3>^muZo?h zq{6x=)&;bFzdlEU62Oh13Vw=~-=Kz(yiN7wj8-V<>6Cbk zitf*q3AZ^oYYg+lBL4i5R+>VO&ZCw^3Qy5zQa!jEje6Ff8W(qTx}8Du&3&IOP<=3I zb1_57e-Zo!ge9qfux2#_`;my{^_BEBH+uNJQP|rVH>?3)$ea8KVvy*O zZk(c#+^40(_T$fg4}G%!tp1`TDLo(%F&5X1{h!SP{=cPZU;GF_9q8~D;Y}?JXGo>j3EBYuJ_)Wi!W-h<)MJ%&^RO{n|daKbZZW}A9mVvEGYa^^k8wd|Ckb3Dz5xfOPxh}0;x0{z@OJ1xviQR;$ z=K@v_*!P6@b9X7{&R=F&#;Mwl4;T6LVTl1u`iWcJTby0D1BnM!J{yaTm-hQtsUQ(v zCy1D~=1&-@3VgZ1_}3i`d{PUrE#HWTq~*TFbek%fe`7uW_$E{QEQUTBm|1yll_N-r zSzeNVpzjX18Z&}FXLnn`u7#fU`sSQiVDt37zU)pgBLvqpggyMy)!H+TWnlnDIDVru z5YIveSyJzGb6YcfNE$X9SUd0vIBs(ZsciaieoyLReNPB8Z(=ckJnEXIKE9a$qe4kM zEVBSt}eE0IBMp!4u)Uwev5p0zccQ!yc#_sNv2YYv|M9@D$Pm7d&5a-km zZ=5i-+~X1Kyf-Y!Ug+ztu6|qsO+~4q+Bg9iFU3CwU}yVx{%rri!WnYxBINj0)`$$2 zt#6ccthv_Mlv{pfv&It=d+yk^MJ0w_x)t-zMXS}YvC3jxR~#G0q*lmEvGbIfU z->?TSJH_;Akr_jSzw!S#CL-8zsquIXi`P%g1)Je@%TDX%9u7HE_nzI&?ZLfu|7AOE zVmE&?uA=Zt+g7#HMVR%sSE-!g)9vWNtp~TW-*vABVs~1JYQSTqwP(V*#4R*0Z+YE3 zG?jTQFx6?mWAzX+{GtESdb)Lot$Ry=&#V8c=wP|Z0|#fm)%?2UxEbqLlmPk>l_gR8 zg<|@EkL#MxvG|jvJEFDoZ<48osh1(cnk_Ytq%o<-0?owZ+lk0s99_3Vg5idz_WsJ9 zf~D`15BRmbawnJ7`wvg=ySA)_EO`d7zGV#Cc6O1DRXA}IzTa2R|6E-}w&x7CAb5!n zHov|Q@A?g-qV8Gj`NS@lxNbhrwWF(0>!v>a-R zcvPnBXYS_Ak|sMddXdKU)rSRB-wPGBj`N;QFCqD`u5oI}4^zCxg$#yxuOFp1XSr3| z|F7DD-|_!G%)k~-xnT0^+wmN%Sg=A(7~O~cw-=whQ&6{waCU~toYjQ>w{gE2K3{Fk zJ=y(37o>RQF5tgC{&9XcUxj8`6d)>@@J%4oWuG;4^%3JgDtDlat%j{9Pu4 zvy_%{riBiL3QO_@d_%y%m;QnSqoK?cgLVt}_cY#ypAo~agzR4@+V%1kMSaicUBBl_!|rvxWyy~8J)|M2gz!5*!2baA+9y2>61;K@TbpK{~}no&n(9 zeRt(rzUO^6>HNE4hbUVQKWM2TQc`j=LFs6%+}*nJm2Q^wd5&A4Zz515_IryMZPf2j zSbl(P`eqF(3WIQ2L(Y_z5cN{Me$XcvFF#PZg=lumLujN=m-{Phh<$5t=p@m6Tm2*z6Pa_8Hg#|CWUA29GW@fhy~T$1ie9lRn((y0&Y z5MDG}8=+Fs&78~#5D-GKv``elt`LJ6P29UPzIFZ@^OxrVyACQ2X)a=v*!n^%Xh7w4 zZ+eThoxy36;QQOLX?XO}wL^a}>FxULjrIRF^p(|(6wwpM4OEOo&L5!u)9lz7Dp{t) zOyTFExISK!!06>eX&-BJhHZ2P;JEcj^nUy?*aYCZms86D0s&6BDPy zB+E0d&tGRKFyxUFM-Xb3ifHrnETy#OMdU!JhVleoIPZQ7QmrT>lYw6!|4)_(iGRa{ z497X4iqG4+I6>^b5X3s2pGjW4I8tZT{Cr783k;AA{B^`cD z9)2*9BbGmR0`pQ>Kk`{Ve{H`xTe-z%(;+*KU9FYE?^@tnKU--6Wlq-at*x)n(v>ut zyMFq)Z(uffW3}@!-wwPgu@?sklKD_S8r3{S3z9VJS_{quU{U)i`*fJeWWaVDsAkVk zz-9yQlPB_ZrRJ*58FA14QqGa+$8dMy7 z?0!qSF|4v>JPXaIKkYkuxCfBPR>U2bjsKn9NeB@EKvm$&RF(C8rr0k$rBEcJmM5n+ z^Yt(0)U6iI%<`4%L;Buvyq-6lJz3r;ZGBK$BPrIF2-!#!bo!~){UadcG-Y|c_oYm% zd~ud>P)Ca|Ro8djI+tj8sS#RPOa6KuFIilZZ7R3?nVBpxogbzrtMALE~inp_WFnNO*{~Qvf?@)qLQyyoOEYR zj~w!vbJl(Nk9)J*_lOFUU)MK$c^NG(y+rZ6kd}q!Tj5ZLs!+ySl#A&Hq^_R&)7S}j z089V&L28ipKQmvSITkW$Yb#%x$LQyF{H8E|A)VpRWVoEb3-JMk|I~&p{`{BDt^<&i zNKq{Z@6tU9X5?Ro?{a@hAO)oX^18s*LkTqus5ED!uWfo1WyN&e}tBIoND6ZU4` zWFgftW#E6GoGKU>W<6x^_)GNx4Pzrir6m?Eyf?8jmRc;1^+g9m)eh$0{#z&X;AuUn zmoj;EC45)z4i6O`F2~9^JO&B1-)14?ELP3jq)1Ckb;SWBr);EaA3uc`1O&dT6GbBi zjR_~I;!VGt^lR@KoQ@H{APx0CP%{5y`MZ*L%>!gdl z%qFh0KVo2**y$P%7k1LFPtr}xnfb}3YIS&}%HYffoVHhTCT?V!-}1d2a~%4mnqBp? zO66_U*_U}eSBLu;GJ8gyumR>_br{*5Lds){nqZTM`I1C=Ys-6Bsm1bSsU36Z>>e!! zK2^d)7Ub*vXtF~p&0TVB&Rz+QLTdi%YtlrSbuIi!%ZxOvX~*aSj+j(03+987uzVp` zE8&v>X!UyUc&`0dAtY!Cg+s1L<^}z`A1fkMhE z-Ix)P{V+Gx2(G~xTt&pc!KpxUc}b}7={HtHl;eMYBd1IC3tJ6UH$Ew1w>1D^{VgHhHa?QBN`Tvi(RlAUJHWoN2yy)i?w zBqPOM4J1 zC_?s|JsE68A?K=vrk@B$N`o&Ooc?GCT6Jf3Kfv>jG^|yw0EB8jmlfvrEM^|Rz0PCV ziy{4^-|tR|2gN$6mkf1YD=;Afj zkHI(lSIekZ30_8?d{wpuh(Eor^Ho|yb-3|Agj|;|46{DU5lw$5e9d+tgq(%D^5<;E z47St^Yi3U1EG~>eS+>1hic^AQEB{sAS|VF3s`>k6jhdny z#19wwj^-+0mSf&DZHcAQo};%|0B^^uaWq>3Af~hb=%5p-D_kFLi+vB=+G{lGEq?sO zA(yxAd?ADfeM(T0cTXlVGV2hdRqTv(;fxHOj>j}V-3n9LTgEja?bKFDf95k)XJ-^Y zwZ&dNOfmDpWY=SI>D?y6X$WLYLwUhCRxyjEUkmqw{?F zVeXvTee;FfXAwqUk)BQtCAl`-IbXYPBMYp*E?Do@XBBbMTOU|Fc_ur5771RNe>f;K zQ*AbGJuPO1Tz&dKJjX@DD7O;~CS^Ip!^H{XXObVmPG(4#GuirS39%WP`LQs0G$_7J9+k)gM3aS5Q}-v}$h>UH1}g4_po*yIcyRMt(q& zOqF&1`2AXSKFL|ysDfJ$kE8dCY)>@-2cvqw~^zWza5*$BlZ(kTsqoi@thZ;ibh5=ySR(5!N3wEAY1(Wgx-=~r>@LG3ppo&V=s$0tGvm88xf@^=ayLk8;r8^ zFyB29R4KG!s0UL5wUJJxZ|zGiR7U}A044}Bxd<6uLLgxz_6_bCAmT_%1l5RUsPikl zZ+H<&To9k#^Z5%~3UohM0;hYBn!%(HLN8!lN{8$FOU`o%74SV1kNBgZa4ARdH>_fo zk+2`FW*!gWEi#%)g1n0ift=Dj-mK+4YDi3c<*Qpvw>J!UYVwf+akS3g3l9v5T{vdK zypC2PRZFJCmYQ4{7-HpO{{+32_OihZ^ga*u^pS5r*vIy@ZBz@4QOl{ z&*xsGwKb%1=(W5xm|de_4C!_oS}-XvA}fQ~PVR<|r5oAO>AWj7zhlK9<_b|$BodL{ zR}nXbJ<1YT7SB1V-_Er?=&BjKV66RgSz--GlPns)w6@z?UZ(tLS?XLipH7eMGp$;% zjeM_>>uCv65_3uqel4@Kz>e2iO?KA}m76pWIG1K#UHB$$laj{!qsI&-o+e&aAbCcq z$?Y-OuLmyO22sWA&3s=*L3V#{i>@`CAgFcV?!7_6Z5RVA2joz$Xv4^h!e}+PHDb{P zwV7!zRE%Z5?{EpDWHVh#xObyI7O}KGe82mJAm!$;Q?T=~?~}jrwxW#*ZgahdoFKm) zM9H5*Bd&@LJc;CTt;@Onp0PNk$Q-F`B144c_wfb zrD1oJkAqtO>>tt1c~k$8Er^1>p5pb?yo5SJZv-U`m`*VxSq@LxL~hwQ9{ROO4(Rjk zQr>>6-0U5zcI&A}M>&;@o1g=$A^KS>dvdBS1}RIJ7`?gL@@L~X13O{&?m1uoN zDM@y1aSGQa{I=>~7FCwR7Q~L;(83xW`O(ESo_#^>dt>B=jT=|!%On&h>; zP@Xw>4~N-~wx+$QD+&vyEtt7wQ6fMrMuA02%)|ATP_&<)RnaWFL?I*JBQka93NE5i z4_6h3$2xU9w^#icZ2lAdZtl#EyL7f2j52E8+n?52`?&5T1e zez@yhu-=LS-q(}a2Pj4TznSU5h!$p9A9}7_*5!4*Cd6y6<*;Vqq$yTux20nHnOpZlNs4X9TT$p}t> zAKzy}v2Ds%z!c1W`@8(j1vyP37fjWM4asvo z_hIIc?X@|b)9(6b1bpEf{ejn0&YvR7e+g61X;;^JC}oJ93?ZN4Ah$;#1eh5&Q|1}Z z)WQP$ibU1E80nQ`PkP3%)Rr1h8~4S@53-N_!IgDFE3op4Vr?N~>vR)-&MKc22c<4| z>O%H>iY~6jYkE_hrp4{6Cw}fFL#INmMi`iP;kk7P5tzcV7MZhoYCQk3K()8nH_~7} z=grit-gpDzzU75)8KB21%2D09?iOJtXm)(B#u7(-F!e!now(`#&bnetS%~X}q;|CE z=R~r6cFANY$SQP2YTCQcxAwF(84jTqEOnnJLttwm$p_)C70o*a3c}N|jjix`0zl16 zLvbxdL;JjsW&(9*SPiLlH1?hlU;A5x8nDEf^~X&qnLhVL>j@KMMNp$;q6s8dH_`B< zM=$XxFhF4IeZC_61f!kDeQbC48xUXAVAfrKitM;n9eFvMl2?($t1C#Sp6+9zKK>oj zK&7ed(ALjVb`VkRe{pi;92xq#LY9e5=l3 z18UB*x1mBVSCaRhV6UVC6#I+np(9pAz%Gch{qp)Y;5e}&n$^%Fwj27Sl*Q}rI~;2U zQw>z{%Fkb=87Kf)Gn;!kJXUIJCLneZheSISFLGEGYi-<*PVWM?LhG1#CS(ytcH(

6zJy4MLZY9QDJ7g)Q~xMa?7R81@k8Lt?!vZs-mn112cV`ftPH%g0;B~i`=R> zHQf5!&qyZSglP7?O4wS z+<@?>W|vomq(YCFq;vfGUmu5;?lc4$=O?gaHEc@`KN{*bG&Vo~CbohrvI25w3LyxS z)wtfw8WiCveUBlD1Ruy!`I`F=8`#LcGQ4a}Zu@H5{(NRH_`lzB+tO^^yu6PK1>J4b zZ;zL+?-j3qPf~H-u6O5G?qRmf-WL^#BZTO_(lypACRh_te8y8%HR+=O!yn?rf1JFB9h9V=XXBsYZXIrQfbIIraA|Ff*&# z36$3^TkyKSS4cCCNI#H3evU)JpHpCxDbV{)>5WVZ|Li$l2BPwIDB*&pL~2yz-yp6} zNPO@1G3Qu9$OIDlJ~*Cf|Mj%%tkT0Z2!7puzq^B;l7V?Pq zwoq3-5gI=4d{`g77!2W$JZzr6lAHaAnjQIRuP0XrOmyHM{YqBDjyyBxFXTh46xY%H zm(BGDf{ssof}Lf0Vme%qGeHN!;ydm(?~SN0Q!N`48aSIr)O zTg*MZri^5JkeFM*bWks&%az~va?44hp&rg&gw3FZE;bJ9& z!ep}HuNe?ZmiYR|IZKf4T@l3K_;4})7_|#Vn6+=cAWRP^qKrU6|Jv|$vJ&@s3V`mk zxbwO~t+-^qTE4g?*+ko$ve$h*kB*>I6x>#{v?skOyQ3OD7)~a@b?~@W>~__~AC}0f zg0eJ1DScurb;jp|mQ0onfE0lDF{|VZ)k=qVHxO}s$-rEnELQoe&@Lvc-Jq&e4UC=36Wp z`Tn}ss{h~1%x$Ww-y6+dNOnSwO5JLY{u+yDM$UxuA7V407md|hdgW_2X77rjsFjZL zJTJgm%YbaJq|mQn2(VHS7a8TYjAd!k_Uh+BcfC=RP5`Ao?G<7R)q)bZfuJHH*d1tNIqLa*P|mZ`Xh^{{Xg4K5aTo?G)x)!rrH5p zL=E=u2ik?=-pl7lMJkQZzTYJ8Eh>I9EeoH&(`m30h{pA|sE8p9*U*>qEGl+G?rIo} z&aFzIwR(Ej45vC>YH<{jr#V^TAX=kKQm$y$TfD5c+HBQUd$tC#axi8P^f=S-lAd7e zOruBJlv@^X02}*FZvxGa zloul2E)^Dku_li(i!|!>oP2#kYBbr!`+HVfQS(#?gcVD*r@81Gcecx;*qoQ_>)l~K zp4=H| zCo}5uf3kKMVDWs~)Ai{sb=Z4ACK?5rZD!gE`SPTmxI=)jn-=T8xUQ6Bi9$OeYP z$UK&o2*FTJQe+%`Q2TqN5a1c%%jDi-mY~sqFw$r^gOq5pzrT#K6|h|Y``{(9ib4ao z8JyQPt$95ERpIrdN_`Eh3lz%dK1pUw$3Uxk8qc8K zMx^+E2V&_qDzx&7Elm;&!vpS~T3LW>6p0`pr(Qa{&`97=<;rU^Ho~&B#(|g(xeZtL zk#NNH<6<)U`zzfyYTK1~4M-N(3m4%G9`L~w)G$c|SAqu!4?(Y(3fy#a9F+(KDuxMg z#1YvzZtrd?cNOW}Yh)`R-GY-C*t~Q06BzJ8DE#iRkR?+Ra!TXPRvG$`flKro3JvqSm1NF|0J_fZq9kcdoxKk3Uh+_@H>9_2iMPRW(@R& zpKT|M2KGDwmFVQa*@PO5MJF?ty+>~rSGq?}9QP;Go0n=Jr70)U+{sX3{G+yb_K#C6myteIr*Yn+K*%USf zARo`@jzg)o@0vb5+1O(<7}0RDdB3vY>wq=H8M787n4RWkC7d$>+pCp2e!)3;w;IW8 zB(gIHNduEpNrg{hDZYfzvrp+OoL| z=X%!I9-hmD15(PtT>b^Kl?LJIeASa$_m5(DieMLG2C0ZDK2n$oL;7`$`1IV+@!VlR zN;TdAT%oZXT?O;SngE?fPZ?d}j3#`(@3NXr#-YEN)z#&Xf%m-oYB)I}kTI33Hv?m9sw@u>)MT?f!47i$NP9X-Mm&y`f*nLMHWojG;iqGlrFbeCV>_ltwVKncv_dO<~FvAYKb zo@adUmcEaRoq8vDrp`P7L@3Eg-zh3~_&qJ`BDC&+4@#e{5q63<@_vOOAM9HJZW>Um zl`P3b@|MabaY)GFz$vu{?=)+s1HVg-kOf)aB7a50hS@IGNMMujzs>HOIDe4++b7gF z>p(E+*d6wm?;m;z=UD(|v?`2eZRo^c=4}F6C1WDp_j+<7zj7M&$|da2pu+X|1gIt?k)=-u|i7A^?(~MEs$!B zs$6dtkkT4?wO7(qVNB*7D=eM*n_(F>33$%xbuItVU2D0Q$TvUUp0&(_16B$fi3cs# zReygBbKRBySafyRy7aps9eq&CDf*N=bh}+XK5+{eiepAa0xn0ACz~uRJPxr??b<_L zYtga)*VS1^RTXt>Uyza%1nEXVB&18aMHJ*5x|9^@lr8~LI;Fb~-Q7s1aOjejly3Oe z(f7Ugj&J|LV2smyuf5iqb3V^+a?CFL5%hdi_Wnn>ow1klqRa7=0vJTiXab>2HP)JZ ztp{k`loVd9@3fK@ZFhc@VT=iVd^9NVc%^;zu=X?hEma=_+DfDU(Q_WF%Zh{{fj841 znT8Ip4H~e|wV6|=shD;>!rG<}X6NnENREsL8lEIY?N)7kv_N;y72ckpGQ2&w5#WDE zPImO|D0Fl+l+CbV**&Z!==VWs4Yk2d44b~bUbFiXpnMKLj>mK<;g~J&-km*Ta9${1 z*pPDzb~JVgt*T+C*o^{tW@;ZOUo{(3UBg}Gj$b*lsGqU<&%!A42GahoH{<7)OCpe? zs+wdQ&&4wP7VQ?O>6%=o&RF#ghZ9H29S}Z67mSGS3KMha!VP!1V@Kg9*z2R;(M?+@ zOWC>&F1L6S3mmQY&$c)DcjHdh78gTBYWc+L_#}L5Bk5cSL?+}`dmKD>xcR#JWF zf(y?dbOQzo?v5a%{s|ERHZ4LR*>e(68&|;CeU@>FxkTC_gSnyVJa8{n=86umaUy{d zO+I1#DT7J|%2rc@GqpUBQNkbK`=d67@sR@`PC6zFn1$ZeJoLfNopIDIm^3S3P>8gK z&{lw=G(uHB7F3-rq(HWhkPw-VTRr8sno~=)?Mz-Pi{@2C`!IVcNA#VX&po4$ju${# zRH23du2GiIb^ao@t}y@JXByV7ck$mI_c9@>-ltM_=EC>IooOzG zT=tvs!)d?M_aj!=Ghqr=TAtT97GaEo!h)=Y_c047g#cfVF({f%^%b-t1fM>^liw!o zzE7m_Ru8Z&JW(HFHD3V5&q7Rlv1nNKdRwG^O&rp$-(Yf$4j?UO!OA;B9x1j5;AAd3 zV*6hRoy_!9Yz)J16Zw>0PS|6Qn0#_dIgrk{twNbY+)<_hbq^H=XqwT9j_yx6gm{`)dsO4wd$oo};Cj+y0V{ z@dK22A!gGhf%^h`;~z&}Ba*kg!G<`Yg0;2XWyIaDoM)unu1=GWcllpZhIRPkYrPZR@Ru4&w6wwb_a9mixO_oX>d~1 zs@%$17fidh8~VdhDCJJD^fFU`?G);U!dUF6&kiE~+mFN)jOKL%0Yv34~d{{VJd_ z8{|UsPCkp{4rx~_JI#P%wNUr+9NPEPr>YwTjy}Un`|mvDfaqiptFKFd|5C;vCPW1n z>{#oVn?AHu&dCm)-dW9~D*$0`jPxRd@+|Vz!_zAu2$8?@G0ZaF-XmagV%nZnqe8Cz%>RP7m$@=!(&2ip$t&&(C$Xx@>nRw(*UzuO%*gA@=Oxg_qSuRv? zy(>i6Tu>#pjVrU!RQs`GXUld!M+>D~X{dTh(o;9*qVib>ywj@&=RwK`WRv33gx5bp ziB#W9hZP5212y|$MZbD+;_FSXyd?}wJv74V(dPH@KYxrrPvM+v)YRQy!g}4xY*3^x ztX}0+{_D);ZLx)|V@7+j07|<#go6Y9v>3XdIVhVnH)1<df$d@z38#iifksVG!wZLYP>yU^e0IQAZx9@4SNxQ1YpN^=i% zOKq|-Evy#Hr5Tk-=pJ?9L|P`s^#Ie&EB|C}*O!wIcO=u#uNv#d7rV|3E=r3%$;b2u z_vbG9SApQ^_nR*t$O|E<5;W#_RH+v-vkGG-tbd?pZCtn%JZhF$Jn5w$I*|8URGQ3P zKwsvsDL?s*=+ZoX8Y|}Hxg_#$@H+xJD}rk$bs+Da^^ z?i#n660?2Jt0r!4(VgwOq};Fxq$U;>t^vqdj)~bX?^+W47IHlLoIHdtJ>J#hQhJ>% z;F=AiF9y0uAQAY>4KQlf5COmp`G5s=bVxns_~+b$p<82?Dj?GGZnFOk5sKg$Uh))~MO(AW4H>J3;VeLF1?`BQ%d-!GNXnS~p-vc^Z{(s}Tm&@5)}p z8$ttk6a4F_8naSxB%N1*UU^-<$m=BEHp=W%OnyZUmsXfEVms<1Q z=&ao3*i1?!g{Pz-4{!s(IC;Th(Q9mrv(3i&EW-4)AB=2}b1KfXfAC6X2Jnu%O2i`> zqbr{Fh1Hr!#Jn>?rhEPo%SQ&~txB=&MnGA?Hfc+=;Ycj+sbl|Ir~AtHXkZI-0a>|} zpMrPA94*&Q*I3=q=&I!(-wgTvBER@DB}u8XCg%L-hAdFG&WjS+%e%})vNqEHRS~N! z(ArCi8VQ^+TJw&wK27?ILt9pd8BYq5w$%ORyxE>oxwijGaTwQJ4@4Vzd(|0C`bK`~;@^Bjk>Jv013l(u6Ir291196X zte<64+>XD>9j1k#wH-gc;jebODMVe)waJ86n1(qy6==-*NfKjr)@sicexNK6r=B1x z=Xs9DTzmVb#JW+h%)Cjf8Ym0%hsnYzXEHQBc&%egb!|sKTYf$@3=cs&C$Ie@Inp2T z=U0b|DT6yE!a+s!-LNY-eQ!xyl4DDQk?=Oew;HTuPgqR@j`L5#uarM^`^O)+wbtiB z%Ce$aG^gItjxOwDjlFZ#v7t$`vOsUG{ykb$lyK&9q@HKowO{X$pJ6T-F7o~Xk~!qQ zR60SbW2e~ct^FNihI9}Es0yj}I~>KXtl#_6!@YD~?f%2nkz(uUqg;WZ>}QN$Yu&QE z@H&L^XKN~~l{Qombmc4;GWqo>0L$Ho^Vd|p@wU}XD4vnYFAdA2jz7WnWX;qYR7rlw z?Wa?iwQXMvpICifXxgX=7E<&SF+E3L$2*D#SsS(8!FM9GrGAD2$a=#mb&FjkyDs;P z6TcF@giXd8i(2SkQzpGp*#lKm-16T`eKF))(e7Hx5k9)_25uA^_LqXcMInbBM{^LsrG zb_~2Bmtc3}KQFt}j&&J*^>TtG4k*M1+Y7DHItHs4!GS4V?zY~YiSg0`N&jK=f_b`lnPUBFsIdH z{ZU%yC6aOxZv7ES90>zTxTp&Uu?gJD%uIBEu2921a#O|{(RIma@Sz^iQE ztrRN%PPM$N^LnH~Tx-_3*on%5@4{!~O`GSJ!x+wKjHML5)F|M1_&i3&VSPu&(Y?(1 zK591}{UfW5;v1Gyg9j4Svny;?zBGSGN#5qZ-rJrs-=Eu6zXgL3+KzHCE{M{G5Bc@u zjAVn`D1x>CE8ent4O7l$NIs@7$=2+t)!fu&3?8IL)YhTmczbMG-Gy-Z2ad+{eYm;` zqhyt3QVI4{u5v#L5_hJv5i9=${W`az$>IiQbWnwHf1VWsH89qOIkKf08yekZScD!q z0!GM*2K5w2X|Oj<9NRco`Mh)RHX2Ho#bU*-00A@)m>rAyl9(r_+FjdjgFMi{d>Yyp zcJ8qvG6$(DyzvX324=iG@B9iAGmGhll!K3gXh%LP!I`_(I(M}M;U{Zxi=q1ONWJ#v z)QyARdwp1_ES_Ja!1rh(31<*c#>HkHL91%mc*(e{v&9dPwHph zApvAbdNv124=w7hd`hOdhz=ENqXOho$Td$brwLinG1WgJfyS;9TxLA~X zB%cpH#tbnF>lU*?r|{2&f8iAKX3lvcHW#OiPT?xNdRN? z$+Y~uZ^s~UT?-qz&0lISvPx!wDF?Tc3Y{t>v$vQ(!XUhSQ%F(+ylXAnAJ|sTf-26nYaqXsO z`=ez8yG2qvxXK&?DIsRlV_PK7qlvE|LZk{+_?z)bPOwm>t{(;eIHc3?&5<9eeh<`M zkSJy(AA9;4>`v|(C7xK+4N@L|UvG*TXaG7Mb$)fi;=l54wc8#yazriCMzMEnh;0Yn zhSpr33|eZ_h2+(yw>|&RVC=P9h(W7Rn`?*Ymz%+q$`6D5L{y^g7F8_*EMjz1Ul;>z zRN>|J;PUZtrC8q29-4Y)eNjcRVxM%;wP!~*DTbJ2F3xIJFk{O;9KB{K4cS@*x^jD$ zxhEMVpg6S3BoIUvLg0nDM~LDprEM)xw-U;?R9vx3sZNNZY>bZZfMc#y<5eDfL5^fI zQcwHlp-mDNjOv^o9@OHua5hpvh55_O+x=Gt6!vhJIcog+%)^?A@Kzb4vU}9|7u~)p z8l|p80P@VS1Q7x2qiU=fON)^_zb%q?Zv?ut z=8gv8u**jL_+l*NevZeBOFW;K;9n?-@@)RsUiID2mIo`cSokjNbDf-_+FuyDkJRvTXohj4yB_9rh>)Y<#+RSvwwIo-)%~eJAYJRL$|z1JGjxE zjqA@ox`M2%fL-6Hsy6qX$KvdRqt(s*mAw*?l2@n;t+>k<(8JXr=*mvq(0+`FusSSY z0iU&lD}&#bLJ_&V3AufAzfEi#XO?33Ou&yO%h(5>l8xqwh%~d7)W<@BC(+xSKa?xL zqOM`L<=EQy%L{54&E*N>)YaKgSYr5dBy-&JCe6^a0WomfwA)fkq)l^o_pD)E8Evos zV37~n|53M+o9b4;hk2je{PTF>{L8BB5ZI5KE2jwF0+Zg~QM2$&r@@!5FYlz#n1^Cy z`KbE5fvWn^IDJ8hSZ$nGYA>`qEt+xS7^u_qLQSiLt5Ny%@P@$U+u{Dxw&(aq3*j7FLTOUX5A`@;n zu4VSuY@_vK6)XNGacO;OV6@HPuNU=@+oHu}WwsG0Fxl3ZrEMM6H9(OX;lzG@8p>7d zi%Dp6c|u);iDH(D0kv1g&~09J7V#If@2|ZdMuk3)|I7p%$_9lVl#Y?y0!;G!klqhU z&EvEXR$)Sh(M0N*c1b86#JFpI#_g<#{5?OW(mDCLcJ=c&)zFv_76sWDeQoAEVa$3| zKDi?UaasaP5rph1ltQPZoY9suz?>gUiau>BewSUY_PLfXErvHoGxNXELKG`DdZSG3 zR@N!2Jm_WrTr_7Xv_$M5u0n_{)l23fSDYl!(WHZI#H|$66XADi>;DCr-M!Xk>-(3Z z#HK<9qx=|l?Kf}|aU!i~kO*ol8rnWxI``cV0+mN+gQv9!QbaxH2aF!U4t!`pgNHiM z-<<^JSh=2UF?ZuyLy+h!!t6KZ_IrL%GAE`py`N*19iHB#Bu&1bPWe7_I)0mfhoeUS zmkCEK*7T?I*AN*wacpqIEq8@v53QKx&XzZ=bFeuTe&6b%KgHRhPbD`edpG}%y7Sg# z7**Gwa+kiIC5l9ONSu61Ht{$Q$u}{R`9AtF|4ypit79`2D6h$N$_`tu?afMGu9m*1 zzk!%J>93mEN(D+%&caD198Ut3KM2!iR)(0@FVA83RZ;p+B;S2VvyN6)Yn}Js75J9@ zD$Eym1*(Fr*Oq;RdyaF5PtwwaKMoN%%27A*sFkR_7IHSv-{T6SjQFmQhuUwZ;b^K< z+YGc`;P~)Wj($l%4h3;>XpwG>Op`31AhL@@%xD&>mF$erQT9Fkd0{4!P`!iFRi9>nI0O zte19Wz8Tujwb`hAjU7S@$MA#2khRZAgiCt1}`slls#traR!O&7|2 zZkb_EuYJBdrR}MmYVL7?TI@l_d#kX!p(|8{2p{uS9^glCgL(ze+x4e^NO6qOCn$|t zDGifoB7%57Z3;M=E_hR6T;Y)v?HNquLp75d{a^?iMX%~QtYJsY*`gRT8KatR7PrYQ zBi$j>CXDJlTqMYtO3XGE&XuWN|E_}x#-gZMVWQk093_ODgi(JpWBQq8i7fc{d{GxU zn>nGq$@GmJEZ0O4BLS#|TPK}=A*TcPm-&kYxnI-X!)ctKwDT~bKwMfC4C47edQOBW zqa>H0gq8oRHJrxtg{lv0P#=r*tcdI{Qh%dL)DOqF*5Y`n?;0D&QSV`9$NF0uby!T* z>JsUUeiTBC9JB%M^?Bw^Ln_Y3cPqHO96-BG_%!KF)m5!$hf_S4V>8_Uek}UclUSnU z>nRhR3};Pu>9R;9cohu)6Dc08hWR-a{it)nrG;lKx}$Pke%_jZRhLm9_AY2=h)aTE!=03 zjpiqeRY_=a8&-Eo+%`I0oTMA7{R1!Mx|g#Q9_8Ev{Fss_89efvjCNq$_+v7@mO5!U-Tx%53+&)zbm3OO%Re5IlGos3nkHdk&*a1A*z zzo=mwj;k%cxOKBHnMl1~ZQN83g@^q3o#WDt^L8K%sU;A|+6x>5$p83w%=<6uhnTN5 z^(V!)K(Cde@k@(sR0-hoH^`F|Q!S$r^a*PcMPaQhHDCwkcT;u{RyDr=p{MjgbEra- zQa2UKxz0uOc0c7gxl}-?dM|F3xmxu4;MSp5(&jU76keS;w9i1=)&`yBUljwZH`Jd6 z;#@k&dc;9!7Kpo-zem8KS+9mD!B7gH#4rvDDcnlNK={aKXg`dX1_*sXm+3Jgw_7 zk6*b9@&hbMOM5I6VVokc#|;fu%GoHyI?Os5>c7dVwDL|F`k`iN*{QxJ|?{W z2;nG+&DRaodt2e8cvAD+nqF4v)UoXgOrFxq9LQ_*N?&}Z>i=x%_pD`4^V+bynTYT{pP&YRK=SY zvop_k9rw=SYWOBmykoC6lX{%HB#8M>8j~*MNv8`oNcwTxB((|N>n728MZSUM;BG>J zAS-2r#&@}(cE5-G2DteKXI`Z{F4s{?9;2j$xYoS^R0#r#Z(&rz`LU0@Is}~MxMEuV zq&xxRlgs!O6DroK3S3b!Ha|18qpR%PC!fNHCj80VoYg{Q%2~b4#r?I11L|c2yyHr9 z6Kv#{2{qo5@z~JH(OBl0OTHl`P-jq!|JdXDL&>Pr!VU#~xYGSClf8KaX(;^NO4IPS zA%o;4p3fgfQ82+Q>3@TB#W3+~qC|y}dnoR?nRt!3K5_9u5nXuL5TBv|#VW!6hFNi1H+d@~xC6GJopmZKd$wNNqpZ6)~`n z7ejl^1w(#EY(8WRj+@cfMO6}mU)ptpw5rkYg47mopZn9d8!-&MEmc?IHi@9dYvojF zF|UMo5e5dYZFo&>gq#ls>ON~ccODB;-phv?%Es^EUrB#vxQOsJ0w&pXy=7San83#_ zGg86pNt#*Q2fhYakBi?n^}h2|DJ^B4qj{9(G23NcACZ~+$&{GREJ!bfEsoT=T}5MS zwjs(KW2St`d*9+`Z8btrazj~2{VEJLZbm42Rx^DZJ@m_MrrIiNrdApW-G`iz#wPGl zNDhZK7nB9cnx8Y@w2_uzEiTclpZXd$s6bf#r$*)EG3uTS79-5InziN`EEx3qe7(yL zOtpB&XfqLmz8N}LF{MA+@YGds z*Py<(Qb+B6SGwgBaX2?b!amNc`ex%=NvF^ZBwQJYT(ORXYg zj0?VoQLl^Dm4-Mhxer z_=be%>#J4b9Wi$3q)zulQWNMxvlNmg;XL(*drLY~lg&G4K?IcSm06L0Sv0w&k)uv9xOdjBsk#%61*;r8N@kVQ%GE?65zysv^tdDjes98b)3)&ZAr+zBI=79F>Jv`p&;im zFM)N;IyGbDsd>VAsCdJdyWT0`(qU?i3_aJV?%q+Sq6jx4k4zP7H;vHZLBj_&6(%L1jJijPo(I!i4)leE zAT&;g7GlnO&lFTwV=r-L3sQ==;YEi(tmT(oiFSuFDb*cX%^YZq9c2ANMJ*{ z?PTsp^(Q0p616f$6V()J(&hfOT=4514HKz8L~61ln0Q0ragn z2)EekT6opxh1^&1F3v}tWmr+}zeyI&)oXFTc=enVU5ST4wdnw;*$Gk^-V zYvFWAV$Irl&{*Wvti_g_)2=8ph@`s&|c zVZkOH@0c{4PY^7(yKq1-8+OEOAQOf2g<>)i&{~^!*)m6$c3a0TRvdI0!=L{$Pr%Jk z5t8VXcW(uVDf7H>87>)pIzz}6Q8D)Cz0$fgxsXFnwxKaby##>CHOn%Np-e{l6(JBv zVcVT5)BlT^Nc-`PSh)lUpwkx}S%F+3^7*?Lf}b{PuaAbkTnl+2lMA4j4*EN5yv=z9 z>%Pkd{s=X~DFsdU zxD+Djf{ccu^}>jPh@J(I$r|jh4+^^$lESp8L5TFeJ2c;QZ2;(7F|TRzK0JtzQ7_hu zGaJp2?<4Fz*5HQZ&I3OLLh1louPd|x)6X`5z4tD9-4*1PbLce+uqc08>%$RrzKx{% zIG9(gTRU<_z%A<_GT}uh6P%ZFcd?;gwNs3c_f^Ha91d5oC~{ce9xhnE<@+$9us2s% z;#-G-Ld>oPziI;Z@}9ffv$)&G)Cfiyh^rKQ+GC1lY6{;L<(IIW0A+XV{$Y%&rKu>8 zB;jq zzB`J24y8}wm`IFTWK!g=^ zC^(}LOf6QtdUdN!G$PrRkqs;my$B_eWkPh$@8!)h6O6}n263EHTJUBgpyFxt3znUY z4*lffqHypYb$ro*&_>eH(ukXGg!Cas8fkwFpXEygb;WhJ4_FJbw9h1d-sb8gSt1AjXeQ^^KGUR7JND!2~eAY{oj@yhQVgs zV+Vdi$rCiXzZ0WKC01>|hcnXa-5uLo!<*N=?nEL0twZ6LqzjxYnnkCduF-Q z6x7g)Mg7p%iQ1<0uY&)?_mWCS^0@*_2#TCP_c_H6$fGHR> z@sUK`3Z=YKHHMH4CDo<*nhDyQlwq#^+NTvJ7>;3yjYPm0&lKZWU|mHTb#2D*O$$wK z*eiv~U9&`$50FXOQf@~WA)qYLXye7F>(=9h&%5Em3@nppVon;0-`$p_04)Wp;$x*) z6?)hH*pP1!Mi#DHV9a&l}Q6*w<&{Rec!Dv$MLu98=MQnt?i9tK$*KQH2G@$%G>+6v)3M?9bcd} z5Dr|H`FuPFb(Gq1FPA$Xz+FWh&OhIN&Kr0GDUpL-cc4%711b6Vu{J-O+AON2vsU{h z373WMC6&cd2fC+vYMbHW9^AZnokJrXIR}lk188L6T%kaX!K%$A!`8t0rf0_yg_4`r zAT%_r)E%rJzkw2%tJ3qIOx_0-3In%1%-3hp{gHmyNd%qCTev90OvLqmORg*H%-a_k zLSK8MpB9n5ekhjmVTQr(*D}%mfH2CKGAhZ~bTxGlwxWZ)-#v|#7wZd+NF|&@$wQXm zwm~!#C)+j9Hw&@BTQjIwo~D=0@~(v*80Fc+ke-Ii(gXRFEn%n(nGZh-raEhKj!^3X zmeCz;|A@EwpG_n(p!n?B5;2koZcXhX{>tG88XFi?bgh$l?{EsXGXPdj0kLa${A5Hj zgT|0<7-$m7^MOu;f#r{OnWVxAS@HwYq12YbYSrwoPvGTRFM+rXG`wtb4cUQ;-}`hr zmrWq*F7SIgvnc`n{>*mgVwL}HozwhdUNJ=02m`-;+gaL}%ZV>UD&}VG*@T-Z&)!^> zKD40=@Q2Gwoj0U4@G!b&Oy;#D(_Tmg@UW;7^~*_1pZ*d^WZS?4O?fy)y7oaifRrWL zWt4Tl59|-YtkO!;kKe`$S*w~HqVqKZi88fh_*|Xan-Hn+olda20aTqreEXaxwrH4M zB@hV&^~v9-p9*@#i^9i-knurfWMlqhDYzq>V3=oY)e0xi(lGmbQ*rG-Y^_|4+nVWd zF9Zs%K5-t{(?IMG&9}_JG6DMF=uH%-u=Oe(t*dJ|SN9V2Mlz}mV_Z)+n^rcAb!`1B)=%6nla)6SMVGhZZEn6c3iQ&~G2!lcgac z&FhVd2Z0ml7X*0-Cnz^9(=D4jMGl&ZdsryVx0&3q&c`<@%1Fo-IXqb}X~Pec!2kfk zgL>~27WN1~04V>j{X!)SxgNjMwQm5-4Yg z;&d0ScMwbI<{vZ(T0fPw*`8AI;kll#igA3|(Ans7cKWF=79vLFq{@37y09;Sor zah7+ayC{3p_C3OWW99YL_~CYFnqdH}`4$=-V&1nsF%SXRVZHT>HL#Qf&f(j495kEy z7$fN-az+m9ATu-LxS2hp#sf2%wIeigLcQt$!x<8DsbZLez&{uPwbrFBcj3pvN07wo znnfDr1W>JX?ZNhutu>ipsz0M(op;49cf}?OiOloH#AdJ2w^M7C{ibtP0*YZL@>gQa*YJ>A%tQwdvAEUTi%wAT+2->?s zd5I~(b$8kt7d(l-zG$xM(~?4?_Vvr7`sPhkPLHp{H51Q6BOm01ffULl%s2Hb zS61!YhLUP=q<&&{9RU#|9C+`%&kejq!kCWkjcHQjHEw;ZL!11=+n&h zea;?UJHYF3np=#5Bv}d!&r+n!bqr35LtwL@4TV|>Iu2>>%+%#vDaAi~Xm0*H2N*Yy z69@u4u1=GK_+yUmS-!1E{!7+_Ger;@V)($-*)H8jwZX_SrkVn|-GrFqXcIH_@si+= z^;Wtw2!v*m=vSh_k{ncvS@n_~^ubgSZgjz*FFkj`F+e^~tZ@9l0N;pU6M zF68ILpu$TQAJUW2V0Tgv!wfNBhLDB7LwJRzzrw-6(MmZ}#m=XkkkklSXb$BIF~qiuhwZ z-oYGD z+U1tSyP!sP#F%Q!85Ax;_R^SWlj^?M$|gI4mwGx@y+*?&D=2@DlQ*-FyU47s86MA_ z9R-+?km(#?RjIkez8$K4zzt<&aHV~dGBB4Z=&I9QuMe=FuTsl;u7pRUC+A%Hp%wRr zNVc&>+`uI5p?CS>=Jp2tWQQ7TkS*q6=@#>i8q4vR2+ALzE(kn4?@tkl>MO5^t1=Nk zv4GTh04&UwJSsF}S3)>)(}P^12(Zbusg=)?~xY&sV6>0P`>gI#WB~O+#$*D zLh{ta05Vzqd8@Zbi%+@4d1tN!_SuUaV>yN$3PHU^pz)Ow93w&9sen}yd&L7~C;Z3x zpHah`r!5lV#L&tRZ~KU4IDF9zxP6U!bTkkwS2!#dI;N!-QLNnj8Nouwr4nKXTx8JR z)GhDgnNDHC%$pI~hp?+|c-!miWIYkDw@1OPxd-u~$(OJJ`D2RE*XARhPfjUY>!}4Z zYB0U&o@)UMWz#hrGQ21|rgv(PDr>?N+h5(J3c^L76<(CA#m8Vfi~PFs-+8IUB0IF-l7HROm*lsh}Y@+)eGZD zMl<_CGdyFkl1K7eza}buVY^&;8@;&J7Y|(@rp%q7MIKX(D^itM0*2F`l;mgGG8LCB zOKu30bt$$+w<;q}OmmZP{;OF8JUzJqH!t|-nqsa(Kl z{_}RCnE(oy$MQ1|_gYZcGJWak%jg~$Jq6@wvEnQo3iWbOa z)_1!Dc;gQgrB~Y@&D$o8dX{+o*E#1IaC%+uVD$vV@ljHNVcTV3pO`^J`72E|9)Rle zhfL~ENV0_TB}{oYH*?emzryCbhjEZjG%ModffzLV)$jKRh`r$9YOm@E1-&Z|xFP?u zFNkPKvrhz0k)F>%70`xA=^>gc+wGhmF!Vige(U~>9|ce*-pwAf_F4%TC08ZwMO`}6 z8NzSb>>-q|*9=yzI;J-R8}4&S#~e=P|LZ8U(kyoVk3m$o3{F3-@kd@s(`F|fHX#VPLG)P7?epEQEIL`g|GZ23~ zNcCmJxuh<6U|Xj!IQbfYhW_ms@b3v8R^U+r^GO&30yG;_z_$gGZ~r+~LlIpD&L319 zE4RAyBJ~1Qt2}V-L6rZENW|}(I|5Moa+6&XglCxQ(Q{hSX16~oE&q19i0@qfw#t>8 zD0&WP)jPwe<-$%{N*Mn$1QD;lUI$$%?0^R=U$@|W8VbD+>Hk_ZjnwNfaaNbJ3+xQ` zwGbH~D)Q$8|A7@F-daCLNuLIxEC7B2bgkzEfVTeMZ!WEdm=deDs}Y&xXw;*w?J^&~ z|Kz_AFFkZZuvOK|(my{~c>;2*F(jNBznziM3oH^5E|h;UJK~9#^&up@nt*`9o+)>V zb8NuieuC5s-dFzDduX<5oaf)(0DsTq=wt!NwsAQ*h&_e;5f)*z@i>82rvfVLR!6F+ zy#zccRmkPLbSQ;F%fH11@l}0(7Yv67gS=g#2A}R%;6y^lrHVR?A?pRQ`jfwK^uIrd z@cUFimICO7V+3SXKY=6!Ze|+vv&y)wJ?8L${^|+=I{tfLzS=RK?HbZW&+6vzgT@+F z7m^xphNna_I%3|i|GVCQfx{d5_ACoT>^d3%^!;A^XDBg;Uf=h{oZjgCBlsWC(Iv|+ z`{x$#|GdD)Z?yrSqk0cxw$~<#8W7fUE~}k;H+vgDyt5u{4CzNkWai}IK+@eGAm*BX z2lJ+|{b|My0A4ul&2R*UU8i@+1d~W0+6Rd76+R7m!~|Ip^@{%i9&z7gbmGi1Kesboq=00|L{sxedc z2mM!I{ndM8sgE(1>0U24+YDM4E z19yZ$zGd8cYRJLnQ)ah7wfk?#U&cdui9rO5-Ab1w5lJ<6_%c3UuwWWF$Q$JAmn1|# z)rENVH6y_KfD5g2+D;{C$3QAno7)*osvA~TqSTOb2rR|fffOId>LXT%7q>X6g0M%qFVx64$}?1!(@fPwR5~#Unt1LriCgvTHi% zNYeXeX{Jj1aMLRcxFrYxTp6?m0su);pGWHGTt?;-1=>)rzOoNv#I@K0NWKomxHAIG zqSf-KP7G=To9r7nZ5^P=O~IONwta3aJGa0#ui3_g^Nn(28qaN6EI{A&UjSnt_{<18 znP@h0VuU5@m}C**?CVX)1Qg6-03iICsVyvGqz{q-tE;J|;0Fr4(-UZr1KPR$wzInX zpki;&2&zDbOKtRk;OScofTVyk39O29m0q#J@+) zw@QovP=nF-M(ivC*zw)aM+cD{w#muy+XK~)Vp-#lSe@r;0X(UO%5ETeg#knh$|<7u z5o=T|(O~1(wq~~nFf^~PZDsKUWBHUYp8_ucRr!8zRBpM=sCgC606%5$E6?qd$81^< z1a9Lt+R)7AYQbZY|N3W+LjI6fF$KtMiMpSs1@3$!coLZfoTGSx0hX7mz1$+J-}Twm zbQB(n6>D4GED7vw@sqJYCyyQNHk^sLjN~u);SFKM@gns1S^V6wu;W@MbuPThpBwat(KXq;*T)HpgzA)mGcBH+Ew2-JBBGydH|ks; z|K`>#wTWqTz~8|z6iNY2X`oLq1S$Z~x^6$i{cD27THoywR{_TK+m@0=77o4lf{Tb0 z|G3>WJI4^Jq3nrcT}xeJ;uox!{fqY(!VA^75oM$C;ES1Rzo7+iFz&m>dANmWnhuQRf9F2|Int96KWF^Q=3p zoh)PP)0CIC%>fH?xy8jkm<#Q5odxH`PC4ZYL1kNJWysn^+zIto%eV3xxb zdJ`%s*eF}{*j&uUc&M=KZC={Z&HZhh?w9JNsa2rX2nM;`Xz8y<@RJ~w$Gm;nvcAEb z#VUSe?ETSuonkiEHXCGZ^A?JJ4aN&;h@J&_M(lqs2;KjDx;aqhr!OK=p_Oygnte!Wibdderm`nw8YV9b3Dj~8uSN=pb zS#M-{`4Q4b=UV%*G(hhuf1mB9xyKY{4IT6n@MV1QH7*apXSd-Memoy^zyu$--|k%R z`)wIyj{u$)5se~BQ-MkLxz7_AE^01g?DGgJ0Ae4kR*1epoy;3!B&T~?YsVUZ<3b5H z`X>C)fC}K8!35;IYkRgpRC#89FsCPCL>(#p$6Yqb|E?&qyF%U$!F3r^hRP)TBh!QK}u3codV*b??`OGZR6hsHFKcX)&&f#U7 z_kyYp+>nACKp!E+tJm!{Zo_Threy)EQzXM8B2ENnqo+M@GOb9PICzhuV?LSR=yqqJ zto?(kY%TM%ODRWy=G{ouhLJH0fd+0$#yhod%cbANbA8AIlapylI+s@)CODCojz>Bs zw^{uf$}!VAZT@`7!b3}qfk&yrL#fld4&(>`1tQl9P7~9VO174&=fTH%8`~IE zo3p+9n)P!hKbWqS8@9~a2_X5_BC@(>gS=au7<`+bG7`P)O#?FaH!~8Ehu>oCt9LHpMNBqFqnV)D=RF0P*V4t3 H2EP9fC-k>^ literal 0 HcmV?d00001 diff --git a/samples/shared-heap/src/main.c b/samples/shared-heap/src/main.c index 1d93a6a2f7..111da33999 100644 --- a/samples/shared-heap/src/main.c +++ b/samples/shared-heap/src/main.c @@ -251,7 +251,7 @@ main(int argc, char **argv) heap_init_args.size = 65536; shared_heap = wasm_runtime_create_shared_heap(&heap_init_args); if (!shared_heap) { - printf("Create shared heap failed. error: %s\n", error_buf); + printf("Create shared heap failed.\n"); goto fail; } diff --git a/samples/shared-heap/src/shared_heap_chain.c b/samples/shared-heap/src/shared_heap_chain.c new file mode 100644 index 0000000000..beb17b137e --- /dev/null +++ b/samples/shared-heap/src/shared_heap_chain.c @@ -0,0 +1,321 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "wasm_export.h" +#include "bh_platform.h" +#include "bh_read_file.h" + +#define BUF_SIZE 4096 +static char preallocated_buf[BUF_SIZE]; + +static bool +produce_data(wasm_module_inst_t module_inst, wasm_exec_env_t exec_env, + bh_queue *queue, wasm_function_inst_t func, uint32 *argv, + uint32 buf_size, bool free_on_fail) +{ + uint8 *buf; + + wasm_runtime_call_wasm(exec_env, func, 2, argv); + + if (wasm_runtime_get_exception(module_inst)) { + printf("Failed to call function: %s\n", + wasm_runtime_get_exception(module_inst)); + return false; + } + if (free_on_fail && argv[0] == 0) { + printf("Failed to allocate memory from shared heap\n"); + return false; + } + + buf = wasm_runtime_addr_app_to_native(module_inst, argv[0]); + printf("wasm app1 send buf: %s\n\n", buf); + + /* Passes wasm address directly between wasm apps since memory in shared + * heap chain is viewed as single address space in wasm's perspective */ + buf = (uint8 *)(uint64)argv[0]; + if (!bh_post_msg(queue, 1, buf, buf_size)) { + printf("Failed to post message to queue\n"); + if (free_on_fail) + wasm_runtime_shared_heap_free(module_inst, argv[0]); + return false; + } + + return true; +} + +static void * +wasm_producer(wasm_module_inst_t module_inst, bh_queue *queue) +{ + wasm_exec_env_t exec_env; + wasm_function_inst_t my_shared_heap_malloc_func, my_shared_heap_free_func, + produce_str_func; + uint32 i, argv[2]; + + /* lookup wasm functions */ + if (!(my_shared_heap_malloc_func = wasm_runtime_lookup_function( + module_inst, "my_shared_heap_malloc")) + || !(my_shared_heap_free_func = wasm_runtime_lookup_function( + module_inst, "my_shared_heap_free")) + || !(produce_str_func = + wasm_runtime_lookup_function(module_inst, "produce_str"))) { + printf("Failed to lookup function.\n"); + } + + /* create exec env */ + if (!(exec_env = wasm_runtime_create_exec_env(module_inst, 32768))) { + printf("Failed to create exec env.\n"); + return NULL; + } + + /* allocate memory by calling my_shared_heap_malloc function and send it + to wasm app2 */ + for (i = 0; i < 8; i++) { + argv[0] = 1024 * (i + 1); + argv[1] = i + 1; + if (!produce_data(module_inst, exec_env, queue, + my_shared_heap_malloc_func, argv, 1024 * (i + 1), + true)) { + break; + } + } + + /* use pre-allocated shared heap memory by calling produce_str function and + send it to wasm app2, the pre-allocated shared heap is the last one in + chain, so its end address is calculated from UIN32_MAX */ + uint32 wasm_start_addr = UINT32_MAX - BUF_SIZE + 1; + for (i = 8; i < 16; i++) { + argv[0] = wasm_start_addr + 512 * (i - 8); + argv[1] = i + 1; + if (!produce_data(module_inst, exec_env, queue, produce_str_func, argv, + 512, false)) { + break; + } + } + + wasm_runtime_destroy_exec_env(exec_env); + + return NULL; +} + +static void +wasm_consumer(wasm_module_inst_t module_inst, bh_queue *queue) +{ + wasm_function_inst_t print_buf_func, consume_str_func; + wasm_exec_env_t exec_env; + uint32 argv[2], i; + bh_message_t msg; + char *buf; + + /* lookup wasm function */ + if (!(print_buf_func = + wasm_runtime_lookup_function(module_inst, "print_buf")) + || !(consume_str_func = + wasm_runtime_lookup_function(module_inst, "consume_str"))) { + printf("Failed to lookup function.\n"); + return; + } + + /* create exec env */ + if (!(exec_env = wasm_runtime_create_exec_env(module_inst, 32768))) { + printf("Failed to create exec env.\n"); + return; + } + + for (i = 0; i < 16; i++) { + msg = bh_get_msg(queue, BHT_WAIT_FOREVER); + if (!msg) + return; + buf = bh_message_payload(msg); + + /* call wasm function */ + argv[0] = (uint32)(uint64)buf; + if (i < 8) + wasm_runtime_call_wasm(exec_env, print_buf_func, 1, argv); + else + wasm_runtime_call_wasm(exec_env, consume_str_func, 1, argv); + + if (wasm_runtime_get_exception(module_inst)) { + printf( + "Failed to call 'print_buf' or 'consumer_str' function: %s\n", + wasm_runtime_get_exception(module_inst)); + } + + bh_free_msg(msg); + } + + wasm_runtime_destroy_exec_env(exec_env); +} + +static char global_heap_buf[512 * 1024]; + +int +main(int argc, char **argv) +{ + char *wasm_file1 = NULL, *wasm_file2 = NULL; + uint8 *wasm_file1_buf = NULL, *wasm_file2_buf = NULL; + uint32 wasm_file1_size, wasm_file2_size; + wasm_module_t wasm_module1 = NULL, wasm_module2 = NULL; + wasm_module_inst_t module_inst1 = NULL; + wasm_module_inst_t module_inst2 = NULL; + wasm_shared_heap_t shared_heap = NULL, shared_heap2 = NULL, + shared_heap_chain = NULL; + bh_queue *queue = NULL; + RuntimeInitArgs init_args; + SharedHeapInitArgs heap_init_args; + char error_buf[128] = { 0 }; + bool aot_mode = false; + int ret = -1; + + if (argc > 1 && !strcmp(argv[1], "--aot")) + aot_mode = true; + + if (!aot_mode) + printf("Test shared heap in interpreter mode\n\n"); + else + printf("Test shared heap in AOT mode\n\n"); + + memset(&init_args, 0, sizeof(RuntimeInitArgs)); + + init_args.mem_alloc_type = Alloc_With_Pool; + init_args.mem_alloc_option.pool.heap_buf = global_heap_buf; + init_args.mem_alloc_option.pool.heap_size = sizeof(global_heap_buf); + + /* init wasm runtime */ + if (!wasm_runtime_full_init(&init_args)) { + printf("Init runtime environment failed.\n"); + return -1; + } + + /* create queue */ + if (!(queue = bh_queue_create())) { + printf("Create queue failed.\n"); + goto fail; + } + + /* read wasm file */ + if (!aot_mode) + wasm_file1 = "./wasm-apps/test1.wasm"; + else + wasm_file1 = "./wasm-apps/test1.aot"; + if (!(wasm_file1_buf = + bh_read_file_to_buffer(wasm_file1, &wasm_file1_size))) { + printf("Open wasm file %s failed.\n", wasm_file1); + goto fail; + } + + /* load wasm file */ + wasm_module1 = wasm_runtime_load((uint8 *)wasm_file1_buf, wasm_file1_size, + error_buf, sizeof(error_buf)); + if (!wasm_module1) { + printf("Load wasm module failed. error: %s\n", error_buf); + goto fail; + } + + /* instantiate module */ + module_inst1 = wasm_runtime_instantiate(wasm_module1, 65536, 0, error_buf, + sizeof(error_buf)); + if (!module_inst1) { + printf("Instantiate wasm module failed. error: %s\n", error_buf); + goto fail; + } + + /* read wasm file */ + if (!aot_mode) + wasm_file2 = "./wasm-apps/test2.wasm"; + else + wasm_file2 = "./wasm-apps/test2.aot"; + if (!(wasm_file2_buf = + bh_read_file_to_buffer(wasm_file2, &wasm_file2_size))) { + printf("Open wasm file %s failed.\n", wasm_file1); + goto fail; + } + + /* load wasm file */ + wasm_module2 = wasm_runtime_load((uint8 *)wasm_file2_buf, wasm_file2_size, + error_buf, sizeof(error_buf)); + if (!wasm_module2) { + printf("Load wasm module failed. error: %s\n", error_buf); + goto fail; + } + + /* instantiate module */ + module_inst2 = wasm_runtime_instantiate(wasm_module2, 65536, 0, error_buf, + sizeof(error_buf)); + if (!module_inst2) { + printf("Instantiate wasm module failed. error: %s\n", error_buf); + goto fail; + } + + /* create shared heap */ + memset(&heap_init_args, 0, sizeof(heap_init_args)); + heap_init_args.size = 65536; + shared_heap = wasm_runtime_create_shared_heap(&heap_init_args); + if (!shared_heap) { + printf("Create shared heap failed.\n"); + goto fail; + } + + /* create a preallocated shared heap */ + memset(&heap_init_args, 0, sizeof(heap_init_args)); + heap_init_args.pre_allocated_addr = preallocated_buf; + heap_init_args.size = BUF_SIZE; + shared_heap2 = wasm_runtime_create_shared_heap(&heap_init_args); + if (!shared_heap2) { + printf("Create preallocated shared heap failed\n"); + goto fail; + } + + shared_heap_chain = + wasm_runtime_chain_shared_heaps(shared_heap, shared_heap2); + if (!shared_heap_chain) { + printf("Create shared heap chain failed\n"); + goto fail; + } + + /* attach module instance 1 to the shared heap */ + if (!wasm_runtime_attach_shared_heap(module_inst1, shared_heap_chain)) { + printf("Attach shared heap failed.\n"); + goto fail; + } + + /* attach module instance 2 to the shared heap */ + if (!wasm_runtime_attach_shared_heap(module_inst2, shared_heap_chain)) { + printf("Attach shared heap failed.\n"); + goto fail; + } + + /* wasm 1 produce shared data */ + wasm_producer(module_inst1, queue); + + /* wasm 2 consume shared data */ + wasm_consumer(module_inst2, queue); + ret = 0; + +fail: + if (module_inst2) + wasm_runtime_deinstantiate(module_inst2); + + if (module_inst1) + wasm_runtime_deinstantiate(module_inst1); + + if (wasm_module2) + wasm_runtime_unload(wasm_module2); + + if (wasm_module1) + wasm_runtime_unload(wasm_module1); + + if (wasm_file2_buf) + wasm_runtime_free(wasm_file2_buf); + + if (wasm_file1_buf) + wasm_runtime_free(wasm_file1_buf); + + if (queue) + bh_queue_destroy(queue); + + wasm_runtime_destroy(); + + return ret; +} diff --git a/samples/shared-heap/wasm-apps/CMakeLists.txt b/samples/shared-heap/wasm-apps/CMakeLists.txt index c0010af6a8..7bfa8cd48f 100644 --- a/samples/shared-heap/wasm-apps/CMakeLists.txt +++ b/samples/shared-heap/wasm-apps/CMakeLists.txt @@ -30,9 +30,7 @@ set (CMAKE_EXE_LINKER_FLAGS -Wl,--no-entry,--strip-all, \ -Wl,--export=__heap_base,--export=__data_end \ -Wl,--export=__wasm_call_ctors \ - -Wl,--export=my_shared_heap_malloc \ - -Wl,--export=my_shared_heap_free \ - -Wl,--export=print_buf \ + -Wl,--export-all \ -Wl,--allow-undefined" ) diff --git a/samples/shared-heap/wasm-apps/test1.c b/samples/shared-heap/wasm-apps/test1.c index c8fe0c7553..8ed1ca84cc 100644 --- a/samples/shared-heap/wasm-apps/test1.c +++ b/samples/shared-heap/wasm-apps/test1.c @@ -58,3 +58,10 @@ my_shared_heap_free(void *ptr) { shared_heap_free(ptr); } + +void * +produce_str(char *addr, uint32_t index) +{ + snprintf(addr, 512, "Data: %u stores to pre-allocated shared heap", index); + return addr; +} diff --git a/samples/shared-heap/wasm-apps/test2.c b/samples/shared-heap/wasm-apps/test2.c index b63efcd1a2..36ae748b47 100644 --- a/samples/shared-heap/wasm-apps/test2.c +++ b/samples/shared-heap/wasm-apps/test2.c @@ -4,8 +4,7 @@ */ #include - -#include +#include extern void shared_heap_free(void *ptr); @@ -16,3 +15,11 @@ print_buf(char *buf) printf("wasm app2's wasm func received buf: %s\n\n", buf); shared_heap_free(buf); } + +void +consume_str(char *buf) +{ + printf("wasm app2's wasm func received buf in pre-allocated shared buf: " + "%s\n\n", + buf); +} diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index 9f7a69229f..528ae673cc 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -22,10 +22,18 @@ set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) # Fetch Google test include (FetchContent) -FetchContent_Declare ( - googletest - URL https://github.com/google/googletest/archive/03597a01ee50ed33e9dfd640b249b4be3799d395.zip -) +if(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.24") + FetchContent_Declare ( + googletest + URL https://github.com/google/googletest/archive/03597a01ee50ed33e9dfd640b249b4be3799d395.zip + DOWNLOAD_EXTRACT_TIMESTAMP ON + ) +else() + FetchContent_Declare ( + googletest + URL https://github.com/google/googletest/archive/03597a01ee50ed33e9dfd640b249b4be3799d395.zip + ) +endif() FetchContent_MakeAvailable (googletest) SET(GOOGLETEST_INCLUDED 1) diff --git a/tests/unit/shared-heap/shared_heap_test.cc b/tests/unit/shared-heap/shared_heap_test.cc index 76357711db..626f0d2d5d 100644 --- a/tests/unit/shared-heap/shared_heap_test.cc +++ b/tests/unit/shared-heap/shared_heap_test.cc @@ -9,6 +9,8 @@ #include "bh_read_file.h" #include "wasm_runtime_common.h" +#include + class shared_heap_test : public testing::Test { protected: @@ -26,50 +28,50 @@ struct ret_env { char error_buf[128]; }; -struct ret_env -load_wasm(char *wasm_file_tested, unsigned int app_heap_size) +static void +destroy_module_env(struct ret_env module_env); + +static bool +load_wasm(char *wasm_file_tested, unsigned int app_heap_size, + ret_env &ret_module_env) { - std::string wasm_mem_page = wasm_file_tested; - const char *wasm_file = strdup(wasm_mem_page.c_str()); - wasm_module_inst_t wasm_module_inst = nullptr; - wasm_module_t wasm_module = nullptr; - wasm_exec_env_t exec_env = nullptr; - unsigned char *wasm_file_buf = nullptr; + const char *wasm_file = strdup(wasm_file_tested); unsigned int wasm_file_size = 0; unsigned int stack_size = 16 * 1024, heap_size = app_heap_size; char error_buf[128] = { 0 }; - struct ret_env ret_module_env; - memset(ret_module_env.error_buf, 0, 128); - wasm_file_buf = + ret_module_env.wasm_file_buf = (unsigned char *)bh_read_file_to_buffer(wasm_file, &wasm_file_size); - if (!wasm_file_buf) { + if (!ret_module_env.wasm_file_buf) { goto fail; } - wasm_module = wasm_runtime_load(wasm_file_buf, wasm_file_size, error_buf, - sizeof(error_buf)); - if (!wasm_module) { + ret_module_env.wasm_module = + wasm_runtime_load(ret_module_env.wasm_file_buf, wasm_file_size, + error_buf, sizeof(error_buf)); + if (!ret_module_env.wasm_module) { memcpy(ret_module_env.error_buf, error_buf, 128); goto fail; } - wasm_module_inst = wasm_runtime_instantiate( - wasm_module, stack_size, heap_size, error_buf, sizeof(error_buf)); - if (!wasm_module_inst) { + ret_module_env.wasm_module_inst = + wasm_runtime_instantiate(ret_module_env.wasm_module, stack_size, + heap_size, error_buf, sizeof(error_buf)); + if (!ret_module_env.wasm_module_inst) { memcpy(ret_module_env.error_buf, error_buf, 128); goto fail; } - exec_env = wasm_runtime_create_exec_env(wasm_module_inst, stack_size); + ret_module_env.exec_env = wasm_runtime_create_exec_env( + ret_module_env.wasm_module_inst, stack_size); + if (!ret_module_env.exec_env) { + goto fail; + } + return true; fail: - ret_module_env.exec_env = exec_env; - ret_module_env.wasm_module = wasm_module; - ret_module_env.wasm_module_inst = wasm_module_inst; - ret_module_env.wasm_file_buf = wasm_file_buf; - - return ret_module_env; + destroy_module_env(ret_module_env); + return false; } void @@ -99,38 +101,40 @@ test_shared_heap(WASMSharedHeap *shared_heap, const char *file, struct ret_env tmp_module_env; WASMFunctionInstanceCommon *func_test = nullptr; bool ret = false; - const char *exception = nullptr; - tmp_module_env = load_wasm((char *)file, 0); + if (!load_wasm((char *)file, 0, tmp_module_env)) { + ADD_FAILURE() << "Failed to load wasm file\n"; + goto fail0; + } if (!wasm_runtime_attach_shared_heap(tmp_module_env.wasm_module_inst, shared_heap)) { - printf("Failed to attach shared heap\n"); - goto test_failed; + ADD_FAILURE() << "Failed to attach shared heap\n"; + goto fail1; } + func_test = wasm_runtime_lookup_function(tmp_module_env.wasm_module_inst, func_name); if (!func_test) { - printf("\nFailed to wasm_runtime_lookup_function!\n"); - goto test_failed; + ADD_FAILURE() << "Failed to wasm_runtime_lookup_function!\n"; + goto fail2; } ret = wasm_runtime_call_wasm(tmp_module_env.exec_env, func_test, argc, argv); if (!ret) { - printf("\nFailed to wasm_runtime_call_wasm!\n"); const char *s = wasm_runtime_get_exception(tmp_module_env.wasm_module_inst); - printf("exception: %s\n", s); - goto test_failed; + ADD_FAILURE() << "Failed to wasm_runtime_call_wasm with " + << "exception: " << s; } +fail2: wasm_runtime_detach_shared_heap(tmp_module_env.wasm_module_inst); +fail1: destroy_module_env(tmp_module_env); +fail0: return; -test_failed: - destroy_module_env(tmp_module_env); - EXPECT_EQ(1, 0); } TEST_F(shared_heap_test, test_shared_heap_basic) @@ -143,16 +147,13 @@ TEST_F(shared_heap_test, test_shared_heap_basic) shared_heap = wasm_runtime_create_shared_heap(&args); if (!shared_heap) { - printf("Failed to create shared heap\n"); - EXPECT_EQ(1, 0); + FAIL() << "Failed to create shared heap"; } - // test wasm - test_shared_heap(shared_heap, "test.wasm", "test", 1, argv); + test_shared_heap(shared_heap, "test.wasm", "test", 0, argv); EXPECT_EQ(10, argv[0]); - // test aot - test_shared_heap(shared_heap, "test.aot", "test", 1, argv); + test_shared_heap(shared_heap, "test.aot", "test", 0, argv); EXPECT_EQ(10, argv[0]); } @@ -166,23 +167,169 @@ TEST_F(shared_heap_test, test_shared_heap_malloc_fail) shared_heap = wasm_runtime_create_shared_heap(&args); if (!shared_heap) { - printf("Failed to create shared heap\n"); - EXPECT_EQ(1, 0); + FAIL() << "Failed to create shared heap"; } - // test wasm - test_shared_heap(shared_heap, "test.wasm", "test_malloc_fail", 1, argv); + test_shared_heap(shared_heap, "test.wasm", "test_malloc_fail", 0, argv); EXPECT_EQ(1, argv[0]); - // test aot - test_shared_heap(shared_heap, "test.aot", "test_malloc_fail", 1, argv); + test_shared_heap(shared_heap, "test.aot", "test_malloc_fail", 0, argv); EXPECT_EQ(1, argv[0]); } +TEST_F(shared_heap_test, test_preallocated_shared_heap_malloc_fail) +{ + SharedHeapInitArgs args = { 0 }; + WASMSharedHeap *shared_heap = nullptr; + uint32 argv[1] = { 0 }, BUF_SIZE = os_getpagesize(); + uint8 preallocated_buf[BUF_SIZE]; + + /* create a preallocated shared heap */ + args.pre_allocated_addr = preallocated_buf; + args.size = BUF_SIZE; + shared_heap = wasm_runtime_create_shared_heap(&args); + if (!shared_heap) { + FAIL() << "Create preallocated shared heap failed.\n"; + } + + /* test wasm can't malloc with preallocated shared heap */ + argv[0] = 1024; + test_shared_heap(shared_heap, "test.wasm", "my_shared_heap_malloc", 1, + argv); + EXPECT_EQ(0, argv[0]); + + argv[0] = 1024; + test_shared_heap(shared_heap, "test.aot", "my_shared_heap_malloc", 1, argv); + EXPECT_EQ(0, argv[0]); +} + +TEST_F(shared_heap_test, test_shared_heap_chain_rmw) +{ + SharedHeapInitArgs args = { 0 }; + WASMSharedHeap *shared_heap = nullptr, *shared_heap2 = nullptr, + *shared_heap_chain = nullptr; + uint32 argv[2] = { 0 }, BUF_SIZE = os_getpagesize(); + uint8 preallocated_buf[BUF_SIZE] = { 0 }, + preallocated_buf2[BUF_SIZE] = { 0 }; + uint32 start1, end1, start2, end2; + + args.pre_allocated_addr = preallocated_buf; + args.size = BUF_SIZE; + shared_heap = wasm_runtime_create_shared_heap(&args); + if (!shared_heap) { + FAIL() << "Create preallocated shared heap failed.\n"; + } + + memset(&args, 0, sizeof(args)); + args.pre_allocated_addr = preallocated_buf2; + args.size = BUF_SIZE; + shared_heap2 = wasm_runtime_create_shared_heap(&args); + if (!shared_heap2) { + FAIL() << "Create preallocated shared heap failed.\n"; + } + + shared_heap_chain = + wasm_runtime_chain_shared_heaps(shared_heap, shared_heap2); + if (!shared_heap_chain) { + FAIL() << "Create shared heap chain failed.\n"; + } + + /* app addr for shared heap */ + start1 = 0xFFFFFFFF - 2 * BUF_SIZE + 1; + end1 = 0xFFFFFFFF - BUF_SIZE; + start2 = 0xFFFFFFFF - BUF_SIZE + 1; + end2 = 0xFFFFFFFF; + + /* shared heap 1 */ + argv[0] = end1; + argv[1] = 101; + test_shared_heap(shared_heap_chain, "test.wasm", "read_modify_write_8", 2, + argv); + EXPECT_EQ(0, argv[0]); + EXPECT_EQ(preallocated_buf[BUF_SIZE - 1], 101); + + /* shared heap 2 */ + argv[0] = start2; + argv[1] = 129; + test_shared_heap(shared_heap_chain, "test.wasm", "read_modify_write_8", 2, + argv); + EXPECT_EQ(0, argv[0]); + EXPECT_EQ(preallocated_buf2[0], 129); + + /* TODO: test aot when chain is supported in AOT */ + /* + argv[0] = start1; + argv[1] = 98; + test_shared_heap(shared_heap_chain, "test.aot", "read_modify_write_8", 2, + argv); + EXPECT_EQ(0, argv[0]); + EXPECT_EQ(preallocated_buf[0], 98); + + argv[0] = end2; + argv[1] = 81; + test_shared_heap(shared_heap_chain, "test.aot", "read_modify_write_8", 2, + argv); + EXPECT_EQ(0, argv[0]); + EXPECT_EQ(preallocated_buf2[BUF_SIZE - 1], 81); + */ +} + +TEST_F(shared_heap_test, test_shared_heap_chain_rmw_oob) +{ + SharedHeapInitArgs args = { 0 }; + WASMSharedHeap *shared_heap = nullptr, *shared_heap2 = nullptr, + *shared_heap_chain = nullptr; + uint32 argv[2] = { 0 }, BUF_SIZE = os_getpagesize(); + uint8 preallocated_buf[BUF_SIZE], preallocated_buf2[BUF_SIZE]; + uint32 start1, end1, start2, end2; + + args.pre_allocated_addr = preallocated_buf; + args.size = BUF_SIZE; + shared_heap = wasm_runtime_create_shared_heap(&args); + if (!shared_heap) { + FAIL() << "Create preallocated shared heap failed.\n"; + } + + memset(&args, 0, sizeof(args)); + args.pre_allocated_addr = preallocated_buf2; + args.size = BUF_SIZE; + shared_heap2 = wasm_runtime_create_shared_heap(&args); + if (!shared_heap2) { + FAIL() << "Create preallocated shared heap failed.\n"; + } + + shared_heap_chain = + wasm_runtime_chain_shared_heaps(shared_heap, shared_heap2); + if (!shared_heap_chain) { + FAIL() << "Create shared heap chain failed.\n"; + } + + /* app addr for shared heap */ + start1 = 0xFFFFFFFF - 2 * BUF_SIZE + 1; + end1 = 0xFFFFFFFF - BUF_SIZE; + start2 = 0xFFFFFFFF - BUF_SIZE + 1; + end2 = 0xFFFFFFFF; + + /* try to rmw an u16, first u8 is in the first shared heap and second u8 is + * in the second shared heap, will be seen as oob */ + argv[0] = end1; + argv[1] = 12025; + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, "test.wasm", + "read_modify_write_16", 2, argv), + "Exception: out of bounds memory access"); + + /* TODO: test aot when chain is supported in AOT */ + /*argv[0] = end1; + argv[1] = 12025; + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, "test.wasm", + "read_modify_write_16", 2, argv), + "Exception: out of bounds memory access");*/ +} + #ifndef native_function /* clang-format off */ #define native_function(func_name, signature) \ - { #func_name, (void *)glue_##func_name, signature, NULL } + { #func_name, (void *)glue_## func_name, signature, NULL } /* clang-format on */ #endif #ifndef nitems @@ -192,13 +339,13 @@ uintptr_t glue_test_addr_conv(wasm_exec_env_t env, uintptr_t addr) { wasm_module_inst_t module_inst = get_module_inst(env); - uintptr_t ret; void *native_addr = (void *)addr; uintptr_t app_addr = addr_native_to_app(native_addr); native_addr = addr_app_to_native(app_addr); if (native_addr != (void *)addr) { - EXPECT_EQ(1, 0); + ADD_FAILURE() << "address conversion incorrect"; + return 0; } return app_addr; } @@ -212,31 +359,370 @@ TEST_F(shared_heap_test, test_addr_conv) SharedHeapInitArgs args = { 0 }; WASMSharedHeap *shared_heap = nullptr; uint32 argv[1] = { 0 }; - struct ret_env tmp_module_env; - WASMFunctionInstanceCommon *func_test = nullptr; bool ret = false; - const char *exception = nullptr; - wasm_module_inst_t module_inst = tmp_module_env.wasm_module_inst; ret = wasm_native_register_natives("env", g_test_native_symbols, nitems(g_test_native_symbols)); if (!ret) { - EXPECT_EQ(1, 0); - return; + FAIL() << "Failed to register natives"; + } + + args.size = 1024; + shared_heap = wasm_runtime_create_shared_heap(&args); + if (!shared_heap) { + FAIL() << "Failed to create shared heap"; + } + + test_shared_heap(shared_heap, "test_addr_conv.wasm", "test", 0, argv); + EXPECT_EQ(1, argv[0]); + + test_shared_heap(shared_heap, "test_addr_conv.aot", "test", 0, argv); + EXPECT_EQ(1, argv[0]); +} + +TEST_F(shared_heap_test, test_addr_conv_pre_allocated_oob) +{ + SharedHeapInitArgs args = { 0 }; + WASMSharedHeap *shared_heap = nullptr; + uint32 argv[1] = { 0 }, BUF_SIZE = os_getpagesize(), + app_addr = 0xFFFFFFFF - BUF_SIZE; + uint8 preallocated_buf[BUF_SIZE]; + bool ret = false; + + /* create a preallocated shared heap */ + ret = wasm_native_register_natives("env", g_test_native_symbols, + nitems(g_test_native_symbols)); + if (!ret) { + FAIL() << "Failed to register natives"; + } + + args.pre_allocated_addr = preallocated_buf; + args.size = BUF_SIZE; + shared_heap = wasm_runtime_create_shared_heap(&args); + if (!shared_heap) { + FAIL() << "Failed to create shared heap"; + } + + argv[0] = app_addr; + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap, "test_addr_conv.wasm", + "test_preallocated", 1, argv), + "Exception: out of bounds memory access"); + + argv[0] = app_addr; + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap, "test_addr_conv.aot", + "test_preallocated", 1, argv), + "Exception: out of bounds memory access"); +} + +TEST_F(shared_heap_test, test_shared_heap_chain) +{ + SharedHeapInitArgs args = { 0 }; + WASMSharedHeap *shared_heap = nullptr, *shared_heap2 = nullptr, + *shared_heap_chain = nullptr; + uint32 argv[1] = { 0 }, BUF_SIZE = os_getpagesize(); + uint8 preallocated_buf[BUF_SIZE]; + bool ret = false; + + ret = wasm_native_register_natives("env", g_test_native_symbols, + nitems(g_test_native_symbols)); + if (!ret) { + FAIL() << "Failed to register natives"; + } + + args.size = 1024; + shared_heap = wasm_runtime_create_shared_heap(&args); + if (!shared_heap) { + FAIL() << "Failed to create shared heap"; + } + + /* create a preallocated shared heap */ + memset(&args, 0, sizeof(args)); + args.pre_allocated_addr = preallocated_buf; + args.size = BUF_SIZE; + shared_heap2 = wasm_runtime_create_shared_heap(&args); + if (!shared_heap2) { + FAIL() << "Create preallocated shared heap failed.\n"; + } + + shared_heap_chain = + wasm_runtime_chain_shared_heaps(shared_heap, shared_heap2); + if (!shared_heap_chain) { + FAIL() << "Create shared heap chain failed.\n"; + } + + test_shared_heap(shared_heap_chain, "test_addr_conv.wasm", "test", 0, argv); + EXPECT_EQ(1, argv[0]); + + /* TODO: test aot when chain is supported in AOT */ + /*test_shared_heap(shared_heap, "test_addr_conv.aot", "test", 1, argv); + EXPECT_EQ(1, argv[0]);*/ +} + +TEST_F(shared_heap_test, test_shared_heap_chain_create_fail) +{ + SharedHeapInitArgs args = { 0 }; + WASMSharedHeap *shared_heap = nullptr, *shared_heap2 = nullptr, + *shared_heap_chain = nullptr; + + args.size = 1024; + shared_heap = wasm_runtime_create_shared_heap(&args); + if (!shared_heap) { + FAIL() << "Failed to create shared heap"; + } + + args.size = 4096; + shared_heap2 = wasm_runtime_create_shared_heap(&args); + if (!shared_heap2) { + FAIL() << "Create preallocated shared heap failed.\n"; } + shared_heap_chain = + wasm_runtime_chain_shared_heaps(shared_heap, shared_heap2); + EXPECT_EQ(shared_heap_chain, nullptr); +} + +TEST_F(shared_heap_test, test_shared_heap_chain_create_fail2) +{ + SharedHeapInitArgs args = { 0 }; + WASMSharedHeap *shared_heap = nullptr, *shared_heap2 = nullptr, + *shared_heap_chain = nullptr; + uint32 argv[1] = { 0 }, BUF_SIZE = os_getpagesize(); + uint8 preallocated_buf[BUF_SIZE]; + struct ret_env tmp_module_env; + + args.size = 1024; + shared_heap = wasm_runtime_create_shared_heap(&args); + if (!shared_heap) { + FAIL() << "Failed to create shared heap"; + } + + memset(&args, 0, sizeof(args)); + args.pre_allocated_addr = preallocated_buf; + args.size = BUF_SIZE; + shared_heap2 = wasm_runtime_create_shared_heap(&args); + if (!shared_heap2) { + FAIL() << "Create preallocated shared heap failed.\n"; + } + + if (!load_wasm((char *)"test.wasm", 0, tmp_module_env)) { + FAIL() << "Failed to load wasm file\n"; + } + + if (!wasm_runtime_attach_shared_heap(tmp_module_env.wasm_module_inst, + shared_heap)) { + FAIL() << "Failed to attach shared heap\n"; + } + + /* can't create shared heap chain when shared heap is attached to a wasm + * app */ + shared_heap_chain = + wasm_runtime_chain_shared_heaps(shared_heap, shared_heap2); + EXPECT_EQ(shared_heap_chain, nullptr); + + wasm_runtime_detach_shared_heap(tmp_module_env.wasm_module_inst); + destroy_module_env(tmp_module_env); +} + +TEST_F(shared_heap_test, test_shared_heap_chain_create_fail3) +{ + SharedHeapInitArgs args = { 0 }; + WASMSharedHeap *shared_heap = nullptr, *shared_heap2 = nullptr, + *shared_heap3 = nullptr, *shared_heap_chain = nullptr; + uint32 argv[1] = { 0 }, BUF_SIZE = os_getpagesize(); + uint8 preallocated_buf[BUF_SIZE], preallocated_buf2[BUF_SIZE]; + + args.size = 1024; + shared_heap = wasm_runtime_create_shared_heap(&args); + if (!shared_heap) { + FAIL() << "Failed to create shared heap"; + } + + memset(&args, 0, sizeof(args)); + args.pre_allocated_addr = preallocated_buf; + args.size = BUF_SIZE; + shared_heap2 = wasm_runtime_create_shared_heap(&args); + if (!shared_heap2) { + FAIL() << "Create preallocated shared heap failed.\n"; + } + + shared_heap_chain = + wasm_runtime_chain_shared_heaps(shared_heap, shared_heap2); + if (!shared_heap_chain) { + FAIL() << "Create shared heap chain failed.\n"; + } + + memset(&args, 0, sizeof(args)); + args.pre_allocated_addr = preallocated_buf2; + args.size = BUF_SIZE; + shared_heap3 = wasm_runtime_create_shared_heap(&args); + if (!shared_heap3) { + FAIL() << "Failed to create shared heap"; + } + + /* The head and body can't be already in other shared heap chain as body */ + shared_heap_chain = + wasm_runtime_chain_shared_heaps(shared_heap3, shared_heap2); + EXPECT_EQ(shared_heap_chain, nullptr); + shared_heap_chain = + wasm_runtime_chain_shared_heaps(shared_heap2, shared_heap); + EXPECT_EQ(shared_heap_chain, nullptr); +} + +TEST_F(shared_heap_test, test_shared_heap_chain_unchain) +{ + SharedHeapInitArgs args = { 0 }; + WASMSharedHeap *shared_heap = nullptr, *shared_heap2 = nullptr, + *shared_heap3 = nullptr, *shared_heap_chain = nullptr; + uint32 argv[1] = { 0 }, BUF_SIZE = os_getpagesize(); + uint8 preallocated_buf[BUF_SIZE], preallocated_buf2[BUF_SIZE]; + args.size = 1024; shared_heap = wasm_runtime_create_shared_heap(&args); if (!shared_heap) { - printf("Failed to create shared heap\n"); - EXPECT_EQ(1, 0); + FAIL() << "Failed to create shared heap"; + } + + memset(&args, 0, sizeof(args)); + args.pre_allocated_addr = preallocated_buf; + args.size = BUF_SIZE; + shared_heap2 = wasm_runtime_create_shared_heap(&args); + if (!shared_heap2) { + FAIL() << "Create preallocated shared heap failed.\n"; + } + + shared_heap_chain = + wasm_runtime_chain_shared_heaps(shared_heap, shared_heap2); + if (!shared_heap_chain) { + FAIL() << "Create shared heap chain failed.\n"; + } + + memset(&args, 0, sizeof(args)); + args.pre_allocated_addr = preallocated_buf2; + args.size = BUF_SIZE; + shared_heap3 = wasm_runtime_create_shared_heap(&args); + if (!shared_heap3) { + FAIL() << "Failed to create shared heap"; } - // test wasm - test_shared_heap(shared_heap, "test_addr_conv.wasm", "test", 1, argv); + /* unchain shared heap so that the 'body' can be another chain 'body' + * again(1->2 to 1->3->2) */ + EXPECT_EQ(shared_heap2, + wasm_runtime_unchain_shared_heaps(shared_heap_chain, false)); + shared_heap_chain = + wasm_runtime_chain_shared_heaps(shared_heap3, shared_heap2); + EXPECT_EQ(shared_heap_chain, shared_heap3); + shared_heap_chain = + wasm_runtime_chain_shared_heaps(shared_heap, shared_heap3); + EXPECT_EQ(shared_heap, shared_heap_chain); + + /* break down the entire shared heap chain */ + EXPECT_EQ(shared_heap2, + wasm_runtime_unchain_shared_heaps(shared_heap_chain, true)); +} + +TEST_F(shared_heap_test, test_shared_heap_chain_addr_conv) +{ + SharedHeapInitArgs args = { 0 }; + WASMSharedHeap *shared_heap = nullptr, *shared_heap2 = nullptr, + *shared_heap_chain = nullptr; + uint32 argv[1] = { 0 }, BUF_SIZE = os_getpagesize(); + uint8 preallocated_buf[BUF_SIZE]; + bool ret = false; + + ret = wasm_native_register_natives("env", g_test_native_symbols, + nitems(g_test_native_symbols)); + if (!ret) { + FAIL() << "Failed to register natives"; + } + + args.size = 4096; + shared_heap = wasm_runtime_create_shared_heap(&args); + if (!shared_heap) { + FAIL() << "Failed to create shared heap"; + } + + /* create a preallocated shared heap */ + memset(&args, 0, sizeof(args)); + args.pre_allocated_addr = preallocated_buf; + args.size = BUF_SIZE; + shared_heap2 = wasm_runtime_create_shared_heap(&args); + if (!shared_heap2) { + FAIL() << "Create preallocated shared heap failed.\n"; + } + + shared_heap_chain = + wasm_runtime_chain_shared_heaps(shared_heap, shared_heap2); + if (!shared_heap_chain) { + FAIL() << "Create shared heap chain failed.\n"; + } + + argv[0] = 0xFFFFFFFF; + test_shared_heap(shared_heap_chain, "test_addr_conv.wasm", + "test_preallocated", 1, argv); + EXPECT_EQ(1, argv[0]); + + argv[0] = 0xFFFFF000; + test_shared_heap(shared_heap_chain, "test_addr_conv.wasm", + "test_preallocated", 1, argv); EXPECT_EQ(1, argv[0]); - // test aot + /* TODO: test aot when chain is supported in AOT */ + /*argv[0] = 0xFFFFFFFF; test_shared_heap(shared_heap, "test_addr_conv.aot", "test", 1, argv); EXPECT_EQ(1, argv[0]); + + argv[0] = 0xFFFFF000; + test_shared_heap(shared_heap, "test_addr_conv.aot", "test", 1, argv); + EXPECT_EQ(1, argv[0]); */ +} + +TEST_F(shared_heap_test, test_shared_heap_chain_addr_conv_oob) +{ + SharedHeapInitArgs args = { 0 }; + WASMSharedHeap *shared_heap = nullptr, *shared_heap2 = nullptr, + *shared_heap_chain = nullptr; + uint32 argv[1] = { 0 }, BUF_SIZE = os_getpagesize(); + uint8 preallocated_buf[BUF_SIZE]; + bool ret = false; + + ret = wasm_native_register_natives("env", g_test_native_symbols, + nitems(g_test_native_symbols)); + if (!ret) { + FAIL() << "Failed to register natives"; + } + + args.size = 4096; + shared_heap = wasm_runtime_create_shared_heap(&args); + if (!shared_heap) { + FAIL() << "Failed to create shared heap"; + } + + /* create a preallocated shared heap */ + memset(&args, 0, sizeof(args)); + args.pre_allocated_addr = preallocated_buf; + args.size = BUF_SIZE; + shared_heap2 = wasm_runtime_create_shared_heap(&args); + if (!shared_heap2) { + FAIL() << "Create preallocated shared heap failed.\n"; + } + + shared_heap_chain = + wasm_runtime_chain_shared_heaps(shared_heap, shared_heap2); + if (!shared_heap_chain) { + FAIL() << "Create shared heap chain failed.\n"; + } + + /* test wasm */ + argv[0] = 0xFFFFFFFF - BUF_SIZE - 4096; + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, + "test_addr_conv.wasm", + "test_preallocated", 1, argv), + "Exception: out of bounds memory access"); + + /* TODO: test aot when chain is supported in AOT */ + /*argv[0] = 0xFFFFFFFF - BUF_SIZE - 4096; + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, + "test_addr_conv.aot", + "test_preallocated", 1, argv), + "Exception: out of bounds memory access");*/ } diff --git a/tests/unit/shared-heap/wasm-apps/test.c b/tests/unit/shared-heap/wasm-apps/test.c index bd0df19c20..b0bfc17b36 100644 --- a/tests/unit/shared-heap/wasm-apps/test.c +++ b/tests/unit/shared-heap/wasm-apps/test.c @@ -32,3 +32,31 @@ test_malloc_fail() shared_heap_free(ptr); return 0; } + +void * +my_shared_heap_malloc(int size) +{ + return shared_heap_malloc(size); +} + +void +my_shared_heap_free(void *addr) +{ + shared_heap_free(addr); +} + +char +read_modify_write_8(char *addr, char value) +{ + char original_value = *addr; + *addr = value; + return original_value; +} + +short +read_modify_write_16(short *addr, short value) +{ + short original_value = *addr; + *addr = value; + return original_value; +} diff --git a/tests/unit/shared-heap/wasm-apps/test_addr_conv.c b/tests/unit/shared-heap/wasm-apps/test_addr_conv.c index f91764c84c..5e64526a04 100644 --- a/tests/unit/shared-heap/wasm-apps/test_addr_conv.c +++ b/tests/unit/shared-heap/wasm-apps/test_addr_conv.c @@ -30,3 +30,17 @@ test() shared_heap_free(ptr); return 1; } + +int +test_preallocated(void *app_addr) +{ + int *ptr = (int *)app_addr; + int *ptr2 = NULL; + + ptr2 = test_addr_conv(ptr); + if (ptr2 != ptr) { + return 0; + } + + return 1; +} From ccdc8369d679e9c35a46bbd8933c18d2a257d010 Mon Sep 17 00:00:00 2001 From: TianlongLiang <111852609+TianlongLiang@users.noreply.github.com> Date: Thu, 19 Jun 2025 21:42:53 +0800 Subject: [PATCH 19/20] Shared heap enhancement for AOT and update tests and samples (#4226) * shared heap enhancement: modify memory check for aot_check_memory_overflow to accomodate shared heap chain * shared heap enhancement in AOT * use alloca for func ctx shared heap cache value * use correct alloca for func ctx shared heap cache value * enable shared heap chain aot test and bug fix * Fix a missing argument on 32bits platform, still has the shared heap chain iteration problem * Fix shared heap chain iteration problem on 32bits platform * fix AOT bulk memory bounds checks compliation issue * fix AOT bulk memory bounds checks on 64 bits platform * refactor aot memory check * refactor AOT bulk memory bounds checks * add more unit test for shared heap * finished organizing unit test for shared heap and enable x86_32 for shared heap unit test * cover a corner case for bulk memory overflow check * try func call to replace shared heap chain traverse * fix compilation error in JIT and potentially load nullptr * add option for wamrc to enable single shared heap/multi shared heap, and update shared heap unit tests and sample * cr suggestions: 1. check potiential underflow 2. refactor and use separate function for bulk memory and normal memroy 3. static assert 4. add more comments 5. remove unused code --- core/iwasm/aot/aot_reloc.h | 8 + core/iwasm/aot/aot_runtime.c | 10 + core/iwasm/common/wasm_memory.c | 15 +- core/iwasm/common/wasm_runtime_common.c | 37 + core/iwasm/common/wasm_runtime_common.h | 10 + core/iwasm/compilation/aot_emit_memory.c | 751 ++++++++++++------ core/iwasm/compilation/aot_llvm.c | 173 ++-- core/iwasm/compilation/aot_llvm.h | 5 + core/iwasm/include/aot_comp_option.h | 1 + core/iwasm/interpreter/wasm_runtime.c | 1 + core/iwasm/interpreter/wasm_runtime.h | 8 +- samples/shared-heap/CMakeLists.txt | 22 +- samples/shared-heap/src/shared_heap_chain.c | 8 +- samples/shared-heap/wasm-apps/test1.c | 4 + samples/shared-heap/wasm-apps/test2.c | 9 +- tests/unit/CMakeLists.txt | 28 +- tests/unit/shared-heap/CMakeLists.txt | 12 +- tests/unit/shared-heap/shared_heap_test.cc | 449 +++++++++-- .../unit/shared-heap/wasm-apps/CMakeLists.txt | 103 ++- .../bulk-memory/test_bulk_memory.wasm | Bin 0 -> 63 bytes .../bulk-memory/test_bulk_memory.wat | 12 + .../wasm-apps/memory64/CMakeLists.txt | 68 ++ tests/unit/shared-heap/wasm-apps/test.c | 2 +- .../shared-heap/wasm-apps/test_addr_conv.c | 2 +- wamr-compiler/CMakeLists.txt | 2 + wamr-compiler/main.c | 14 +- 26 files changed, 1356 insertions(+), 398 deletions(-) create mode 100644 tests/unit/shared-heap/wasm-apps/bulk-memory/test_bulk_memory.wasm create mode 100644 tests/unit/shared-heap/wasm-apps/bulk-memory/test_bulk_memory.wat create mode 100644 tests/unit/shared-heap/wasm-apps/memory64/CMakeLists.txt diff --git a/core/iwasm/aot/aot_reloc.h b/core/iwasm/aot/aot_reloc.h index ca9ba17f1f..eb41e8c2b7 100644 --- a/core/iwasm/aot/aot_reloc.h +++ b/core/iwasm/aot/aot_reloc.h @@ -185,6 +185,13 @@ typedef struct { #define REG_STRINGREF_SYM() #endif +#if WASM_ENABLE_SHARED_HEAP != 0 +#define REG_SHARED_HEAP_SYM() \ + REG_SYM(wasm_runtime_update_last_used_shared_heap), +#else +#define REG_SHARED_HEAP_SYM() +#endif + #define REG_COMMON_SYMBOLS \ REG_SYM(aot_set_exception_with_id), \ REG_SYM(aot_invoke_native), \ @@ -218,6 +225,7 @@ typedef struct { REG_LLVM_PGO_SYM() \ REG_GC_SYM() \ REG_STRINGREF_SYM() \ + REG_SHARED_HEAP_SYM() \ #define CHECK_RELOC_OFFSET(data_size) do { \ if (!check_reloc_offset(target_section_size, \ diff --git a/core/iwasm/aot/aot_runtime.c b/core/iwasm/aot/aot_runtime.c index 341f64cd5a..c39cee6fdb 100644 --- a/core/iwasm/aot/aot_runtime.c +++ b/core/iwasm/aot/aot_runtime.c @@ -63,6 +63,14 @@ bh_static_assert(offsetof(AOTModuleInstanceExtra, shared_heap_start_off) == 16); bh_static_assert(offsetof(AOTModuleInstanceExtra, shared_heap_end_off) == 24); bh_static_assert(offsetof(AOTModuleInstanceExtra, shared_heap) == 32); +bh_static_assert(offsetof(WASMSharedHeap, next) == 0); +bh_static_assert(offsetof(WASMSharedHeap, chain_next) == 8); +bh_static_assert(offsetof(WASMSharedHeap, heap_handle) == 16); +bh_static_assert(offsetof(WASMSharedHeap, base_addr) == 24); +bh_static_assert(offsetof(WASMSharedHeap, size) == 32); +bh_static_assert(offsetof(WASMSharedHeap, start_off_mem64) == 40); +bh_static_assert(offsetof(WASMSharedHeap, start_off_mem32) == 48); + bh_static_assert(sizeof(CApiFuncImport) == sizeof(uintptr_t) * 3); bh_static_assert(sizeof(wasm_val_t) == 16); @@ -1991,6 +1999,8 @@ aot_instantiate(AOTModule *module, AOTModuleInstance *parent, #else extra->shared_heap_start_off.u32[0] = UINT32_MAX; #endif + /* After shared heap chain, will early stop if shared heap is NULL */ + extra->shared_heap = NULL; #if WASM_ENABLE_PERF_PROFILING != 0 total_size = sizeof(AOTFuncPerfProfInfo) diff --git a/core/iwasm/common/wasm_memory.c b/core/iwasm/common/wasm_memory.c index a67c4209ea..e962b13b18 100644 --- a/core/iwasm/common/wasm_memory.c +++ b/core/iwasm/common/wasm_memory.c @@ -615,7 +615,8 @@ is_app_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst, (uint64)get_last_used_shared_heap_start_offset(module_inst); shared_heap_end = (uint64)get_last_used_shared_heap_end_offset(module_inst); if (app_offset >= shared_heap_start - && app_offset <= shared_heap_end - bytes + 1) { + && app_offset <= shared_heap_end - bytes + 1 + && bytes - 1 <= shared_heap_end) { return true; } @@ -624,7 +625,8 @@ is_app_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst, is_memory64 ? heap->start_off_mem64 : heap->start_off_mem32; shared_heap_end = is_memory64 ? UINT64_MAX : UINT32_MAX; if (app_offset < shared_heap_start - || app_offset > shared_heap_end - bytes + 1) { + || app_offset > shared_heap_end - bytes + 1 + || bytes - 1 > shared_heap_end) { goto fail; } @@ -635,7 +637,8 @@ is_app_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst, is_memory64 ? cur->start_off_mem64 : cur->start_off_mem32; shared_heap_end = shared_heap_start - 1 + cur->size; if (app_offset >= shared_heap_start - && app_offset <= shared_heap_end - bytes + 1) { + && app_offset <= shared_heap_end - bytes + 1 + && bytes - 1 <= shared_heap_end) { update_last_used_shared_heap(module_inst, cur, is_memory64); return true; } @@ -1227,7 +1230,9 @@ wasm_runtime_addr_native_to_app(WASMModuleInstanceCommon *module_inst_comm, #if WASM_ENABLE_SHARED_HEAP != 0 if (is_native_addr_in_shared_heap(module_inst_comm, memory_inst->is_memory64, addr, 1)) { - return addr - get_last_used_shared_heap_base_addr_adj(module_inst_comm); + return (uint64)(uintptr_t)(addr + - get_last_used_shared_heap_base_addr_adj( + module_inst_comm)); } #endif @@ -1349,7 +1354,7 @@ wasm_check_app_addr_and_convert(WASMModuleInstance *module_inst, bool is_str, (WASMModuleInstanceCommon *)module_inst); shared_heap_end_off = get_last_used_shared_heap_end_offset( (WASMModuleInstanceCommon *)module_inst); - native_addr = shared_heap_base_addr_adj + app_buf_addr; + native_addr = shared_heap_base_addr_adj + (uintptr_t)app_buf_addr; /* The whole string must be in the shared heap */ str = (const char *)native_addr; diff --git a/core/iwasm/common/wasm_runtime_common.c b/core/iwasm/common/wasm_runtime_common.c index dcee0aeafc..02a217ff00 100644 --- a/core/iwasm/common/wasm_runtime_common.c +++ b/core/iwasm/common/wasm_runtime_common.c @@ -7883,3 +7883,40 @@ wasm_runtime_is_underlying_binary_freeable(WASMModuleCommon *const module) return true; } + +#if WASM_ENABLE_SHARED_HEAP != 0 +bool +wasm_runtime_update_last_used_shared_heap(WASMModuleInstanceCommon *module_inst, + uintptr_t app_offset, size_t bytes, + uintptr_t *shared_heap_start_off_p, + uintptr_t *shared_heap_end_off_p, + uint8 **shared_heap_base_addr_adj_p, + bool is_memory64) +{ + WASMSharedHeap *heap = wasm_runtime_get_shared_heap(module_inst), *cur; + uint64 shared_heap_start, shared_heap_end; + + if (bytes == 0) { + bytes = 1; + } + + /* Find the exact shared heap that app addr is in, and update last used + * shared heap info in func context */ + for (cur = heap; cur; cur = cur->chain_next) { + shared_heap_start = + is_memory64 ? cur->start_off_mem64 : cur->start_off_mem32; + shared_heap_end = shared_heap_start - 1 + cur->size; + if (app_offset >= shared_heap_start + && app_offset <= shared_heap_end - bytes + 1 + && bytes - 1 <= shared_heap_end) { + *shared_heap_start_off_p = (uintptr_t)shared_heap_start; + *shared_heap_end_off_p = (uintptr_t)shared_heap_end; + *shared_heap_base_addr_adj_p = + cur->base_addr - (uintptr_t)shared_heap_start; + return true; + } + } + + return false; +} +#endif diff --git a/core/iwasm/common/wasm_runtime_common.h b/core/iwasm/common/wasm_runtime_common.h index 64a6cd7936..e2b227c0a7 100644 --- a/core/iwasm/common/wasm_runtime_common.h +++ b/core/iwasm/common/wasm_runtime_common.h @@ -1336,6 +1336,16 @@ void wasm_runtime_set_linux_perf(bool flag); #endif +#if WASM_ENABLE_SHARED_HEAP != 0 +bool +wasm_runtime_update_last_used_shared_heap(WASMModuleInstanceCommon *module_inst, + uintptr_t app_offset, size_t bytes, + uintptr_t *shared_heap_start_off_p, + uintptr_t *shared_heap_end_off_p, + uint8 **shared_heap_base_addr_adj_p, + bool is_memory64); +#endif + #ifdef __cplusplus } #endif diff --git a/core/iwasm/compilation/aot_emit_memory.c b/core/iwasm/compilation/aot_emit_memory.c index e685fccd92..dfb90e7622 100644 --- a/core/iwasm/compilation/aot_emit_memory.c +++ b/core/iwasm/compilation/aot_emit_memory.c @@ -10,6 +10,40 @@ #include "aot_intrinsic.h" #include "aot_emit_control.h" +#define BUILD_IS_NOT_NULL(value, res, name) \ + do { \ + if (!(res = LLVMBuildIsNotNull(comp_ctx->builder, value, name))) { \ + aot_set_last_error("llvm build is not null failed."); \ + goto fail; \ + } \ + } while (0) + +#define BUILD_BR(llvm_block) \ + do { \ + if (!LLVMBuildBr(comp_ctx->builder, llvm_block)) { \ + aot_set_last_error("llvm build br failed."); \ + goto fail; \ + } \ + } while (0) + +#define BUILD_COND_BR(value_if, block_then, block_else) \ + do { \ + if (!LLVMBuildCondBr(comp_ctx->builder, value_if, block_then, \ + block_else)) { \ + aot_set_last_error("llvm build cond br failed."); \ + goto fail; \ + } \ + } while (0) + +#define BUILD_TRUNC(value, data_type) \ + do { \ + if (!(value = LLVMBuildTrunc(comp_ctx->builder, value, data_type, \ + "val_trunc"))) { \ + aot_set_last_error("llvm build trunc failed."); \ + goto fail; \ + } \ + } while (0) + #define BUILD_ICMP(op, left, right, res, name) \ do { \ if (!(res = \ @@ -111,6 +145,417 @@ ffs(int n) static LLVMValueRef get_memory_curr_page_count(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx); +#if WASM_ENABLE_SHARED_HEAP != 0 +uint32 +get_module_inst_extra_offset(AOTCompContext *comp_ctx); + +#define BUILD_LOAD_PTR(ptr, data_type, res) \ + do { \ + if (!(res = LLVMBuildLoad2(comp_ctx->builder, data_type, ptr, \ + "load_value"))) { \ + aot_set_last_error("llvm build load failed"); \ + goto fail; \ + } \ + } while (0) + +/* Update last used shared heap info(alloc ptr) in function ctx: + * 1. shared_heap_start_off 2. shared_heap_end_off 3. shared_heap_base_addr_adj + */ +bool +aot_check_shared_heap_chain(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + LLVMBasicBlockRef check_succ, + LLVMValueRef start_offset, LLVMValueRef bytes, + bool is_memory64) +{ + LLVMValueRef param_values[7], ret_value, func, value, cmp; + LLVMTypeRef param_types[7], ret_type, func_type, func_ptr_type; + + param_types[0] = INT8_PTR_TYPE; + param_types[1] = INTPTR_T_TYPE; + param_types[2] = SIZE_T_TYPE; + param_types[3] = INTPTR_T_PTR_TYPE; + param_types[4] = INTPTR_T_PTR_TYPE; + param_types[5] = INT8_PTR_TYPE; + param_types[6] = INT8_TYPE; + ret_type = INT8_TYPE; + + GET_AOT_FUNCTION(wasm_runtime_update_last_used_shared_heap, 7); + + /* Call function */ + param_values[0] = func_ctx->aot_inst; + param_values[1] = start_offset; + param_values[2] = bytes; + /* pass alloc ptr */ + param_values[3] = func_ctx->shared_heap_start_off; + param_values[4] = func_ctx->shared_heap_end_off; + param_values[5] = func_ctx->shared_heap_base_addr_adj; + param_values[6] = is_memory64 ? I8_ONE : I8_ZERO; + + if (!(ret_value = LLVMBuildCall2(comp_ctx->builder, func_type, func, + param_values, 7, "call"))) { + aot_set_last_error("llvm build call failed."); + goto fail; + } + + BUILD_ICMP(LLVMIntEQ, ret_value, I8_ZERO, cmp, "shared_heap_oob"); + if (!aot_emit_exception(comp_ctx, func_ctx, + EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS, true, cmp, + check_succ)) { + goto fail; + } + + return true; +fail: + return false; +} + +/* + * Setup the basic blocks for shared heap and shared chain memory checks. + * + * Arguments: + * block_curr: The current basic block. + * app_addr_in_cache_shared_heap: Output, block for cache shared heap. + * app_addr_in_linear_mem: Output, block for linear memory. + * app_addr_in_shared_heap_chain: Output, block for shared heap chain + * (only for shared heap chain). + * check_shared_heap_chain: Output, block for checking shared heap chain + * (only for shared heap chain). + * + * Topology: + * If enable_shared_heap: + * block_curr -> app_addr_in_cache_shared_heap + * -> app_addr_in_linear_mem + * If enable_shared_chain: + * block_curr -> app_addr_in_shared_heap_chain + * -> app_addr_in_cache_shared_heap + * -> check_shared_heap_chain + * -> app_addr_in_linear_mem + */ +static bool +setup_shared_heap_blocks(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + LLVMBasicBlockRef block_curr, + LLVMBasicBlockRef *app_addr_in_cache_shared_heap, + LLVMBasicBlockRef *app_addr_in_linear_mem, + LLVMBasicBlockRef *app_addr_in_shared_heap_chain, + LLVMBasicBlockRef *check_shared_heap_chain) +{ + ADD_BASIC_BLOCK(*app_addr_in_cache_shared_heap, + "app_addr_in_cache_shared_heap"); + ADD_BASIC_BLOCK(*app_addr_in_linear_mem, "app_addr_in_linear_mem"); + + if (comp_ctx->enable_shared_heap) { + LLVMMoveBasicBlockAfter(*app_addr_in_cache_shared_heap, block_curr); + LLVMMoveBasicBlockAfter(*app_addr_in_linear_mem, + *app_addr_in_cache_shared_heap); + } + else if (comp_ctx->enable_shared_chain) { + ADD_BASIC_BLOCK(*app_addr_in_shared_heap_chain, + "app_addr_in_shared_heap_chain"); + ADD_BASIC_BLOCK(*check_shared_heap_chain, "check_shared_heap_chain"); + LLVMMoveBasicBlockAfter(*app_addr_in_shared_heap_chain, block_curr); + LLVMMoveBasicBlockAfter(*app_addr_in_cache_shared_heap, + *app_addr_in_shared_heap_chain); + LLVMMoveBasicBlockAfter(*check_shared_heap_chain, + *app_addr_in_cache_shared_heap); + LLVMMoveBasicBlockAfter(*app_addr_in_linear_mem, + *app_addr_in_cache_shared_heap); + } + + return true; +fail: + return false; +} + +/* + * Build a branch to check if start_offset is in the shared heap chain region. + * + * Arguments: + * start_offset: The offset to check. + * app_addr_in_shared_heap_chain: Block to branch if in shared heap chain. + * app_addr_in_linear_mem: Block to branch if not in shared heap chain. + */ +static bool +build_check_app_addr_in_shared_heap_chain( + AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + LLVMValueRef start_offset, LLVMBasicBlockRef app_addr_in_shared_heap_chain, + LLVMBasicBlockRef app_addr_in_linear_mem) +{ + LLVMValueRef is_in_shared_heap = NULL; + + /* Use start_offset > func_ctx->shared_heap_head_start_off to test + * start_off falls in shared heap chain memory region. The shared heap + * chain oob will be detected in app_addr_in_shared_heap block or + * aot_check_shared_heap_chain function + */ + BUILD_ICMP(LLVMIntUGT, start_offset, func_ctx->shared_heap_head_start_off, + is_in_shared_heap, "shared_heap_lb_cmp"); + BUILD_COND_BR(is_in_shared_heap, app_addr_in_shared_heap_chain, + app_addr_in_linear_mem); + + SET_BUILD_POS(app_addr_in_shared_heap_chain); + + return true; +fail: + return false; +} + +/* + * Build the conditional branch for cache shared heap or shared heap chain. + * + * Arguments: + * cmp: The condition for being in cache shared heap. + * app_addr_in_cache_shared_heap: Block for cache shared heap. + * app_addr_in_linear_mem: Block for linear memory. + * check_shared_heap_chain: Block for checking shared heap chain. + * bytes: The access size in bytes. + * start_offset: The offset to check. + * is_memory64: Whether memory is 64-bit. + */ +static bool +build_shared_heap_conditional_branching( + AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, LLVMValueRef cmp, + LLVMBasicBlockRef app_addr_in_cache_shared_heap, + LLVMBasicBlockRef app_addr_in_linear_mem, + LLVMBasicBlockRef check_shared_heap_chain, LLVMValueRef bytes, + LLVMValueRef start_offset, bool is_memory64) +{ + if (comp_ctx->enable_shared_heap) { + BUILD_COND_BR(cmp, app_addr_in_cache_shared_heap, + app_addr_in_linear_mem); + } + else if (comp_ctx->enable_shared_chain) { + BUILD_COND_BR(cmp, app_addr_in_cache_shared_heap, + check_shared_heap_chain); + SET_BUILD_POS(check_shared_heap_chain); + if (!aot_check_shared_heap_chain(comp_ctx, func_ctx, + app_addr_in_cache_shared_heap, + start_offset, bytes, is_memory64)) + goto fail; + } + return true; +fail: + return false; +} + +/* + * Get the native address in the cache shared heap. + * + * Arguments: + * start_offset: The offset to use for address calculation. + * maddr: Output, the native address that in the cache shared heap. + */ +static bool +build_get_maddr_in_cache_shared_heap(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, + LLVMValueRef start_offset, + LLVMValueRef *maddr) +{ + LLVMValueRef shared_heap_base_addr_adj; + /* load the local variable */ + BUILD_LOAD_PTR(func_ctx->shared_heap_base_addr_adj, INT8_PTR_TYPE, + shared_heap_base_addr_adj); + if (!(*maddr = LLVMBuildInBoundsGEP2( + comp_ctx->builder, INT8_TYPE, shared_heap_base_addr_adj, + &start_offset, 1, "maddr_cache_shared_heap"))) { + aot_set_last_error("llvm build inbounds gep failed"); + goto fail; + } + + return true; +fail: + return false; +} + +/* + * Check for memory overflow in shared heap for normal memory access. + * + * Arguments: + * block_curr: The current basic block. + * block_maddr_phi: The phi block for memory address. + * maddr_phi: The phi node for memory address. + * start_offset: The first offset to check. + * mem_base_addr: The base address of memory. Only used with segue. + * bytes_u32: The access size in bytes. + * is_memory64: Whether memory is wasm64 memory. + * is_target_64bit: Whether target is 64-bit. + * enable_segue: Whether to use segment register addressing. + */ +static bool +aot_check_shared_heap_memory_overflow( + AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + LLVMBasicBlockRef block_curr, LLVMBasicBlockRef block_maddr_phi, + LLVMValueRef maddr_phi, LLVMValueRef start_offset, + LLVMValueRef mem_base_addr, uint32 bytes_u32, bool is_memory64, + bool is_target_64bit, bool enable_segue) +{ + LLVMBasicBlockRef app_addr_in_cache_shared_heap, app_addr_in_linear_mem; + LLVMBasicBlockRef app_addr_in_shared_heap_chain = NULL, + check_shared_heap_chain = NULL; + LLVMValueRef cmp, cmp1, cmp2, shared_heap_start_off, shared_heap_end_off, + shared_heap_check_bound, maddr = NULL; + /* On 64/32-bit target, the offset is 64/32-bit */ + LLVMTypeRef offset_type = is_target_64bit ? I64_TYPE : I32_TYPE; + LLVMValueRef length, bytes; + + if (!setup_shared_heap_blocks( + comp_ctx, func_ctx, block_curr, &app_addr_in_cache_shared_heap, + &app_addr_in_linear_mem, &app_addr_in_shared_heap_chain, + &check_shared_heap_chain)) + goto fail; + LLVMMoveBasicBlockAfter(block_maddr_phi, app_addr_in_linear_mem); + + /* Early branching when it's not in shared heap chain at all */ + if (comp_ctx->enable_shared_chain + && !build_check_app_addr_in_shared_heap_chain( + comp_ctx, func_ctx, start_offset, app_addr_in_shared_heap_chain, + app_addr_in_linear_mem)) + goto fail; + + /* Load the local variable of the function */ + BUILD_LOAD_PTR(func_ctx->shared_heap_start_off, offset_type, + shared_heap_start_off); + BUILD_LOAD_PTR(func_ctx->shared_heap_end_off, offset_type, + shared_heap_end_off); + /* Check if the app address is in the cache shared heap range. + * If yes, branch to the cache branch; if not, check the shared heap chain + */ + BUILD_ICMP(LLVMIntUGE, start_offset, shared_heap_start_off, cmp, + "cmp_cache_shared_heap_start"); + length = + is_target_64bit ? I64_CONST(bytes_u32 - 1) : I32_CONST(bytes_u32 - 1); + CHECK_LLVM_CONST(length); + BUILD_OP(Sub, shared_heap_end_off, length, shared_heap_check_bound, + "cache_shared_heap_end_bound"); + BUILD_ICMP(LLVMIntULE, start_offset, shared_heap_check_bound, cmp1, + "cmp_cache_shared_heap_end"); + BUILD_OP(And, cmp, cmp1, cmp2, "is_in_cache_shared_heap"); + /* Conditional branching based on whether in cached shared heap */ + bytes = is_target_64bit ? I64_CONST(bytes_u32) : I32_CONST(bytes_u32); + if (!build_shared_heap_conditional_branching( + comp_ctx, func_ctx, cmp2, app_addr_in_cache_shared_heap, + app_addr_in_linear_mem, check_shared_heap_chain, bytes, + start_offset, is_memory64)) + goto fail; + + SET_BUILD_POS(app_addr_in_cache_shared_heap); + if (!build_get_maddr_in_cache_shared_heap(comp_ctx, func_ctx, start_offset, + &maddr)) + goto fail; + + if (enable_segue) { + LLVMValueRef mem_base_addr_u64, maddr_u64, offset_to_mem_base; + if (!(maddr_u64 = LLVMBuildPtrToInt(comp_ctx->builder, maddr, I64_TYPE, + "maddr_u64")) + || !(mem_base_addr_u64 = + LLVMBuildPtrToInt(comp_ctx->builder, mem_base_addr, + I64_TYPE, "mem_base_addr_u64"))) { + aot_set_last_error("llvm build ptr to int failed"); + goto fail; + } + if (!(offset_to_mem_base = + LLVMBuildSub(comp_ctx->builder, maddr_u64, mem_base_addr_u64, + "offset_to_mem_base"))) { + aot_set_last_error("llvm build sub failed"); + goto fail; + } + if (!(maddr = LLVMBuildIntToPtr(comp_ctx->builder, offset_to_mem_base, + INT8_PTR_TYPE_GS, + "maddr_shared_heap_segue"))) { + aot_set_last_error("llvm build int to ptr failed."); + goto fail; + } + } + + LLVMAddIncoming(maddr_phi, &maddr, &app_addr_in_cache_shared_heap, 1); + BUILD_BR(block_maddr_phi); + SET_BUILD_POS(app_addr_in_linear_mem); + + return true; +fail: + return false; +} + +/* + * Check for memory overflow in shared heap for bulk memory access. + * + * Arguments: + * block_curr: The current basic block. + * block_maddr_phi: The phi block for memory address. + * check_succ: The block to branch to on success. + * maddr_phi: The phi node for memory address. + * start_offset: The offset to check. + * max_addr: The maximum address to check. + * bytes: The access size in bytes (LLVMValueRef). + * is_memory64: Whether memory is wasm64 memory. + * is_target_64bit: Whether target is 64-bit. + */ +static bool +aot_check_bulk_memory_shared_heap_memory_overflow( + AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + LLVMBasicBlockRef block_curr, LLVMBasicBlockRef block_maddr_phi, + LLVMBasicBlockRef check_succ, LLVMValueRef maddr_phi, + LLVMValueRef start_offset, LLVMValueRef max_addr, LLVMValueRef bytes, + bool is_memory64, bool is_target_64bit) +{ + LLVMBasicBlockRef app_addr_in_cache_shared_heap, app_addr_in_linear_mem; + LLVMBasicBlockRef app_addr_in_shared_heap_chain = NULL, + check_shared_heap_chain = NULL; + LLVMValueRef cmp, cmp1, cmp2, shared_heap_start_off, shared_heap_end_off, + maddr = NULL, max_offset; + /* On 64/32-bit target, the offset is 64/32-bit */ + LLVMTypeRef offset_type = is_target_64bit ? I64_TYPE : I32_TYPE; + + if (!setup_shared_heap_blocks( + comp_ctx, func_ctx, block_curr, &app_addr_in_cache_shared_heap, + &app_addr_in_linear_mem, &app_addr_in_shared_heap_chain, + &check_shared_heap_chain)) + goto fail; + LLVMMoveBasicBlockAfter(block_maddr_phi, check_succ); + + /* Early branching when it's not in shared heap chain at all */ + if (comp_ctx->enable_shared_chain + && !build_check_app_addr_in_shared_heap_chain( + comp_ctx, func_ctx, start_offset, app_addr_in_shared_heap_chain, + app_addr_in_linear_mem)) + goto fail; + + /* Load the local variable of the function */ + BUILD_LOAD_PTR(func_ctx->shared_heap_start_off, offset_type, + shared_heap_start_off); + BUILD_LOAD_PTR(func_ctx->shared_heap_end_off, offset_type, + shared_heap_end_off); + /* Check if the app address is in the cache shared heap range. + * If yes, branch to the cache branch; if not, check the shared heap chain + */ + BUILD_ICMP(LLVMIntUGE, start_offset, shared_heap_start_off, cmp, + "cmp_cache_shared_heap_start"); + BUILD_OP(Add, max_addr, is_target_64bit ? I64_NEG_ONE : I32_NEG_ONE, + max_offset, "max_offset"); + BUILD_ICMP(LLVMIntULE, max_offset, shared_heap_end_off, cmp1, + "cmp_cache_shared_heap_end"); + BUILD_OP(And, cmp, cmp1, cmp2, "is_in_cache_shared_heap"); + /* Conditional branching based on whether in cached shared heap */ + if (!build_shared_heap_conditional_branching( + comp_ctx, func_ctx, cmp2, app_addr_in_cache_shared_heap, + app_addr_in_linear_mem, check_shared_heap_chain, bytes, + start_offset, is_memory64)) + goto fail; + + SET_BUILD_POS(app_addr_in_cache_shared_heap); + if (!build_get_maddr_in_cache_shared_heap(comp_ctx, func_ctx, start_offset, + &maddr)) + goto fail; + + LLVMAddIncoming(maddr_phi, &maddr, &app_addr_in_cache_shared_heap, 1); + BUILD_BR(block_maddr_phi); + SET_BUILD_POS(app_addr_in_linear_mem); + + return true; +fail: + return false; +} +#endif + LLVMValueRef aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, mem_offset_t offset, uint32 bytes, bool enable_segue, @@ -118,10 +563,10 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, { LLVMValueRef offset_const = MEMORY64_COND_VALUE(I64_CONST(offset), I32_CONST(offset)); - LLVMValueRef addr, maddr, maddr_phi = NULL, offset1, cmp1, cmp2, cmp; + LLVMValueRef addr, maddr, offset1, cmp1, cmp; LLVMValueRef mem_base_addr, mem_check_bound; LLVMBasicBlockRef block_curr = LLVMGetInsertBlock(comp_ctx->builder); - LLVMBasicBlockRef check_succ, block_maddr_phi = NULL; + LLVMBasicBlockRef check_succ; AOTValue *aot_value_top; uint32 local_idx_of_aot_value = 0; uint64 const_value; @@ -136,6 +581,10 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, #else bool is_memory64 = IS_MEMORY64; #endif +#if WASM_ENABLE_SHARED_HEAP != 0 + LLVMValueRef maddr_phi = NULL; + LLVMBasicBlockRef block_maddr_phi = NULL; +#endif is_target_64bit = (comp_ctx->pointer_size == sizeof(uint64)) ? true : false; @@ -262,6 +711,13 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, *alignp = 1; } + /* The overflow check needs to be done under following conditions: + * 1. In 64-bit target, offset and addr will be extended to 64-bit + * 1.1 offset + addr can overflow when it's memory64 + * 1.2 no overflow when it's memory32 + * 2. In 32-bit target, offset and addr will be 32-bit + * 2.1 offset + addr can overflow when it's memory32 + */ if (is_target_64bit) { if (!(offset_const = LLVMBuildZExt(comp_ctx->builder, offset_const, I64_TYPE, "offset_i64")) @@ -275,7 +731,9 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, /* offset1 = offset + addr; */ BUILD_OP(Add, offset_const, addr, offset1, "offset1"); - if (is_memory64 && comp_ctx->enable_bound_check) { + /* 1.1 offset + addr can overflow when it's memory64 + * 2.1 Or when it's on 32-bit platform */ + if (is_memory64 || !is_target_64bit) { /* Check whether integer overflow occurs in offset + addr */ LLVMBasicBlockRef check_integer_overflow_end; ADD_BASIC_BLOCK(check_integer_overflow_end, @@ -289,23 +747,14 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, goto fail; } SET_BUILD_POS(check_integer_overflow_end); + block_curr = check_integer_overflow_end; } - if (comp_ctx->enable_shared_heap /* TODO: && mem_idx == 0 */) { - LLVMBasicBlockRef app_addr_in_shared_heap, app_addr_in_linear_mem; - LLVMValueRef is_in_shared_heap, shared_heap_check_bound = NULL; - - /* Add basic blocks */ - ADD_BASIC_BLOCK(app_addr_in_shared_heap, "app_addr_in_shared_heap"); - ADD_BASIC_BLOCK(app_addr_in_linear_mem, "app_addr_in_linear_mem"); +#if WASM_ENABLE_SHARED_HEAP != 0 + if (comp_ctx->enable_shared_heap + || comp_ctx->enable_shared_chain /* TODO: && mem_idx == 0 */) { ADD_BASIC_BLOCK(block_maddr_phi, "maddr_phi"); - - LLVMMoveBasicBlockAfter(app_addr_in_shared_heap, block_curr); - LLVMMoveBasicBlockAfter(app_addr_in_linear_mem, - app_addr_in_shared_heap); - LLVMMoveBasicBlockAfter(block_maddr_phi, app_addr_in_linear_mem); - - LLVMPositionBuilderAtEnd(comp_ctx->builder, block_maddr_phi); + SET_BUILD_POS(block_maddr_phi); if (!(maddr_phi = LLVMBuildPhi(comp_ctx->builder, enable_segue ? INT8_PTR_TYPE_GS : INT8_PTR_TYPE, @@ -313,110 +762,16 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, aot_set_last_error("llvm build phi failed"); goto fail; } + SET_BUILD_POS(block_curr); - LLVMPositionBuilderAtEnd(comp_ctx->builder, block_curr); - - if (!is_target_64bit) { - /* Check whether integer overflow occurs in addr + offset */ - LLVMBasicBlockRef check_integer_overflow_end; - ADD_BASIC_BLOCK(check_integer_overflow_end, - "check_integer_overflow_end"); - LLVMMoveBasicBlockAfter(check_integer_overflow_end, block_curr); - - BUILD_ICMP(LLVMIntULT, offset1, addr, cmp1, "cmp1"); - if (!aot_emit_exception(comp_ctx, func_ctx, - EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS, true, - cmp1, check_integer_overflow_end)) { - goto fail; - } - SET_BUILD_POS(check_integer_overflow_end); - } - - shared_heap_check_bound = - is_memory64 ? I64_CONST(UINT64_MAX - bytes + 1) - : (comp_ctx->pointer_size == sizeof(uint64) - ? I64_CONST(UINT32_MAX - bytes + 1) - : I32_CONST(UINT32_MAX - bytes + 1)); - CHECK_LLVM_CONST(shared_heap_check_bound); - - /* Check whether the bytes to access are in shared heap */ - if (!comp_ctx->enable_bound_check) { - /* Use IntUGT but not IntUGE to compare, since (1) in the ems - memory allocator, the hmu node includes hmu header and hmu - memory, only the latter is returned to the caller as the - allocated memory, the hmu header isn't returned so the - first byte of the shared heap won't be accessed, (2) using - IntUGT gets better performance than IntUGE in some cases */ - BUILD_ICMP(LLVMIntUGT, offset1, func_ctx->shared_heap_start_off, - is_in_shared_heap, "is_in_shared_heap"); - /* We don't check the shared heap's upper boundary if boundary - check isn't enabled, the runtime may also use the guard pages - of shared heap to check the boundary if hardware boundary - check feature is enabled. */ - } - else { - /* Use IntUGT but not IntUGE to compare, same as above */ - BUILD_ICMP(LLVMIntUGT, offset1, func_ctx->shared_heap_start_off, - cmp1, "cmp1"); - /* Check the shared heap's upper boundary if boundary check is - enabled */ - BUILD_ICMP(LLVMIntULE, offset1, shared_heap_check_bound, cmp2, - "cmp2"); - BUILD_OP(And, cmp1, cmp2, is_in_shared_heap, "is_in_shared_heap"); - } - - if (!LLVMBuildCondBr(comp_ctx->builder, is_in_shared_heap, - app_addr_in_shared_heap, app_addr_in_linear_mem)) { - aot_set_last_error("llvm build cond br failed"); - goto fail; - } - - LLVMPositionBuilderAtEnd(comp_ctx->builder, app_addr_in_shared_heap); - - /* Get native address inside shared heap */ - if (!(maddr = - LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, - func_ctx->shared_heap_base_addr_adj, - &offset1, 1, "maddr_shared_heap"))) { - aot_set_last_error("llvm build inbounds gep failed"); - goto fail; - } - - if (enable_segue) { - LLVMValueRef mem_base_addr_u64, maddr_u64, offset_to_mem_base; - - if (!(maddr_u64 = LLVMBuildPtrToInt(comp_ctx->builder, maddr, - I64_TYPE, "maddr_u64")) - || !(mem_base_addr_u64 = - LLVMBuildPtrToInt(comp_ctx->builder, mem_base_addr, - I64_TYPE, "mem_base_addr_u64"))) { - aot_set_last_error("llvm build ptr to int failed"); - goto fail; - } - if (!(offset_to_mem_base = - LLVMBuildSub(comp_ctx->builder, maddr_u64, - mem_base_addr_u64, "offset_to_mem_base"))) { - aot_set_last_error("llvm build sub failed"); - goto fail; - } - if (!(maddr = LLVMBuildIntToPtr( - comp_ctx->builder, offset_to_mem_base, INT8_PTR_TYPE_GS, - "maddr_shared_heap_segue"))) { - aot_set_last_error("llvm build int to ptr failed."); - goto fail; - } - } - - LLVMAddIncoming(maddr_phi, &maddr, &app_addr_in_shared_heap, 1); - - if (!LLVMBuildBr(comp_ctx->builder, block_maddr_phi)) { - aot_set_last_error("llvm build br failed"); + if (!aot_check_shared_heap_memory_overflow( + comp_ctx, func_ctx, block_curr, block_maddr_phi, maddr_phi, + offset1, mem_base_addr, bytes, is_memory64, is_target_64bit, + enable_segue)) { goto fail; } - - LLVMPositionBuilderAtEnd(comp_ctx->builder, app_addr_in_linear_mem); - block_curr = LLVMGetInsertBlock(comp_ctx->builder); } +#endif if (comp_ctx->enable_bound_check && !(is_local_of_aot_value @@ -449,21 +804,7 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, goto fail; } - if (is_target_64bit) { - BUILD_ICMP(LLVMIntUGT, offset1, mem_check_bound, cmp, "cmp"); - } - else { - if (comp_ctx->enable_shared_heap /* TODO: && mem_idx == 0 */) { - /* Check integer overflow has been checked above */ - BUILD_ICMP(LLVMIntUGT, offset1, mem_check_bound, cmp, "cmp"); - } - else { - /* Check integer overflow */ - BUILD_ICMP(LLVMIntULT, offset1, addr, cmp1, "cmp1"); - BUILD_ICMP(LLVMIntUGT, offset1, mem_check_bound, cmp2, "cmp2"); - BUILD_OP(Or, cmp1, cmp2, cmp, "cmp"); - } - } + BUILD_ICMP(LLVMIntUGT, offset1, mem_check_bound, cmp, "cmp"); /* Add basic blocks */ ADD_BASIC_BLOCK(check_succ, "check_succ"); @@ -509,17 +850,20 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, } } - if (comp_ctx->enable_shared_heap /* TODO: && mem_idx == 0 */) { +#if WASM_ENABLE_SHARED_HEAP != 0 + if (comp_ctx->enable_shared_heap + || comp_ctx->enable_shared_chain /* TODO: && mem_idx == 0 */) { block_curr = LLVMGetInsertBlock(comp_ctx->builder); LLVMAddIncoming(maddr_phi, &maddr, &block_curr, 1); if (!LLVMBuildBr(comp_ctx->builder, block_maddr_phi)) { aot_set_last_error("llvm build br failed"); goto fail; } - LLVMPositionBuilderAtEnd(comp_ctx->builder, block_maddr_phi); + SET_BUILD_POS(block_maddr_phi); return maddr_phi; } else +#endif return maddr; fail: return NULL; @@ -544,15 +888,6 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, LLVMSetAlignment(value, known_align); \ } while (0) -#define BUILD_TRUNC(value, data_type) \ - do { \ - if (!(value = LLVMBuildTrunc(comp_ctx->builder, value, data_type, \ - "val_trunc"))) { \ - aot_set_last_error("llvm build trunc failed."); \ - goto fail; \ - } \ - } while (0) - #define BUILD_STORE() \ do { \ LLVMValueRef res; \ @@ -1150,16 +1485,23 @@ LLVMValueRef check_bulk_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, LLVMValueRef offset, LLVMValueRef bytes) { - LLVMValueRef maddr, max_addr, cmp; - LLVMValueRef mem_base_addr, maddr_phi = NULL; + LLVMValueRef maddr, max_addr, cmp, cmp1; + LLVMValueRef mem_base_addr; LLVMBasicBlockRef block_curr = LLVMGetInsertBlock(comp_ctx->builder); - LLVMBasicBlockRef check_succ, block_maddr_phi = NULL; + LLVMBasicBlockRef check_succ; LLVMValueRef mem_size; + bool is_target_64bit; #if WASM_ENABLE_MEMORY64 == 0 bool is_memory64 = false; #else bool is_memory64 = IS_MEMORY64; #endif +#if WASM_ENABLE_SHARED_HEAP != 0 + LLVMValueRef maddr_phi = NULL; + LLVMBasicBlockRef block_maddr_phi = NULL; +#endif + + is_target_64bit = (comp_ctx->pointer_size == sizeof(uint64)) ? true : false; /* Get memory base address and memory data size */ #if WASM_ENABLE_SHARED_MEMORY != 0 @@ -1221,111 +1563,71 @@ check_bulk_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, ADD_BASIC_BLOCK(check_succ, "check_succ"); LLVMMoveBasicBlockAfter(check_succ, block_curr); - offset = - LLVMBuildZExt(comp_ctx->builder, offset, I64_TYPE, "extend_offset"); - bytes = LLVMBuildZExt(comp_ctx->builder, bytes, I64_TYPE, "extend_len"); - if (!offset || !bytes) { - aot_set_last_error("llvm build zext failed."); - goto fail; + /* Same logic with aot_check_memory_overflow, offset and bytes are 32/64 + * bits on 32/64 bits platform */ + if (is_target_64bit) { + offset = + LLVMBuildZExt(comp_ctx->builder, offset, I64_TYPE, "extend_offset"); + bytes = LLVMBuildZExt(comp_ctx->builder, bytes, I64_TYPE, "extend_len"); + if (!offset || !bytes) { + aot_set_last_error("llvm build zext failed."); + goto fail; + } } BUILD_OP(Add, offset, bytes, max_addr, "max_addr"); - if (is_memory64 && comp_ctx->enable_bound_check) { - /* Check whether integer overflow occurs in offset + addr */ + /* Check overflow when it's memory64 or it's on 32 bits platform */ + if (is_memory64 || !is_target_64bit) { + /* Check whether integer overflow occurs in offset + bytes */ LLVMBasicBlockRef check_integer_overflow_end; ADD_BASIC_BLOCK(check_integer_overflow_end, "check_integer_overflow_end"); LLVMMoveBasicBlockAfter(check_integer_overflow_end, block_curr); + /* offset + bytes can overflow yet is valid(for example, 0xffffffff, 1), + * allow it to be 0(either 0, 0 or overflow and valid) */ BUILD_ICMP(LLVMIntULT, max_addr, offset, cmp, "cmp"); + BUILD_ICMP(LLVMIntNE, max_addr, is_target_64bit ? I64_ZERO : I32_ZERO, + cmp1, "cmp1"); + BUILD_OP(And, cmp, cmp1, cmp, "overflow"); if (!aot_emit_exception(comp_ctx, func_ctx, EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS, true, cmp, check_integer_overflow_end)) { goto fail; } SET_BUILD_POS(check_integer_overflow_end); + block_curr = check_integer_overflow_end; } - if (comp_ctx->enable_shared_heap /* TODO: && mem_idx == 0 */) { - LLVMBasicBlockRef app_addr_in_shared_heap, app_addr_in_linear_mem; - LLVMValueRef shared_heap_start_off, shared_heap_check_bound; - LLVMValueRef max_offset, cmp1, cmp2, is_in_shared_heap; - - /* Add basic blocks */ - ADD_BASIC_BLOCK(app_addr_in_shared_heap, "app_addr_in_shared_heap"); - ADD_BASIC_BLOCK(app_addr_in_linear_mem, "app_addr_in_linear_mem"); +#if WASM_ENABLE_SHARED_HEAP != 0 + if (comp_ctx->enable_shared_heap + || comp_ctx->enable_shared_chain /* TODO: && mem_idx == 0 */) { ADD_BASIC_BLOCK(block_maddr_phi, "maddr_phi"); - - LLVMMoveBasicBlockAfter(app_addr_in_shared_heap, block_curr); - LLVMMoveBasicBlockAfter(app_addr_in_linear_mem, - app_addr_in_shared_heap); - LLVMMoveBasicBlockAfter(block_maddr_phi, check_succ); - - LLVMPositionBuilderAtEnd(comp_ctx->builder, block_maddr_phi); + SET_BUILD_POS(block_maddr_phi); if (!(maddr_phi = LLVMBuildPhi(comp_ctx->builder, INT8_PTR_TYPE, "maddr_phi"))) { aot_set_last_error("llvm build phi failed"); goto fail; } + SET_BUILD_POS(block_curr); - LLVMPositionBuilderAtEnd(comp_ctx->builder, block_curr); - - shared_heap_start_off = func_ctx->shared_heap_start_off; - if (comp_ctx->pointer_size == sizeof(uint32)) { - if (!(shared_heap_start_off = - LLVMBuildZExt(comp_ctx->builder, shared_heap_start_off, - I64_TYPE, "shared_heap_start_off_u64"))) { - aot_set_last_error("llvm build zext failed"); - goto fail; - } - } - shared_heap_check_bound = - is_memory64 ? I64_CONST(UINT64_MAX) : I64_CONST(UINT32_MAX); - CHECK_LLVM_CONST(shared_heap_check_bound); - - /* Check whether the bytes to access are in shared heap */ - if (!comp_ctx->enable_bound_check) { - /* Use IntUGT but not IntUGE to compare, same as the check - in aot_check_memory_overflow */ - BUILD_ICMP(LLVMIntUGT, offset, func_ctx->shared_heap_start_off, - is_in_shared_heap, "is_in_shared_heap"); - } - else { - BUILD_ICMP(LLVMIntUGT, offset, func_ctx->shared_heap_start_off, - cmp1, "cmp1"); - BUILD_OP(Add, max_addr, I64_NEG_ONE, max_offset, "max_offset"); - BUILD_ICMP(LLVMIntULE, max_offset, shared_heap_check_bound, cmp2, - "cmp2"); - BUILD_OP(And, cmp1, cmp2, is_in_shared_heap, "is_in_shared_heap"); - } - - if (!LLVMBuildCondBr(comp_ctx->builder, is_in_shared_heap, - app_addr_in_shared_heap, app_addr_in_linear_mem)) { - aot_set_last_error("llvm build cond br failed"); - goto fail; - } - - LLVMPositionBuilderAtEnd(comp_ctx->builder, app_addr_in_shared_heap); - - /* Get native address inside shared heap */ - if (!(maddr = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, - func_ctx->shared_heap_base_addr_adj, - &offset, 1, "maddr_shared_heap"))) { - aot_set_last_error("llvm build inbounds gep failed"); - goto fail; - } - LLVMAddIncoming(maddr_phi, &maddr, &app_addr_in_shared_heap, 1); - - if (!LLVMBuildBr(comp_ctx->builder, block_maddr_phi)) { - aot_set_last_error("llvm build br failed"); + if (!aot_check_bulk_memory_shared_heap_memory_overflow( + comp_ctx, func_ctx, block_curr, block_maddr_phi, check_succ, + maddr_phi, offset, max_addr, bytes, is_memory64, + is_target_64bit)) { goto fail; } - - LLVMPositionBuilderAtEnd(comp_ctx->builder, app_addr_in_linear_mem); - block_curr = LLVMGetInsertBlock(comp_ctx->builder); } +#endif + /* mem_size is always 64-bit, extend max_addr on 32 bits platform */ + if (!is_target_64bit + && !(max_addr = LLVMBuildZExt(comp_ctx->builder, max_addr, I64_TYPE, + "extend_max_addr"))) { + aot_set_last_error("llvm build zext failed."); + goto fail; + } BUILD_ICMP(LLVMIntUGT, max_addr, mem_size, cmp, "cmp_max_mem_addr"); if (!aot_emit_exception(comp_ctx, func_ctx, @@ -1341,7 +1643,9 @@ check_bulk_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, goto fail; } - if (comp_ctx->enable_shared_heap /* TODO: && mem_idx == 0 */) { +#if WASM_ENABLE_SHARED_HEAP != 0 + if (comp_ctx->enable_shared_heap + || comp_ctx->enable_shared_chain /* TODO: && mem_idx == 0 */) { block_curr = LLVMGetInsertBlock(comp_ctx->builder); LLVMAddIncoming(maddr_phi, &maddr, &block_curr, 1); if (!LLVMBuildBr(comp_ctx->builder, block_maddr_phi)) { @@ -1352,6 +1656,7 @@ check_bulk_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, return maddr_phi; } else +#endif return maddr; fail: return NULL; diff --git a/core/iwasm/compilation/aot_llvm.c b/core/iwasm/compilation/aot_llvm.c index c1708e3f9d..9694a3e184 100644 --- a/core/iwasm/compilation/aot_llvm.c +++ b/core/iwasm/compilation/aot_llvm.c @@ -1517,73 +1517,153 @@ create_memory_info(const AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, return true; } +#define BUILD_IS_NOT_NULL(value, res, name) \ + do { \ + if (!(res = LLVMBuildIsNotNull(comp_ctx->builder, value, name))) { \ + aot_set_last_error("llvm build is not null failed."); \ + goto fail; \ + } \ + } while (0) + +#define get_module_extra_field_offset(field) \ + do { \ + offset_u32 = get_module_inst_extra_offset(comp_ctx); \ + if (comp_ctx->is_jit_mode) \ + offset_u32 += offsetof(WASMModuleInstanceExtra, field); \ + else \ + offset_u32 += offsetof(AOTModuleInstanceExtra, field); \ + } while (0) + +#define LOAD_MODULE_EXTRA_FIELD_AND_ALLOCA(field, type) \ + do { \ + get_module_extra_field_offset(field); \ + offset = I32_CONST(offset_u32); \ + CHECK_LLVM_CONST(offset); \ + if (!(field_p = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, \ + func_ctx->aot_inst, &offset, 1, \ + #field "_p"))) { \ + aot_set_last_error("llvm build inbounds gep failed"); \ + goto fail; \ + } \ + if (!(load_val = \ + LLVMBuildLoad2(comp_ctx->builder, type, field_p, #field))) { \ + aot_set_last_error("llvm build load failed"); \ + goto fail; \ + } \ + if (!(func_ctx->field = \ + LLVMBuildAlloca(comp_ctx->builder, type, #field))) { \ + aot_set_last_error("llvm build alloca failed"); \ + goto fail; \ + } \ + if (!LLVMBuildStore(comp_ctx->builder, load_val, func_ctx->field)) { \ + aot_set_last_error("llvm build store failed"); \ + goto fail; \ + } \ + } while (0) + static bool create_shared_heap_info(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) { - LLVMValueRef offset, base_addr_p, start_off_p, cmp; +#if WASM_ENABLE_SHARED_HEAP != 0 + LLVMValueRef offset, field_p, load_val, shared_heap_head_p, + shared_heap_head, cmp, field_p_or_default, shared_heap_head_start_off, + shared_heap_head_start_off_minus_one; + LLVMTypeRef shared_heap_offset_type; uint32 offset_u32; - - /* Load aot_inst->e->shared_heap_base_addr_adj */ - offset_u32 = get_module_inst_extra_offset(comp_ctx); -#if WASM_ENABLE_JIT != 0 && WASM_ENABLE_SHARED_HEAP != 0 - if (comp_ctx->is_jit_mode) - offset_u32 += - offsetof(WASMModuleInstanceExtra, shared_heap_base_addr_adj); - else +#if WASM_ENABLE_MEMORY64 == 0 + bool is_memory64 = false; +#else + bool is_memory64 = IS_MEMORY64; #endif - offset_u32 += - offsetof(AOTModuleInstanceExtra, shared_heap_base_addr_adj); + + shared_heap_offset_type = + comp_ctx->pointer_size == sizeof(uint64) ? I64_TYPE : I32_TYPE; + + /* shared_heap_base_addr_adj, shared_heap_start_off, and + * shared_heap_end_off can be updated later, use local variable to + * represent them */ + LOAD_MODULE_EXTRA_FIELD_AND_ALLOCA(shared_heap_base_addr_adj, + INT8_PTR_TYPE); + LOAD_MODULE_EXTRA_FIELD_AND_ALLOCA(shared_heap_start_off, + shared_heap_offset_type); + LOAD_MODULE_EXTRA_FIELD_AND_ALLOCA(shared_heap_end_off, + shared_heap_offset_type); + + /* Shared Heap head start off won't be updated, no need to alloca */ + get_module_extra_field_offset(shared_heap); offset = I32_CONST(offset_u32); CHECK_LLVM_CONST(offset); - - if (!(base_addr_p = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, - func_ctx->aot_inst, &offset, 1, - "shared_heap_base_addr_adj_p"))) { + if (!(shared_heap_head_p = LLVMBuildInBoundsGEP2( + comp_ctx->builder, INT8_TYPE, func_ctx->aot_inst, &offset, 1, + "shared_heap_head_p"))) { aot_set_last_error("llvm build inbounds gep failed"); - return false; + goto fail; } - if (!(func_ctx->shared_heap_base_addr_adj = - LLVMBuildLoad2(comp_ctx->builder, INT8_PTR_TYPE, base_addr_p, - "shared_heap_base_addr_adj"))) { + if (!(shared_heap_head = + LLVMBuildLoad2(comp_ctx->builder, INT8_PTR_TYPE, + shared_heap_head_p, "shared_heap_head"))) { aot_set_last_error("llvm build load failed"); - return false; + goto fail; } + BUILD_IS_NOT_NULL(shared_heap_head, cmp, "has_shared_heap"); - /* Load aot_inst->e->shared_heap_start_off */ - offset_u32 = get_module_inst_extra_offset(comp_ctx); -#if WASM_ENABLE_JIT != 0 && WASM_ENABLE_SHARED_HEAP != 0 - if (comp_ctx->is_jit_mode) - offset_u32 += offsetof(WASMModuleInstanceExtra, shared_heap_start_off); - else -#endif - offset_u32 += offsetof(AOTModuleInstanceExtra, shared_heap_start_off); + if (is_memory64) { + offset_u32 = offsetof(WASMSharedHeap, start_off_mem64); + } + else { + offset_u32 = offsetof(WASMSharedHeap, start_off_mem32); + } offset = I32_CONST(offset_u32); CHECK_LLVM_CONST(offset); - - if (!(start_off_p = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, - func_ctx->aot_inst, &offset, 1, - "shared_heap_start_off_p"))) { + if (!(field_p = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, + shared_heap_head, &offset, 1, + "head_start_off_p"))) { aot_set_last_error("llvm build inbounds gep failed"); - return false; + goto fail; } - if (!(func_ctx->shared_heap_start_off = LLVMBuildLoad2( - comp_ctx->builder, - comp_ctx->pointer_size == sizeof(uint64) ? I64_TYPE : I32_TYPE, - start_off_p, "shared_heap_start_off"))) { - aot_set_last_error("llvm build load failed"); - return false; + + /* Select a valid shared heap head ptr or safe alloca ptr stores + * shared_heap_start_off(UINT32_MAX/UINT64_MAX) */ + if (!(field_p_or_default = LLVMBuildSelect(comp_ctx->builder, cmp, field_p, + func_ctx->shared_heap_start_off, + "ptr_or_default"))) { + aot_set_last_error("llvm build select failed"); + goto fail; } - if (!(cmp = LLVMBuildIsNotNull(comp_ctx->builder, - func_ctx->shared_heap_base_addr_adj, - "has_shared_heap"))) { - aot_set_last_error("llvm build is not null failed"); - return false; + if (!(shared_heap_head_start_off = LLVMBuildLoad2( + comp_ctx->builder, shared_heap_offset_type, field_p_or_default, + "shared_heap_head_start_off"))) { + aot_set_last_error("llvm build load failed"); + goto fail; + } + if (!(shared_heap_head_start_off_minus_one = LLVMBuildAdd( + comp_ctx->builder, shared_heap_head_start_off, + comp_ctx->pointer_size == sizeof(uint64) ? I64_NEG_ONE + : I32_NEG_ONE, + "head_start_off_minus_one"))) { + aot_set_last_error("llvm build load failed"); + goto fail; } + /* if there is attached shared heap(s), the value will be valid start_off-1, + * otherwise it will be UINT32_MAX/UINT64_MAX, so during the bounds checks, + * when has attached shared heap: offset > start_off - 1 => offset >= offset + * when no attached shared heap: offset > UINT32_MAX/UINT64_MAX is always + * false + * */ + if (!(func_ctx->shared_heap_head_start_off = LLVMBuildSelect( + comp_ctx->builder, cmp, shared_heap_head_start_off_minus_one, + shared_heap_head_start_off, "head_start_off"))) { + aot_set_last_error("llvm build select failed"); + goto fail; + } return true; fail: return false; +#else /* else of WASM_ENABLE_SHARED_HEAP != 0 */ + return true; +#endif /* end of WASM_ENABLE_SHARED_HEAP != 0 */ } static bool @@ -1877,7 +1957,7 @@ aot_create_func_context(const AOTCompData *comp_data, AOTCompContext *comp_ctx, } /* Load shared heap, shared heap start off mem32 or mem64 */ - if (comp_ctx->enable_shared_heap + if ((comp_ctx->enable_shared_heap || comp_ctx->enable_shared_chain) && !create_shared_heap_info(comp_ctx, func_ctx)) { goto fail; } @@ -2703,6 +2783,9 @@ aot_create_comp_context(const AOTCompData *comp_data, aot_comp_option_t option) if (option->enable_shared_heap) comp_ctx->enable_shared_heap = true; + if (option->enable_shared_chain) + comp_ctx->enable_shared_chain = true; + comp_ctx->opt_level = option->opt_level; comp_ctx->size_level = option->size_level; diff --git a/core/iwasm/compilation/aot_llvm.h b/core/iwasm/compilation/aot_llvm.h index 6b1233c39f..b4228c67a0 100644 --- a/core/iwasm/compilation/aot_llvm.h +++ b/core/iwasm/compilation/aot_llvm.h @@ -254,8 +254,12 @@ typedef struct AOTFuncContext { bool mem_space_unchanged; AOTCheckedAddrList checked_addr_list; + /* The last accessed shared heap info */ LLVMValueRef shared_heap_base_addr_adj; LLVMValueRef shared_heap_start_off; + LLVMValueRef shared_heap_end_off; + /* The start offset of the head of shared heap chain */ + LLVMValueRef shared_heap_head_start_off; LLVMBasicBlockRef got_exception_block; LLVMBasicBlockRef func_return_block; @@ -486,6 +490,7 @@ typedef struct AOTCompContext { bool enable_gc; bool enable_shared_heap; + bool enable_shared_chain; uint32 opt_level; uint32 size_level; diff --git a/core/iwasm/include/aot_comp_option.h b/core/iwasm/include/aot_comp_option.h index 8391766232..e12f3f6fc3 100644 --- a/core/iwasm/include/aot_comp_option.h +++ b/core/iwasm/include/aot_comp_option.h @@ -79,6 +79,7 @@ typedef struct AOTCompOption { bool enable_stack_estimation; bool quick_invoke_c_api_import; bool enable_shared_heap; + bool enable_shared_chain; char *use_prof_file; uint32_t opt_level; uint32_t size_level; diff --git a/core/iwasm/interpreter/wasm_runtime.c b/core/iwasm/interpreter/wasm_runtime.c index 3cc2afe04d..61628904b6 100644 --- a/core/iwasm/interpreter/wasm_runtime.c +++ b/core/iwasm/interpreter/wasm_runtime.c @@ -2818,6 +2818,7 @@ wasm_instantiate(WASMModule *module, WASMModuleInstance *parent, #else module_inst->e->shared_heap_start_off.u32[0] = UINT32_MAX; #endif + module_inst->e->shared_heap = NULL; #endif #if WASM_ENABLE_GC != 0 diff --git a/core/iwasm/interpreter/wasm_runtime.h b/core/iwasm/interpreter/wasm_runtime.h index 8d8e640b49..b1c28e85c2 100644 --- a/core/iwasm/interpreter/wasm_runtime.h +++ b/core/iwasm/interpreter/wasm_runtime.h @@ -95,13 +95,13 @@ typedef union { typedef struct WASMSharedHeap { /* The global shared heap list maintained in runtime, used for runtime * destroy */ - struct WASMSharedHeap *next; + DefPointer(struct WASMSharedHeap *, next); /* The logical shared heap chain the shared heap in */ - struct WASMSharedHeap *chain_next; + DefPointer(struct WASMSharedHeap *, chain_next); /* Will be null if shared heap is created from pre allocated memory chunk * and don't need to dynamic malloc and free */ - void *heap_handle; - uint8 *base_addr; + DefPointer(void *, heap_handle); + DefPointer(uint8 *, base_addr); uint64 size; uint64 start_off_mem64; uint64 start_off_mem32; diff --git a/samples/shared-heap/CMakeLists.txt b/samples/shared-heap/CMakeLists.txt index 5be99d3a22..03906f7c32 100644 --- a/samples/shared-heap/CMakeLists.txt +++ b/samples/shared-heap/CMakeLists.txt @@ -111,21 +111,31 @@ if (WAMR_BUILD_AOT EQUAL 1) ) if (WAMR_COMPILER) message (CHECK_PASS "found") - else() + else () message (CHECK_FAIL "not found") - endif() + endif () if (NOT EXISTS ${WAMR_COMPILER}) message (FATAL_ERROR "Please build wamrc under ${WAMR_ROOT_DIR}/wamr-compiler") - else() + else () message (STATUS "WAMR_COMPILER is ${WAMR_COMPILER}") - endif() + endif () + + if (WAMR_BUILD_TARGET STREQUAL "X86_32") + set (WAMR_COMPILER_FLAGS --enable-shared-heap --target=i386) + set (WAMR_COMPILER_CHAIN_FLAGS --enable-shared-chain --target=i386) + else () + set (WAMR_COMPILER_FLAGS --enable-shared-heap) + set (WAMR_COMPILER_CHAIN_FLAGS --enable-shared-chain) + endif () add_custom_target( wasm_to_aot ALL DEPENDS wasm-apps/test1.wasm wasm-apps/test2.wasm ${WAMR_COMPILER} - COMMAND ${WAMR_COMPILER} --enable-shared-heap -o wasm-apps/test1.aot wasm-apps/test1.wasm - COMMAND ${WAMR_COMPILER} --enable-shared-heap -o wasm-apps/test2.aot wasm-apps/test2.wasm + COMMAND ${WAMR_COMPILER} ${WAMR_COMPILER_FLAGS} -o wasm-apps/test1.aot wasm-apps/test1.wasm + COMMAND ${WAMR_COMPILER} ${WAMR_COMPILER_FLAGS} -o wasm-apps/test2.aot wasm-apps/test2.wasm + COMMAND ${WAMR_COMPILER} ${WAMR_COMPILER_CHAIN_FLAGS} -o wasm-apps/test1_chain.aot wasm-apps/test1.wasm + COMMAND ${WAMR_COMPILER} ${WAMR_COMPILER_CHAIN_FLAGS} -o wasm-apps/test2_chain.aot wasm-apps/test2.wasm WORKING_DIRECTORY ${CMAKE_BINARY_DIR} ) endif() diff --git a/samples/shared-heap/src/shared_heap_chain.c b/samples/shared-heap/src/shared_heap_chain.c index beb17b137e..8d5a4ea97b 100644 --- a/samples/shared-heap/src/shared_heap_chain.c +++ b/samples/shared-heap/src/shared_heap_chain.c @@ -34,7 +34,7 @@ produce_data(wasm_module_inst_t module_inst, wasm_exec_env_t exec_env, /* Passes wasm address directly between wasm apps since memory in shared * heap chain is viewed as single address space in wasm's perspective */ - buf = (uint8 *)(uint64)argv[0]; + buf = (uint8 *)(uintptr_t)argv[0]; if (!bh_post_msg(queue, 1, buf, buf_size)) { printf("Failed to post message to queue\n"); if (free_on_fail) @@ -130,7 +130,7 @@ wasm_consumer(wasm_module_inst_t module_inst, bh_queue *queue) buf = bh_message_payload(msg); /* call wasm function */ - argv[0] = (uint32)(uint64)buf; + argv[0] = (uint32)(uintptr_t)buf; if (i < 8) wasm_runtime_call_wasm(exec_env, print_buf_func, 1, argv); else @@ -198,7 +198,7 @@ main(int argc, char **argv) if (!aot_mode) wasm_file1 = "./wasm-apps/test1.wasm"; else - wasm_file1 = "./wasm-apps/test1.aot"; + wasm_file1 = "./wasm-apps/test1_chain.aot"; if (!(wasm_file1_buf = bh_read_file_to_buffer(wasm_file1, &wasm_file1_size))) { printf("Open wasm file %s failed.\n", wasm_file1); @@ -225,7 +225,7 @@ main(int argc, char **argv) if (!aot_mode) wasm_file2 = "./wasm-apps/test2.wasm"; else - wasm_file2 = "./wasm-apps/test2.aot"; + wasm_file2 = "./wasm-apps/test2_chain.aot"; if (!(wasm_file2_buf = bh_read_file_to_buffer(wasm_file2, &wasm_file2_size))) { printf("Open wasm file %s failed.\n", wasm_file1); diff --git a/samples/shared-heap/wasm-apps/test1.c b/samples/shared-heap/wasm-apps/test1.c index 8ed1ca84cc..321f102185 100644 --- a/samples/shared-heap/wasm-apps/test1.c +++ b/samples/shared-heap/wasm-apps/test1.c @@ -62,6 +62,10 @@ my_shared_heap_free(void *ptr) void * produce_str(char *addr, uint32_t index) { + char c; snprintf(addr, 512, "Data: %u stores to pre-allocated shared heap", index); + /* Actually access it in wasm */ + c = addr[0]; + printf("In WASM: the first char is %c\n", c); return addr; } diff --git a/samples/shared-heap/wasm-apps/test2.c b/samples/shared-heap/wasm-apps/test2.c index 36ae748b47..44d573164e 100644 --- a/samples/shared-heap/wasm-apps/test2.c +++ b/samples/shared-heap/wasm-apps/test2.c @@ -19,7 +19,10 @@ print_buf(char *buf) void consume_str(char *buf) { - printf("wasm app2's wasm func received buf in pre-allocated shared buf: " - "%s\n\n", - buf); + /* Actually access it in wasm */ + char c = buf[0]; + printf("In WASM: wasm app2's wasm func received buf in pre-allocated " + "shared buf: " + "%s with its first char is %c\n\n", + buf, c); } diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index 8c963cadd8..b726f83a12 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -19,8 +19,15 @@ set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS}") set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS}") if(WAMR_BUILD_TARGET STREQUAL "X86_32") - set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -m32") - set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -m32") + # 1) Force -m32 + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -m32" CACHE STRING "" FORCE) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -m32" CACHE STRING "" FORCE) + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -m32" CACHE STRING "" FORCE) + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -m32" CACHE STRING "" FORCE) + + # 2) Make CMake prefer i386 libraries + set(CMAKE_SYSTEM_PROCESSOR i386 CACHE STRING "" FORCE) + set(CMAKE_LIBRARY_ARCHITECTURE "i386-linux-gnu" CACHE STRING "" FORCE) endif() # Prevent overriding the parent project's compiler/linker @@ -29,12 +36,21 @@ set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) # Fetch Google test include (FetchContent) -FetchContent_Declare ( + +if(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.24") + FetchContent_Declare ( + googletest + URL https://github.com/google/googletest/archive/03597a01ee50ed33e9dfd640b249b4be3799d395.zip + DOWNLOAD_EXTRACT_TIMESTAMP ON + ) +else() + FetchContent_Declare ( googletest URL https://github.com/google/googletest/archive/03597a01ee50ed33e9dfd640b249b4be3799d395.zip - DOWNLOAD_EXTRACT_TIMESTAMP TRUE -) -FetchContent_MakeAvailable (googletest) + ) +endif() + +FetchContent_MakeAvailable(googletest) SET(GOOGLETEST_INCLUDED 1) diff --git a/tests/unit/shared-heap/CMakeLists.txt b/tests/unit/shared-heap/CMakeLists.txt index 2b06c537f8..fa7067918a 100644 --- a/tests/unit/shared-heap/CMakeLists.txt +++ b/tests/unit/shared-heap/CMakeLists.txt @@ -12,12 +12,20 @@ set(WAMR_BUILD_AOT 1) set(WAMR_BUILD_INTERP 1) set(WAMR_BUILD_FAST_INTERP 1) set(WAMR_BUILD_JIT 0) -set(WAMR_BUILD_MEMORY64 1) +if(WAMR_BUILD_TARGET STREQUAL "X86_32") + set(WAMR_BUILD_MEMORY64 0) +else() + set(WAMR_BUILD_MEMORY64 1) +endif() set(WAMR_BUILD_SHARED_HEAP 1) # Compile wasm modules add_subdirectory(wasm-apps) +if (WAMR_BUILD_MEMORY64 EQUAL 1) + add_subdirectory(wasm-apps/memory64) +endif () + # if only load this CMake other than load it as subdirectory include(../unit_common.cmake) @@ -56,4 +64,4 @@ add_executable(shared_heap_test ${unit_test_sources}) target_link_libraries(shared_heap_test ${LLVM_AVAILABLE_LIBS} gtest_main) -gtest_discover_tests(shared_heap_test) \ No newline at end of file +gtest_discover_tests(shared_heap_test) diff --git a/tests/unit/shared-heap/shared_heap_test.cc b/tests/unit/shared-heap/shared_heap_test.cc index 626f0d2d5d..cd4e579865 100644 --- a/tests/unit/shared-heap/shared_heap_test.cc +++ b/tests/unit/shared-heap/shared_heap_test.cc @@ -155,6 +155,9 @@ TEST_F(shared_heap_test, test_shared_heap_basic) test_shared_heap(shared_heap, "test.aot", "test", 0, argv); EXPECT_EQ(10, argv[0]); + + test_shared_heap(shared_heap, "test_chain.aot", "test", 0, argv); + EXPECT_EQ(10, argv[0]); } TEST_F(shared_heap_test, test_shared_heap_malloc_fail) @@ -175,6 +178,10 @@ TEST_F(shared_heap_test, test_shared_heap_malloc_fail) test_shared_heap(shared_heap, "test.aot", "test_malloc_fail", 0, argv); EXPECT_EQ(1, argv[0]); + + test_shared_heap(shared_heap, "test_chain.aot", "test_malloc_fail", 0, + argv); + EXPECT_EQ(1, argv[0]); } TEST_F(shared_heap_test, test_preallocated_shared_heap_malloc_fail) @@ -201,20 +208,41 @@ TEST_F(shared_heap_test, test_preallocated_shared_heap_malloc_fail) argv[0] = 1024; test_shared_heap(shared_heap, "test.aot", "my_shared_heap_malloc", 1, argv); EXPECT_EQ(0, argv[0]); + + argv[0] = 1024; + test_shared_heap(shared_heap, "test_chain.aot", "my_shared_heap_malloc", 1, + argv); + EXPECT_EQ(0, argv[0]); } -TEST_F(shared_heap_test, test_shared_heap_chain_rmw) +static void +create_test_shared_heap(uint8 *preallocated_buf, size_t size, + WASMSharedHeap **shared_heap_res) { SharedHeapInitArgs args = { 0 }; - WASMSharedHeap *shared_heap = nullptr, *shared_heap2 = nullptr, - *shared_heap_chain = nullptr; - uint32 argv[2] = { 0 }, BUF_SIZE = os_getpagesize(); - uint8 preallocated_buf[BUF_SIZE] = { 0 }, - preallocated_buf2[BUF_SIZE] = { 0 }; - uint32 start1, end1, start2, end2; + WASMSharedHeap *shared_heap = nullptr; + args.pre_allocated_addr = preallocated_buf; + args.size = size; + shared_heap = wasm_runtime_create_shared_heap(&args); + if (!shared_heap) { + FAIL() << "Create preallocated shared heap failed.\n"; + } + *shared_heap_res = shared_heap; + if (!*shared_heap_res) { + FAIL() << "Create shared heap chain failed.\n"; + } +} + +static void +create_test_shared_heap_chain(uint8 *preallocated_buf, size_t size, + uint8 *preallocated_buf2, size_t size2, + WASMSharedHeap **shared_heap_chain) +{ + SharedHeapInitArgs args = { 0 }; + WASMSharedHeap *shared_heap = nullptr, *shared_heap2 = nullptr; args.pre_allocated_addr = preallocated_buf; - args.size = BUF_SIZE; + args.size = size; shared_heap = wasm_runtime_create_shared_heap(&args); if (!shared_heap) { FAIL() << "Create preallocated shared heap failed.\n"; @@ -222,23 +250,74 @@ TEST_F(shared_heap_test, test_shared_heap_chain_rmw) memset(&args, 0, sizeof(args)); args.pre_allocated_addr = preallocated_buf2; - args.size = BUF_SIZE; + args.size = size2; shared_heap2 = wasm_runtime_create_shared_heap(&args); if (!shared_heap2) { FAIL() << "Create preallocated shared heap failed.\n"; } - shared_heap_chain = + *shared_heap_chain = wasm_runtime_chain_shared_heaps(shared_heap, shared_heap2); - if (!shared_heap_chain) { + if (!*shared_heap_chain) { FAIL() << "Create shared heap chain failed.\n"; } +} + +TEST_F(shared_heap_test, test_shared_heap_rmw) +{ + WASMSharedHeap *shared_heap = nullptr; + uint32 argv[2] = { 0 }, BUF_SIZE = os_getpagesize(); + uint8 preallocated_buf[BUF_SIZE] = { 0 }; + uint32 start1, end1; + + create_test_shared_heap(preallocated_buf, BUF_SIZE, &shared_heap); + + /* app addr for shared heap */ + start1 = UINT32_MAX - BUF_SIZE + 1; + end1 = UINT32_MAX; + + argv[0] = end1; + argv[1] = 101; + test_shared_heap(shared_heap, "test.wasm", "read_modify_write_8", 2, argv); + EXPECT_EQ(0, argv[0]); + EXPECT_EQ(preallocated_buf[BUF_SIZE - 1], 101); + + argv[0] = start1; + argv[1] = 37; + test_shared_heap(shared_heap, "test.wasm", "read_modify_write_8", 2, argv); + EXPECT_EQ(0, argv[0]); + EXPECT_EQ(preallocated_buf[0], 37); + + argv[0] = end1; + argv[1] = 81; + test_shared_heap(shared_heap, "test.aot", "read_modify_write_8", 2, argv); + EXPECT_EQ(101, argv[0]); + EXPECT_EQ(preallocated_buf[BUF_SIZE - 1], 81); + + argv[0] = start1; + argv[1] = 98; + test_shared_heap(shared_heap, "test.aot", "read_modify_write_8", 2, argv); + EXPECT_EQ(37, argv[0]); + EXPECT_EQ(preallocated_buf[0], 98); +} + +TEST_F(shared_heap_test, test_shared_heap_chain_rmw) +{ + SharedHeapInitArgs args = { 0 }; + WASMSharedHeap *shared_heap_chain = nullptr; + uint32 argv[2] = { 0 }, BUF_SIZE = os_getpagesize(); + uint8 preallocated_buf[BUF_SIZE] = { 0 }, + preallocated_buf2[BUF_SIZE] = { 0 }; + uint32 start1, end1, start2, end2; + + create_test_shared_heap_chain(preallocated_buf, BUF_SIZE, preallocated_buf2, + BUF_SIZE, &shared_heap_chain); /* app addr for shared heap */ - start1 = 0xFFFFFFFF - 2 * BUF_SIZE + 1; - end1 = 0xFFFFFFFF - BUF_SIZE; - start2 = 0xFFFFFFFF - BUF_SIZE + 1; - end2 = 0xFFFFFFFF; + start1 = UINT32_MAX - 2 * BUF_SIZE + 1; + end1 = UINT32_MAX - BUF_SIZE; + start2 = UINT32_MAX - BUF_SIZE + 1; + end2 = UINT32_MAX; /* shared heap 1 */ argv[0] = end1; @@ -256,76 +335,309 @@ TEST_F(shared_heap_test, test_shared_heap_chain_rmw) EXPECT_EQ(0, argv[0]); EXPECT_EQ(preallocated_buf2[0], 129); - /* TODO: test aot when chain is supported in AOT */ - /* argv[0] = start1; argv[1] = 98; - test_shared_heap(shared_heap_chain, "test.aot", "read_modify_write_8", 2, - argv); + test_shared_heap(shared_heap_chain, "test_chain.aot", "read_modify_write_8", + 2, argv); EXPECT_EQ(0, argv[0]); EXPECT_EQ(preallocated_buf[0], 98); argv[0] = end2; argv[1] = 81; - test_shared_heap(shared_heap_chain, "test.aot", "read_modify_write_8", 2, - argv); + test_shared_heap(shared_heap_chain, "test_chain.aot", "read_modify_write_8", + 2, argv); EXPECT_EQ(0, argv[0]); EXPECT_EQ(preallocated_buf2[BUF_SIZE - 1], 81); - */ } -TEST_F(shared_heap_test, test_shared_heap_chain_rmw_oob) +TEST_F(shared_heap_test, test_shared_heap_chain_rmw_bulk_memory) { SharedHeapInitArgs args = { 0 }; - WASMSharedHeap *shared_heap = nullptr, *shared_heap2 = nullptr, - *shared_heap_chain = nullptr; + WASMSharedHeap *shared_heap_chain = nullptr; + uint32 argv[3] = { 0 }, BUF_SIZE = os_getpagesize(); + uint8 preallocated_buf[BUF_SIZE] = { 0 }, + preallocated_buf2[BUF_SIZE] = { 0 }; + uint32 start1, end1, start2, end2; + + create_test_shared_heap_chain(preallocated_buf, BUF_SIZE, preallocated_buf2, + BUF_SIZE, &shared_heap_chain); + + /* app addr for shared heap */ + start1 = UINT32_MAX - 2 * BUF_SIZE + 1; + end1 = UINT32_MAX - BUF_SIZE; + start2 = UINT32_MAX - BUF_SIZE + 1; + end2 = UINT32_MAX; + + argv[0] = end1; + argv[1] = 101; + argv[2] = 1; + test_shared_heap(shared_heap_chain, "test_bulk_memory.wasm", + "memory_fill_test", 3, argv); + /* no modification since no return value */ + EXPECT_EQ(end1, argv[0]); + EXPECT_EQ(preallocated_buf[BUF_SIZE - 1], 101); + + argv[0] = start1; + argv[1] = 14; + argv[2] = 1; + test_shared_heap(shared_heap_chain, "test_bulk_memory_chain.aot", + "memory_fill_test", 3, argv); + /* no modification since no return value */ + EXPECT_EQ(start1, argv[0]); + EXPECT_EQ(preallocated_buf[0], 14); + + /* nothing happen when memory fill 0 byte */ + argv[0] = start2; + argv[1] = 68; + argv[2] = 0; + test_shared_heap(shared_heap_chain, "test_bulk_memory_chain.aot", + "memory_fill_test", 3, argv); + /* no modification since no return value */ + EXPECT_EQ(start2, argv[0]); + EXPECT_EQ(preallocated_buf2[0], 0); + + argv[0] = end2; + argv[1] = 98; + argv[2] = 1; + test_shared_heap(shared_heap_chain, "test_bulk_memory_chain.aot", + "memory_fill_test", 3, argv); + /* no modification since no return value */ + EXPECT_EQ(end2, argv[0]); + EXPECT_EQ(preallocated_buf2[BUF_SIZE - 1], 98); +} + +TEST_F(shared_heap_test, test_shared_heap_chain_rmw_bulk_memory_oob) +{ + SharedHeapInitArgs args = { 0 }; + WASMSharedHeap *shared_heap_chain = nullptr; + uint32 argv[3] = { 0 }, BUF_SIZE = os_getpagesize(); + uint8 preallocated_buf[BUF_SIZE] = { 0 }, + preallocated_buf2[BUF_SIZE] = { 0 }; + uint32 start1, end1, start2, end2; + + create_test_shared_heap_chain(preallocated_buf, BUF_SIZE, preallocated_buf2, + BUF_SIZE, &shared_heap_chain); + + /* app addr for shared heap */ + start1 = UINT32_MAX - 2 * BUF_SIZE + 1; + end1 = UINT32_MAX - BUF_SIZE; + start2 = UINT32_MAX - BUF_SIZE + 1; + end2 = UINT32_MAX; + + /* shared heap 1 */ + argv[0] = end1; + argv[1] = 101; + argv[2] = 2; + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, + "test_bulk_memory.wasm", + "memory_fill_test", 3, argv), + "Exception: out of bounds memory access"); + + argv[0] = end2; + argv[1] = 98; + argv[2] = 2; + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, + "test_bulk_memory.wasm", + "memory_fill_test", 3, argv), + "Exception: out of bounds memory access"); + + argv[0] = start1; + argv[1] = 98; + argv[2] = BUF_SIZE + 1; + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, + "test_bulk_memory.wasm", + "memory_fill_test", 3, argv), + "Exception: out of bounds memory access"); + + argv[0] = start2; + argv[1] = 98; + argv[2] = BUF_SIZE + 1; + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, + "test_bulk_memory.wasm", + "memory_fill_test", 3, argv), + "Exception: out of bounds memory access"); + + argv[0] = end1; + argv[1] = 101; + argv[2] = 2; + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, + "test_bulk_memory_chain.aot", + "memory_fill_test", 3, argv), + "Exception: out of bounds memory access"); + + argv[0] = end2; + argv[1] = 98; + argv[2] = 2; + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, + "test_bulk_memory_chain.aot", + "memory_fill_test", 3, argv), + "Exception: out of bounds memory access"); + + argv[0] = start1; + argv[1] = 98; + argv[2] = BUF_SIZE + 1; + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, + "test_bulk_memory_chain.aot", + "memory_fill_test", 3, argv), + "Exception: out of bounds memory access"); + + argv[0] = start2; + argv[1] = 98; + argv[2] = BUF_SIZE + 1; + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, + "test_bulk_memory_chain.aot", + "memory_fill_test", 3, argv), + "Exception: out of bounds memory access"); +} + +TEST_F(shared_heap_test, test_shared_heap_rmw_oob) +{ + WASMSharedHeap *shared_heap = nullptr; uint32 argv[2] = { 0 }, BUF_SIZE = os_getpagesize(); uint8 preallocated_buf[BUF_SIZE], preallocated_buf2[BUF_SIZE]; uint32 start1, end1, start2, end2; - args.pre_allocated_addr = preallocated_buf; - args.size = BUF_SIZE; - shared_heap = wasm_runtime_create_shared_heap(&args); - if (!shared_heap) { - FAIL() << "Create preallocated shared heap failed.\n"; - } + create_test_shared_heap(preallocated_buf, BUF_SIZE, &shared_heap); - memset(&args, 0, sizeof(args)); - args.pre_allocated_addr = preallocated_buf2; - args.size = BUF_SIZE; - shared_heap2 = wasm_runtime_create_shared_heap(&args); - if (!shared_heap2) { - FAIL() << "Create preallocated shared heap failed.\n"; - } + /* app addr for shared heap */ + start1 = UINT32_MAX - BUF_SIZE + 1; + end1 = UINT32_MAX; - shared_heap_chain = - wasm_runtime_chain_shared_heaps(shared_heap, shared_heap2); - if (!shared_heap_chain) { - FAIL() << "Create shared heap chain failed.\n"; - } + /* try to rmw an u16, first u8 is in the first shared heap and second u8 is + * in the second shared heap, will be seen as oob */ + argv[0] = end1; + argv[1] = 12025; + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap, "test.wasm", + "read_modify_write_16", 2, argv), + "Exception: out of bounds memory access"); + + argv[0] = start1 - 1; + argv[1] = 12025; + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap, "test.aot", + "read_modify_write_16", 2, argv), + "Exception: out of bounds memory access"); + + argv[0] = end1; + argv[1] = 12025; + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap, "test.aot", + "read_modify_write_16", 2, argv), + "Exception: out of bounds memory access"); +} + +TEST_F(shared_heap_test, test_shared_heap_chain_rmw_oob) +{ + WASMSharedHeap *shared_heap_chain = nullptr; + uint32 argv[2] = { 0 }, BUF_SIZE = os_getpagesize(); + uint8 preallocated_buf[BUF_SIZE], preallocated_buf2[BUF_SIZE]; + uint32 start1, end1, start2, end2; + + create_test_shared_heap_chain(preallocated_buf, BUF_SIZE, preallocated_buf2, + BUF_SIZE, &shared_heap_chain); /* app addr for shared heap */ - start1 = 0xFFFFFFFF - 2 * BUF_SIZE + 1; - end1 = 0xFFFFFFFF - BUF_SIZE; - start2 = 0xFFFFFFFF - BUF_SIZE + 1; - end2 = 0xFFFFFFFF; + start1 = UINT32_MAX - 2 * BUF_SIZE + 1; + end1 = UINT32_MAX - BUF_SIZE; + start2 = UINT32_MAX - BUF_SIZE + 1; + end2 = UINT32_MAX; /* try to rmw an u16, first u8 is in the first shared heap and second u8 is * in the second shared heap, will be seen as oob */ - argv[0] = end1; + argv[0] = end2; argv[1] = 12025; EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, "test.wasm", "read_modify_write_16", 2, argv), "Exception: out of bounds memory access"); - /* TODO: test aot when chain is supported in AOT */ - /*argv[0] = end1; + argv[0] = end1; argv[1] = 12025; - EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, "test.wasm", + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, + "test_chain.aot", "read_modify_write_16", 2, argv), - "Exception: out of bounds memory access");*/ + "Exception: out of bounds memory access"); } +#if WASM_ENABLE_MEMORY64 != 0 +TEST_F(shared_heap_test, test_shared_heap_chain_memory64_rmw) +{ + WASMSharedHeap *shared_heap_chain = nullptr; + uint32 argv[3] = { 0 }, BUF_SIZE = os_getpagesize(); + uint8 preallocated_buf[BUF_SIZE] = { 0 }, + preallocated_buf2[BUF_SIZE] = { 0 }; + uint64 start1, end1, start2, end2; + + create_test_shared_heap_chain(preallocated_buf, BUF_SIZE, preallocated_buf2, + BUF_SIZE, &shared_heap_chain); + + /* app addr for shared heap */ + start1 = UINT64_MAX - 2 * BUF_SIZE + 1; + end1 = UINT64_MAX - BUF_SIZE; + start2 = UINT64_MAX - BUF_SIZE + 1; + end2 = UINT64_MAX; + + /* shared heap 1 */ + PUT_I64_TO_ADDR(argv, end1); + argv[2] = 101; + test_shared_heap(shared_heap_chain, "test64.wasm", "read_modify_write_8", 3, + argv); + EXPECT_EQ(0, argv[0]); + EXPECT_EQ(preallocated_buf[BUF_SIZE - 1], 101); + + /* shared heap 2 */ + PUT_I64_TO_ADDR(argv, start2); + argv[2] = 129; + test_shared_heap(shared_heap_chain, "test64.wasm", "read_modify_write_8", 3, + argv); + EXPECT_EQ(0, argv[0]); + EXPECT_EQ(preallocated_buf2[0], 129); + + PUT_I64_TO_ADDR(argv, start1); + argv[2] = 98; + test_shared_heap(shared_heap_chain, "test64_chain.aot", + "read_modify_write_8", 3, argv); + EXPECT_EQ(0, argv[0]); + EXPECT_EQ(preallocated_buf[0], 98); + + PUT_I64_TO_ADDR(argv, end2); + argv[2] = 81; + test_shared_heap(shared_heap_chain, "test64_chain.aot", + "read_modify_write_8", 3, argv); + EXPECT_EQ(0, argv[0]); + EXPECT_EQ(preallocated_buf2[BUF_SIZE - 1], 81); +} + +TEST_F(shared_heap_test, test_shared_heap_chain_memory64_rmw_oob) +{ + WASMSharedHeap *shared_heap_chain = nullptr; + uint32 argv[3] = { 0 }, BUF_SIZE = os_getpagesize(); + uint8 preallocated_buf[BUF_SIZE], preallocated_buf2[BUF_SIZE]; + uint64 start1, end1, start2, end2; + + create_test_shared_heap_chain(preallocated_buf, BUF_SIZE, preallocated_buf2, + BUF_SIZE, &shared_heap_chain); + + /* app addr for shared heap */ + start1 = UINT64_MAX - 2 * BUF_SIZE + 1; + end1 = UINT64_MAX - BUF_SIZE; + start2 = UINT64_MAX - BUF_SIZE + 1; + end2 = UINT64_MAX; + + /* try to rmw an u16, first u8 is in the first shared heap and second u8 is + * in the second shared heap, will be seen as oob */ + PUT_I64_TO_ADDR(argv, end1); + argv[2] = 12025; + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, "test64.wasm", + "read_modify_write_16", 3, argv), + "Exception: out of bounds memory access"); + + PUT_I64_TO_ADDR(argv, end1); + argv[2] = 12025; + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, + "test64_chain.aot", + "read_modify_write_16", 3, argv), + "Exception: out of bounds memory access"); +} +#endif + #ifndef native_function /* clang-format off */ #define native_function(func_name, signature) \ @@ -378,6 +690,9 @@ TEST_F(shared_heap_test, test_addr_conv) test_shared_heap(shared_heap, "test_addr_conv.aot", "test", 0, argv); EXPECT_EQ(1, argv[0]); + + test_shared_heap(shared_heap, "test_addr_conv_chain.aot", "test", 0, argv); + EXPECT_EQ(1, argv[0]); } TEST_F(shared_heap_test, test_addr_conv_pre_allocated_oob) @@ -412,6 +727,12 @@ TEST_F(shared_heap_test, test_addr_conv_pre_allocated_oob) EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap, "test_addr_conv.aot", "test_preallocated", 1, argv), "Exception: out of bounds memory access"); + + argv[0] = app_addr; + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap, + "test_addr_conv_chain.aot", + "test_preallocated", 1, argv), + "Exception: out of bounds memory access"); } TEST_F(shared_heap_test, test_shared_heap_chain) @@ -453,9 +774,8 @@ TEST_F(shared_heap_test, test_shared_heap_chain) test_shared_heap(shared_heap_chain, "test_addr_conv.wasm", "test", 0, argv); EXPECT_EQ(1, argv[0]); - /* TODO: test aot when chain is supported in AOT */ - /*test_shared_heap(shared_heap, "test_addr_conv.aot", "test", 1, argv); - EXPECT_EQ(1, argv[0]);*/ + test_shared_heap(shared_heap, "test_addr_conv.aot", "test", 0, argv); + EXPECT_EQ(1, argv[0]); } TEST_F(shared_heap_test, test_shared_heap_chain_create_fail) @@ -666,14 +986,15 @@ TEST_F(shared_heap_test, test_shared_heap_chain_addr_conv) "test_preallocated", 1, argv); EXPECT_EQ(1, argv[0]); - /* TODO: test aot when chain is supported in AOT */ - /*argv[0] = 0xFFFFFFFF; - test_shared_heap(shared_heap, "test_addr_conv.aot", "test", 1, argv); + argv[0] = 0xFFFFFFFF; + test_shared_heap(shared_heap, "test_addr_conv_chain.aot", + "test_preallocated", 1, argv); EXPECT_EQ(1, argv[0]); argv[0] = 0xFFFFF000; - test_shared_heap(shared_heap, "test_addr_conv.aot", "test", 1, argv); - EXPECT_EQ(1, argv[0]); */ + test_shared_heap(shared_heap, "test_addr_conv_chain.aot", + "test_preallocated", 1, argv); + EXPECT_EQ(1, argv[0]); } TEST_F(shared_heap_test, test_shared_heap_chain_addr_conv_oob) @@ -719,10 +1040,10 @@ TEST_F(shared_heap_test, test_shared_heap_chain_addr_conv_oob) "test_preallocated", 1, argv), "Exception: out of bounds memory access"); - /* TODO: test aot when chain is supported in AOT */ - /*argv[0] = 0xFFFFFFFF - BUF_SIZE - 4096; + /* test aot */ + argv[0] = 0xFFFFFFFF - BUF_SIZE - 4096; EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, - "test_addr_conv.aot", + "test_addr_conv_chain.aot", "test_preallocated", 1, argv), - "Exception: out of bounds memory access");*/ + "Exception: out of bounds memory access"); } diff --git a/tests/unit/shared-heap/wasm-apps/CMakeLists.txt b/tests/unit/shared-heap/wasm-apps/CMakeLists.txt index 097f66ae5d..985cf18aeb 100644 --- a/tests/unit/shared-heap/wasm-apps/CMakeLists.txt +++ b/tests/unit/shared-heap/wasm-apps/CMakeLists.txt @@ -29,44 +29,81 @@ set(CMAKE_EXE_LINKER_FLAGS -Wl,--allow-undefined" ) -add_executable(test.wasm test.c) -target_link_libraries(test.wasm) +if (WAMR_BUILD_TARGET STREQUAL "X86_32") + set (WAMR_COMPILER_FLAGS --opt-level=3 --bounds-checks=1 --enable-shared-heap --target=i386) + set (WAMR_COMPILER_CHAIN_FLAGS --opt-level=3 --bounds-checks=1 --enable-shared-chain --target=i386) +else () + set (WAMR_COMPILER_FLAGS --opt-level=3 --bounds-checks=1 --enable-shared-heap) + set (WAMR_COMPILER_CHAIN_FLAGS --opt-level=3 --bounds-checks=1 --enable-shared-chain) +endif () -add_custom_command(TARGET test.wasm POST_BUILD +function(copy_wasm TARGET_NAME) + add_custom_command(TARGET ${TARGET_NAME} POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy - ${CMAKE_CURRENT_BINARY_DIR}/test.wasm - ${CMAKE_CURRENT_BINARY_DIR}/../ - COMMENT "Copy test.wasm to the same directory of google test" - ) + ${CMAKE_CURRENT_BINARY_DIR}/${TARGET_NAME} + ${CMAKE_CURRENT_BINARY_DIR}/../ + COMMENT "Copy ${TARGET_NAME} to the same directory of google test" + ) +endfunction() + +function(compile_and_copy_aot_from TARGET_NAME) + string(REPLACE ".wasm" ".aot" AOT_TARGET ${TARGET_NAME}) + string(REPLACE ".wasm" "_chain.aot" AOT_CHAIN_TARGET ${TARGET_NAME}) -add_custom_command(TARGET test.wasm POST_BUILD - COMMAND ${WAMRC_ROOT_DIR}/wamrc --opt-level=0 --enable-shared-heap --bounds-checks=1 - -o - test.aot - test.wasm + add_custom_command(TARGET ${TARGET_NAME} POST_BUILD + COMMAND ${WAMRC_ROOT_DIR}/wamrc ${WAMR_COMPILER_FLAGS} + -o ${AOT_TARGET} + ${TARGET_NAME} COMMAND ${CMAKE_COMMAND} -E copy - ${CMAKE_CURRENT_BINARY_DIR}/test.aot - ${CMAKE_CURRENT_BINARY_DIR}/../ - COMMENT "Copy test.aot to the same directory of google test" - ) + ${CMAKE_CURRENT_BINARY_DIR}/${AOT_TARGET} + ${CMAKE_CURRENT_BINARY_DIR}/../ + COMMAND ${WAMRC_ROOT_DIR}/wamrc ${WAMR_COMPILER_CHAIN_FLAGS} + -o ${AOT_CHAIN_TARGET} + ${TARGET_NAME} + COMMAND ${CMAKE_COMMAND} -E copy + ${CMAKE_CURRENT_BINARY_DIR}/${AOT_CHAIN_TARGET} + ${CMAKE_CURRENT_BINARY_DIR}/../ + COMMENT "Compile and copy ${AOT_TARGET} to the same directory of google test" + ) +endfunction() -add_executable(test_addr_conv.wasm test_addr_conv.c) +add_executable(test.wasm test.c) target_link_libraries(test.wasm) +copy_wasm(test.wasm) +compile_and_copy_aot_from(test.wasm) -add_custom_command(TARGET test_addr_conv.wasm POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy - ${CMAKE_CURRENT_BINARY_DIR}/test_addr_conv.wasm - ${CMAKE_CURRENT_BINARY_DIR}/../ - COMMENT "Copy test_addr_conv.wasm to the same directory of google test" - ) +add_executable(test_addr_conv.wasm test_addr_conv.c) +target_link_libraries(test_addr_conv.wasm) +copy_wasm(test_addr_conv.wasm) +compile_and_copy_aot_from(test_addr_conv.wasm) -add_custom_command(TARGET test_addr_conv.wasm POST_BUILD - COMMAND ${WAMRC_ROOT_DIR}/wamrc --opt-level=0 --enable-shared-heap --bounds-checks=1 - -o - test_addr_conv.aot - test_addr_conv.wasm - COMMAND ${CMAKE_COMMAND} -E copy - ${CMAKE_CURRENT_BINARY_DIR}/test_addr_conv.aot - ${CMAKE_CURRENT_BINARY_DIR}/../ - COMMENT "Copy test_addr_conv.aot to the same directory of google test" - ) +# copy and compile aot for bulk memory test +set(SOURCE_WASM ${CMAKE_CURRENT_SOURCE_DIR}/bulk-memory/test_bulk_memory.wasm) +set(BUILD_WASM ${CMAKE_CURRENT_BINARY_DIR}/../test_bulk_memory.wasm) +set(OUTPUT_AOT ${CMAKE_CURRENT_BINARY_DIR}/../test_bulk_memory.aot) +set(OUTPUT_CHAIN_AOT ${CMAKE_CURRENT_BINARY_DIR}/../test_bulk_memory_chain.aot) + +add_custom_command( + OUTPUT ${BUILD_WASM} + COMMAND ${CMAKE_COMMAND} -E copy + ${SOURCE_WASM} + ${BUILD_WASM} + DEPENDS ${SOURCE_WASM} + COMMENT "Copying bulk memory WASM to build directory" +) + +add_custom_command( + OUTPUT ${OUTPUT_AOT} + COMMAND ${WAMRC_ROOT_DIR}/wamrc ${WAMR_COMPILER_FLAGS} + -o ${OUTPUT_AOT} + ${BUILD_WASM} + COMMAND ${WAMRC_ROOT_DIR}/wamrc ${WAMR_COMPILER_CHAIN_FLAGS} + -o ${OUTPUT_CHAIN_AOT} + ${BUILD_WASM} + DEPENDS ${BUILD_WASM} + COMMENT "Compiling bulk memory AOT from copied WASM" +) + +add_custom_target(compile_bulk_memory_aot ALL + DEPENDS ${OUTPUT_AOT} +) diff --git a/tests/unit/shared-heap/wasm-apps/bulk-memory/test_bulk_memory.wasm b/tests/unit/shared-heap/wasm-apps/bulk-memory/test_bulk_memory.wasm new file mode 100644 index 0000000000000000000000000000000000000000..eb3d60be3da992383bd142c229388824ba10439d GIT binary patch literal 63 zcmZQbEY4+QU|?WmXG~zOudiodW@2PuWo85lh%gG|rsn1sRmP`f=H$eeq!yPjFmUlQ Rax*9}C@?B8{o!Wd1^{Vt3%&pV literal 0 HcmV?d00001 diff --git a/tests/unit/shared-heap/wasm-apps/bulk-memory/test_bulk_memory.wat b/tests/unit/shared-heap/wasm-apps/bulk-memory/test_bulk_memory.wat new file mode 100644 index 0000000000..e7a0c684d1 --- /dev/null +++ b/tests/unit/shared-heap/wasm-apps/bulk-memory/test_bulk_memory.wat @@ -0,0 +1,12 @@ +(module + (memory 1) + + (func $memory_fill_test (param $dst i32) (param $val i32) (param $len i32) + local.get $dst + local.get $val + local.get $len + memory.fill + ) + + (export "memory_fill_test" (func $memory_fill_test)) +) diff --git a/tests/unit/shared-heap/wasm-apps/memory64/CMakeLists.txt b/tests/unit/shared-heap/wasm-apps/memory64/CMakeLists.txt new file mode 100644 index 0000000000..a82788b586 --- /dev/null +++ b/tests/unit/shared-heap/wasm-apps/memory64/CMakeLists.txt @@ -0,0 +1,68 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required(VERSION 3.14) +project(wasm-apps-wasm64) + +set(WAMR_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../../../..) +set(WAMRC_ROOT_DIR ${WAMR_ROOT_DIR}/wamr-compiler/build) + +set(CMAKE_SYSTEM_PROCESSOR wasm64) +set(CMAKE_SYSROOT ${WAMR_ROOT_DIR}/wamr-sdk/app/libc-builtin-sysroot) + +if (NOT DEFINED WASI_SDK_DIR) + set(WASI_SDK_DIR "/opt/wasi-sdk") +endif () + +set(CMAKE_C_FLAGS "-nostdlib -pthread -Qunused-arguments") +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -z stack-size=8192 -nostdlib -O0 --target=wasm64") +set(CMAKE_C_COMPILER_TARGET "wasm64") +set(CMAKE_C_COMPILER "${WASI_SDK_DIR}/bin/clang") + +set(DEFINED_SYMBOLS + "${WAMR_ROOT_DIR}/wamr-sdk/app/libc-builtin-sysroot/share/defined-symbols.txt") + +set(CMAKE_EXE_LINKER_FLAGS + "-Wl,--no-entry \ + -Wl,--initial-memory=65536 \ + -Wl,--export-all \ + -Wl,--allow-undefined" + ) + +set (WAMR_COMPILER_FLAGS --opt-level=3 --bounds-checks=1 --enable-shared-heap) +set (WAMR_COMPILER_CHAIN_FLAGS --opt-level=3 --bounds-checks=1 --enable-shared-chain) + +function(copy_wasm TARGET_NAME) + add_custom_command(TARGET ${TARGET_NAME} POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy + ${CMAKE_CURRENT_BINARY_DIR}/${TARGET_NAME} + ${CMAKE_CURRENT_BINARY_DIR}/../../ + COMMENT "Copy ${TARGET_NAME} to the same directory of google test" + ) +endfunction() + +function(compile_and_copy_aot_from TARGET_NAME) + string(REPLACE ".wasm" ".aot" AOT_TARGET ${TARGET_NAME}) + string(REPLACE ".wasm" "_chain.aot" AOT_CHAIN_TARGET ${TARGET_NAME}) + + add_custom_command(TARGET ${TARGET_NAME} POST_BUILD + COMMAND ${WAMRC_ROOT_DIR}/wamrc ${WAMR_COMPILER_FLAGS} + -o ${AOT_TARGET} + ${TARGET_NAME} + COMMAND ${CMAKE_COMMAND} -E copy + ${CMAKE_CURRENT_BINARY_DIR}/${AOT_TARGET} + ${CMAKE_CURRENT_BINARY_DIR}/../../ + COMMAND ${WAMRC_ROOT_DIR}/wamrc ${WAMR_COMPILER_CHAIN_FLAGS} + -o ${AOT_CHAIN_TARGET} + ${TARGET_NAME} + COMMAND ${CMAKE_COMMAND} -E copy + ${CMAKE_CURRENT_BINARY_DIR}/${AOT_CHAIN_TARGET} + ${CMAKE_CURRENT_BINARY_DIR}/../../ + COMMENT "Compile and copy ${AOT_TARGET} ${AOT_CHAIN_TARGET} to the same directory of google test" + ) +endfunction() + +add_executable(test64.wasm ../test.c) +target_link_libraries(test64.wasm) +copy_wasm(test64.wasm) +compile_and_copy_aot_from(test64.wasm) diff --git a/tests/unit/shared-heap/wasm-apps/test.c b/tests/unit/shared-heap/wasm-apps/test.c index b0bfc17b36..66df21c1b7 100644 --- a/tests/unit/shared-heap/wasm-apps/test.c +++ b/tests/unit/shared-heap/wasm-apps/test.c @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception */ -#include +#define NULL 0 extern void * shared_heap_malloc(int size); diff --git a/tests/unit/shared-heap/wasm-apps/test_addr_conv.c b/tests/unit/shared-heap/wasm-apps/test_addr_conv.c index 5e64526a04..a00e68c9ce 100644 --- a/tests/unit/shared-heap/wasm-apps/test_addr_conv.c +++ b/tests/unit/shared-heap/wasm-apps/test_addr_conv.c @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception */ -#include +#define NULL 0 extern void * shared_heap_malloc(int size); diff --git a/wamr-compiler/CMakeLists.txt b/wamr-compiler/CMakeLists.txt index 0ce6473944..68648b84b7 100644 --- a/wamr-compiler/CMakeLists.txt +++ b/wamr-compiler/CMakeLists.txt @@ -284,6 +284,7 @@ include (${IWASM_DIR}/interpreter/iwasm_interp.cmake) include (${IWASM_DIR}/aot/iwasm_aot.cmake) include (${IWASM_DIR}/compilation/iwasm_compl.cmake) include (${PROJECT_SOURCE_DIR}/../build-scripts/version.cmake) +include (${IWASM_DIR}/libraries/shared-heap/shared_heap.cmake) if (WAMR_BUILD_LIBC_BUILTIN EQUAL 1) include (${IWASM_DIR}/libraries/libc-builtin/libc_builtin.cmake) @@ -366,6 +367,7 @@ add_library (vmlib ${LIBC_WASI_SOURCE} ${LIB_PTHREAD_SOURCE} ${LIB_WASI_THREADS_SOURCE} + ${LIB_SHARED_HEAP_SOURCE} ${IWASM_COMMON_SOURCE} ${IWASM_INTERP_SOURCE} ${IWASM_AOT_SOURCE} diff --git a/wamr-compiler/main.c b/wamr-compiler/main.c index f74b2fb151..6b7dbaad48 100644 --- a/wamr-compiler/main.c +++ b/wamr-compiler/main.c @@ -210,7 +210,9 @@ print_help() printf(" --enable-linux-perf Enable linux perf support\n"); #endif printf(" --mllvm=