From d0d4ead5ba97c4d9598feeef25f0b2b1a5e79b20 Mon Sep 17 00:00:00 2001 From: Jakub Radtke Date: Thu, 28 Feb 2019 02:36:58 -0500 Subject: [PATCH 01/10] Use separate DPDK submodule for compilation Required for DPDK transport in eRPC Signed-off-by: Jakub Radtke --- .gitmodules | 5 + CMakeLists.txt | 2 +- scripts/libdpdk.mri | 114 +++++++++++++++++++++ scripts/libspdk.mri | 15 --- scripts/prepare_dpdk_libs.sh | 11 ++ tests/functional_thin/tests/tests_base.cpp | 2 +- third-party/CMakeLists.txt | 2 + third-party/dpdk | 1 + third-party/dpdk.cmake | 21 ++++ third-party/erpc.cmake | 4 +- third-party/spdk.cmake | 8 +- 11 files changed, 161 insertions(+), 24 deletions(-) create mode 100644 scripts/libdpdk.mri create mode 100755 scripts/prepare_dpdk_libs.sh create mode 160000 third-party/dpdk create mode 100644 third-party/dpdk.cmake diff --git a/.gitmodules b/.gitmodules index ee09c6a..2a9f751 100644 --- a/.gitmodules +++ b/.gitmodules @@ -29,3 +29,8 @@ [submodule "third-party/eRPC-fork"] path = third-party/eRPC-fork url = https://github.com/daq-db/eRPC.git + ignore = dirty +[submodule "third-party/dpdk"] + path = third-party/dpdk + url = https://github.com/DPDK/dpdk.git + ignore = dirty diff --git a/CMakeLists.txt b/CMakeLists.txt index 35ff65f..b57ed1e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -73,7 +73,7 @@ add_subdirectory(${3RDPARTY}) include_directories(${LIBCONFIG_INCLUDES_EXPORT}) include_directories(${3RDPARTY}/eRPC-fork/src) -include_directories(${3RDPARTY}/spdk/dpdk/build/include) +include_directories(${3RDPARTY}/dpdk/build/include) file(GLOB_RECURSE DAQDB_THIN_SOURCES lib/thin/*.cpp lib/common/*.cpp lib/dht/*.cpp lib/KVStoreBase.cpp) diff --git a/scripts/libdpdk.mri b/scripts/libdpdk.mri new file mode 100644 index 0000000..b92c9b0 --- /dev/null +++ b/scripts/libdpdk.mri @@ -0,0 +1,114 @@ +verbose +create libdpdk.a +addlib build/lib/librte_bbdev.a +addlib build/lib/librte_bitratestats.a +addlib build/lib/librte_bpf.a +addlib build/lib/librte_bus_dpaa.a +addlib build/lib/librte_bus_fslmc.a +addlib build/lib/librte_bus_ifpga.a +addlib build/lib/librte_bus_pci.a +addlib build/lib/librte_bus_vdev.a +addlib build/lib/librte_bus_vmbus.a +addlib build/lib/librte_cfgfile.a +addlib build/lib/librte_cmdline.a +addlib build/lib/librte_common_cpt.a +addlib build/lib/librte_common_dpaax.a +addlib build/lib/librte_common_octeontx.a +addlib build/lib/librte_compressdev.a +addlib build/lib/librte_cryptodev.a +addlib build/lib/librte_distributor.a +addlib build/lib/librte_eal.a +addlib build/lib/librte_efd.a +addlib build/lib/librte_ethdev.a +addlib build/lib/librte_eventdev.a +addlib build/lib/librte_flow_classify.a +addlib build/lib/librte_gro.a +addlib build/lib/librte_gso.a +addlib build/lib/librte_hash.a +addlib build/lib/librte_ip_frag.a +addlib build/lib/librte_ipsec.a +addlib build/lib/librte_jobstats.a +addlib build/lib/librte_kni.a +addlib build/lib/librte_kvargs.a +addlib build/lib/librte_latencystats.a +addlib build/lib/librte_lpm.a +addlib build/lib/librte_mbuf.a +addlib build/lib/librte_member.a +addlib build/lib/librte_mempool.a +addlib build/lib/librte_mempool_bucket.a +addlib build/lib/librte_mempool_dpaa2.a +addlib build/lib/librte_mempool_dpaa.a +addlib build/lib/librte_mempool_octeontx.a +addlib build/lib/librte_mempool_ring.a +addlib build/lib/librte_mempool_stack.a +addlib build/lib/librte_meter.a +addlib build/lib/librte_metrics.a +addlib build/lib/librte_net.a +addlib build/lib/librte_pci.a +addlib build/lib/librte_pdump.a +addlib build/lib/librte_pipeline.a +addlib build/lib/librte_pmd_af_packet.a +addlib build/lib/librte_pmd_ark.a +addlib build/lib/librte_pmd_atlantic.a +addlib build/lib/librte_pmd_avf.a +addlib build/lib/librte_pmd_avp.a +addlib build/lib/librte_pmd_axgbe.a +addlib build/lib/librte_pmd_bbdev_null.a +addlib build/lib/librte_pmd_bnxt.a +addlib build/lib/librte_pmd_bond.a +addlib build/lib/librte_pmd_caam_jr.a +addlib build/lib/librte_pmd_crypto_scheduler.a +addlib build/lib/librte_pmd_cxgbe.a +addlib build/lib/librte_pmd_dpaa2.a +addlib build/lib/librte_pmd_dpaa2_cmdif.a +addlib build/lib/librte_pmd_dpaa2_event.a +addlib build/lib/librte_pmd_dpaa2_qdma.a +addlib build/lib/librte_pmd_dpaa2_sec.a +addlib build/lib/librte_pmd_dpaa.a +addlib build/lib/librte_pmd_dpaa_event.a +addlib build/lib/librte_pmd_dpaa_sec.a +addlib build/lib/librte_pmd_dsw_event.a +addlib build/lib/librte_pmd_e1000.a +addlib build/lib/librte_pmd_ena.a +addlib build/lib/librte_pmd_enetc.a +addlib build/lib/librte_pmd_enic.a +addlib build/lib/librte_pmd_failsafe.a +addlib build/lib/librte_pmd_fm10k.a +addlib build/lib/librte_pmd_i40e.a +addlib build/lib/librte_pmd_ice.a +addlib build/lib/librte_pmd_ifpga_rawdev.a +addlib build/lib/librte_pmd_ixgbe.a +addlib build/lib/librte_pmd_kni.a +addlib build/lib/librte_pmd_lio.a +addlib build/lib/librte_pmd_netvsc.a +addlib build/lib/librte_pmd_nfp.a +addlib build/lib/librte_pmd_null.a +addlib build/lib/librte_pmd_null_crypto.a +addlib build/lib/librte_pmd_octeontx.a +addlib build/lib/librte_pmd_octeontx_crypto.a +addlib build/lib/librte_pmd_octeontx_ssovf.a +addlib build/lib/librte_pmd_octeontx_zip.a +addlib build/lib/librte_pmd_opdl_event.a +addlib build/lib/librte_pmd_qat.a +addlib build/lib/librte_pmd_qede.a +addlib build/lib/librte_pmd_ring.a +addlib build/lib/librte_pmd_sfc_efx.a +addlib build/lib/librte_pmd_skeleton_event.a +addlib build/lib/librte_pmd_skeleton_rawdev.a +addlib build/lib/librte_pmd_sw_event.a +addlib build/lib/librte_pmd_tap.a +addlib build/lib/librte_pmd_thunderx_nicvf.a +addlib build/lib/librte_pmd_vdev_netvsc.a +addlib build/lib/librte_pmd_virtio.a +addlib build/lib/librte_pmd_virtio_crypto.a +addlib build/lib/librte_pmd_vmxnet3_uio.a +addlib build/lib/librte_port.a +addlib build/lib/librte_power.a +addlib build/lib/librte_rawdev.a +addlib build/lib/librte_reorder.a +addlib build/lib/librte_ring.a +addlib build/lib/librte_sched.a +addlib build/lib/librte_security.a +addlib build/lib/librte_timer.a +save +end diff --git a/scripts/libspdk.mri b/scripts/libspdk.mri index 9bc449c..8ff3a4c 100644 --- a/scripts/libspdk.mri +++ b/scripts/libspdk.mri @@ -46,18 +46,3 @@ addlib build/lib/libspdk_util.a addlib build/lib/libspdk_vhost.a addlib build/lib/libspdk_virtio.a save -create libdpdk.a -addlib dpdk/build/lib/librte_bus_pci.a -addlib dpdk/build/lib/librte_bus_vdev.a -addlib dpdk/build/lib/librte_eal.a -addlib dpdk/build/lib/librte_ethdev.a -addlib dpdk/build/lib/librte_kvargs.a -addlib dpdk/build/lib/librte_mbuf.a -addlib dpdk/build/lib/librte_mempool.a -addlib dpdk/build/lib/librte_mempool_bucket.a -addlib dpdk/build/lib/librte_mempool_ring.a -addlib dpdk/build/lib/librte_net.a -addlib dpdk/build/lib/librte_pci.a -addlib dpdk/build/lib/librte_ring.a -save -end diff --git a/scripts/prepare_dpdk_libs.sh b/scripts/prepare_dpdk_libs.sh new file mode 100755 index 0000000..301073f --- /dev/null +++ b/scripts/prepare_dpdk_libs.sh @@ -0,0 +1,11 @@ +#!/bin/sh +set -x +SCRIPT=$(readlink -f $0) +SCRIPTPATH=`dirname $SCRIPT` +DPDKPATH=third-party/dpdk + +cd $SCRIPTPATH/../$DPDKPATH +rm -f x86_64-native-linuxapp-gcc/lib/libdpdk.a +ar -M <$SCRIPTPATH/libdpdk.mri +cd - +set +x diff --git a/tests/functional_thin/tests/tests_base.cpp b/tests/functional_thin/tests/tests_base.cpp index 8c7da70..979e244 100644 --- a/tests/functional_thin/tests/tests_base.cpp +++ b/tests/functional_thin/tests/tests_base.cpp @@ -18,9 +18,9 @@ #include "tests.h" +#include "../base_operations.h" #include #include -#include "../base_operations.h" using namespace std; using namespace DaqDB; diff --git a/third-party/CMakeLists.txt b/third-party/CMakeLists.txt index e9f2f51..2515fd0 100644 --- a/third-party/CMakeLists.txt +++ b/third-party/CMakeLists.txt @@ -13,6 +13,7 @@ set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${DAQDB_BIN_DIR}) if(NOT THIN_LIB) include(pmdk.cmake) + include(dpdk.cmake) include(spdk.cmake) else(NOT THIN_LIB) if(NOT ERPC_RAW_TRANSPORT) @@ -44,4 +45,5 @@ add_custom_target(clean-dep) if(NOT THIN_LIB) add_dependencies(clean-dep libpmdk_clean) add_dependencies(clean-dep libspdk_clean) + add_dependencies(clean-dep libdpdk_clean) endif() diff --git a/third-party/dpdk b/third-party/dpdk new file mode 160000 index 0000000..8d47a75 --- /dev/null +++ b/third-party/dpdk @@ -0,0 +1 @@ +Subproject commit 8d47a753b7cb05e0ef62360324c6a28505038d09 diff --git a/third-party/dpdk.cmake b/third-party/dpdk.cmake new file mode 100644 index 0000000..f954faa --- /dev/null +++ b/third-party/dpdk.cmake @@ -0,0 +1,21 @@ +cmake_minimum_required(VERSION 3.5) + +include(ExternalProject) + +ExternalProject_Add(project_dpdk + PREFIX ${PROJECT_SOURCE_DIR}/dpdk + SOURCE_DIR ${PROJECT_SOURCE_DIR}/dpdk + BUILD_IN_SOURCE ${PROJECT_SOURCE_DIR}/dpdk + CONFIGURE_COMMAND ${CMAKE_MAKE_PROGRAM} config T=x86_64-native-linuxapp-gcc + BUILD_COMMAND ${CMAKE_MAKE_PROGRAM} + INSTALL_COMMAND ${ROOT_DAQDB_DIR}/scripts/prepare_dpdk_libs.sh +) +add_library(dpdk STATIC IMPORTED GLOBAL) +set_property(TARGET dpdk PROPERTY IMPORTED_LOCATION ${PROJECT_SOURCE_DIR}/dpdk/libdpdk.a) +add_dependencies(dpdk project_dpdk) + +add_custom_target(libdpdk_clean + COMMAND ${CMAKE_MAKE_PROGRAM} clean + COMMAND rm -f ${PROJECT_SOURCE_DIR}/dpdk/libdpdk.a + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/dpdk +) diff --git a/third-party/erpc.cmake b/third-party/erpc.cmake index 5af0eb2..10fe82a 100644 --- a/third-party/erpc.cmake +++ b/third-party/erpc.cmake @@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.5) include(ExternalProject) -set(DPDK_PATH ${PROJECT_SOURCE_DIR}/spdk/dpdk/build) +set(DPDK_PATH ${PROJECT_SOURCE_DIR}/dpdk/build) ExternalProject_Add(project_erpc PREFIX ${PROJECT_SOURCE_DIR}/eRPC-fork @@ -11,7 +11,7 @@ ExternalProject_Add(project_erpc CMAKE_ARGS -DPERF=ON -DTRANSPORT=${ERPC_TRANSPORT_MODE} -DLTO=OFF -DCMAKE_PREFIX_PATH=${DPDK_PATH} -DCMAKE_POSITION_INDEPENDENT_CODE=ON BUILD_COMMAND ${CMAKE_MAKE_PROGRAM} INSTALL_COMMAND "" - DEPENDS project_spdk + DEPENDS project_dpdk ) add_library(liberpc STATIC IMPORTED GLOBAL) set_target_properties(liberpc PROPERTIES IMPORTED_LOCATION diff --git a/third-party/spdk.cmake b/third-party/spdk.cmake index f5f2d73..74441c9 100644 --- a/third-party/spdk.cmake +++ b/third-party/spdk.cmake @@ -6,20 +6,18 @@ ExternalProject_Add(project_spdk PREFIX ${PROJECT_SOURCE_DIR}/spdk SOURCE_DIR ${PROJECT_SOURCE_DIR}/spdk BUILD_IN_SOURCE ${PROJECT_SOURCE_DIR}/spdk - CONFIGURE_COMMAND "./configure" + CONFIGURE_COMMAND "./configure" --with-dpdk=${PROJECT_SOURCE_DIR}/dpdk/build --with-igb-uio-driver BUILD_COMMAND ${CMAKE_MAKE_PROGRAM} INSTALL_COMMAND ${ROOT_DAQDB_DIR}/scripts/prepare_spdk_libs.sh ) +add_dependencies(project_spdk project_dpdk) + add_library(spdk STATIC IMPORTED GLOBAL) set_property(TARGET spdk PROPERTY IMPORTED_LOCATION ${PROJECT_SOURCE_DIR}/spdk/libspdk.a) add_dependencies(spdk project_spdk) -add_library(dpdk STATIC IMPORTED GLOBAL) -set_property(TARGET dpdk PROPERTY IMPORTED_LOCATION ${PROJECT_SOURCE_DIR}/spdk/libdpdk.a) -add_dependencies(dpdk project_spdk) add_custom_target(libspdk_clean COMMAND ${CMAKE_MAKE_PROGRAM} clean COMMAND rm -f ${PROJECT_SOURCE_DIR}/spdk/libspdk.a - COMMAND rm -f ${PROJECT_SOURCE_DIR}/spdk/libdpdk.a WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/spdk ) From 9f8d1df1add4324c008e4d41f61c60c093e49b2d Mon Sep 17 00:00:00 2001 From: Jakub Radtke Date: Fri, 1 Mar 2019 12:25:46 +0100 Subject: [PATCH 02/10] Minor fixes in functional tests (#275) Signed-off-by: Jakub Radtke --- tests/functional/tests/tests_base.cpp | 26 +++++++------- tests/functional/tests/tests_offload.cpp | 44 +++++++++++++----------- tests/functional_thin/main.cpp | 12 +++---- 3 files changed, 43 insertions(+), 39 deletions(-) diff --git a/tests/functional/tests/tests_base.cpp b/tests/functional/tests/tests_base.cpp index b52789b..09067d1 100644 --- a/tests/functional/tests/tests_base.cpp +++ b/tests/functional/tests/tests_base.cpp @@ -54,10 +54,10 @@ bool testAsyncOperations(KVStoreBase *kvs) { condition_variable cv; bool ready = false; - daqdb_async_put(kvs, keyId, val, - [&](KVStoreBase *kvs, Status status, const char *argKey, - const size_t keySize, const char *value, - const size_t valueSize) { + daqdb_async_put( + kvs, keyId, val, + [&](KVStoreBase *kvs, Status status, const char *argKey, + const size_t keySize, const char *value, const size_t valueSize) { unique_lock lck(mtx); if (status.ok()) { DAQDB_INFO << boost::format("PutAsync: [%1%]") % @@ -69,7 +69,7 @@ bool testAsyncOperations(KVStoreBase *kvs) { } ready = true; cv.notify_all(); - }); + }); // wait for completion { @@ -83,15 +83,17 @@ bool testAsyncOperations(KVStoreBase *kvs) { result = false; } - daqdb_async_get(kvs, keyId, - [&](KVStoreBase *kvs, Status status, const char *argKey, - size_t keySize, const char *value, size_t valueSize) { + daqdb_async_get( + kvs, keyId, + [&](KVStoreBase *kvs, Status status, const char *argKey, size_t keySize, + const char *value, size_t valueSize) { unique_lock lck(mtx); if (status.ok()) { - DAQDB_INFO << boost::format("GetAsync: [%1%] = %2%") % - keyToStr(argKey) % value; - if (!value || val.compare(value) != 0) { + DAQDB_INFO << boost::format("GetAsync: [%1%]") % + keyToStr(argKey); + + if (!value || memcmp(val.data(), value, val.size())) { DAQDB_INFO << "Error: wrong value returned" << flush; result = false; } @@ -103,7 +105,7 @@ bool testAsyncOperations(KVStoreBase *kvs) { ready = true; cv.notify_all(); - }); + }); // wait for completion { diff --git a/tests/functional/tests/tests_offload.cpp b/tests/functional/tests/tests_offload.cpp index 8ff785c..d450a03 100644 --- a/tests/functional/tests/tests_offload.cpp +++ b/tests/functional/tests/tests_offload.cpp @@ -74,10 +74,10 @@ bool testAsyncOffloadOperations(KVStoreBase *kvs) { condition_variable cv; bool ready = false; - daqdb_async_put(kvs, keyId, val, - [&](KVStoreBase *kvs, Status status, const char *argKey, - const size_t keySize, const char *value, - const size_t valueSize) { + daqdb_async_put( + kvs, keyId, val, + [&](KVStoreBase *kvs, Status status, const char *argKey, + const size_t keySize, const char *value, const size_t valueSize) { unique_lock lck(mtx); if (status.ok()) { DAQDB_INFO << boost::format("PutAsync: [%1%]") % @@ -89,7 +89,7 @@ bool testAsyncOffloadOperations(KVStoreBase *kvs) { } ready = true; cv.notify_all(); - }); + }); // wait for completion { @@ -109,15 +109,16 @@ bool testAsyncOffloadOperations(KVStoreBase *kvs) { result = false; } - daqdb_async_get(kvs, keyId, - [&](KVStoreBase *kvs, Status status, const char *argKey, - size_t keySize, const char *value, size_t valueSize) { + daqdb_async_get( + kvs, keyId, + [&](KVStoreBase *kvs, Status status, const char *argKey, size_t keySize, + const char *value, size_t valueSize) { unique_lock lck(mtx); if (status.ok()) { - DAQDB_INFO << boost::format("GetAsync: [%1%] = %2%") % - keyToStr(argKey) % value; - if (!value || val.compare(value) != 0) { + DAQDB_INFO << boost::format("GetAsync: [%1%]") % + keyToStr(argKey); + if (!value || memcmp(val.data(), value, val.size())) { DAQDB_INFO << "Error: wrong value returned" << flush; result = false; } @@ -129,7 +130,7 @@ bool testAsyncOffloadOperations(KVStoreBase *kvs) { ready = true; cv.notify_all(); - }); + }); // wait for completion { @@ -138,10 +139,10 @@ bool testAsyncOffloadOperations(KVStoreBase *kvs) { ready = false; } - daqdb_async_offload(kvs, keyId, - [&](KVStoreBase *kvs, Status status, const char *argKey, - const size_t keySize, const char *value, - const size_t valueSize) { + daqdb_async_offload( + kvs, keyId, + [&](KVStoreBase *kvs, Status status, const char *argKey, + const size_t keySize, const char *value, const size_t valueSize) { unique_lock lck(mtx); if (status.ok()) { DAQDB_INFO << boost::format("Offload: [%1%]") % @@ -153,7 +154,7 @@ bool testAsyncOffloadOperations(KVStoreBase *kvs) { } ready = true; cv.notify_all(); - }); + }); // wait for completion { @@ -172,9 +173,10 @@ bool testAsyncOffloadOperations(KVStoreBase *kvs) { result = false; } - daqdb_async_get(kvs, keyId, - [&](KVStoreBase *kvs, Status status, const char *argKey, - size_t keySize, const char *value, size_t valueSize) { + daqdb_async_get( + kvs, keyId, + [&](KVStoreBase *kvs, Status status, const char *argKey, size_t keySize, + const char *value, size_t valueSize) { unique_lock lck(mtx); if (status.ok()) { @@ -192,7 +194,7 @@ bool testAsyncOffloadOperations(KVStoreBase *kvs) { ready = true; cv.notify_all(); - }); + }); // wait for completion { diff --git a/tests/functional_thin/main.cpp b/tests/functional_thin/main.cpp index 3ed2279..49e6017 100644 --- a/tests/functional_thin/main.cpp +++ b/tests/functional_thin/main.cpp @@ -11,7 +11,7 @@ * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and - * limitations under the License. + * limitations under the License. */ #include @@ -115,11 +115,11 @@ int main(int argc, const char *argv[]) { if (!executeTest(test.first, test.second, spKVStore.get(), &options)) { failsCount++; } + } - if (failsCount > 0) { - DAQDB_INFO << format("Test(s) failed [%1%]") % failsCount << endl; - } else { - DAQDB_INFO << "All tests passed!" << endl; - } + if (failsCount > 0) { + DAQDB_INFO << format("Test(s) failed [%1%]") % failsCount << endl; + } else { + DAQDB_INFO << "All tests passed!" << endl; } } From dab7d08a524737591ed78fd5b92ffb855ac67add Mon Sep 17 00:00:00 2001 From: Maciej Maciejewski Date: Sun, 3 Mar 2019 12:16:13 +0100 Subject: [PATCH 03/10] improved memory copy in minidaq --- apps/minidaq/MinidaqAroNode.cpp | 3 ++- apps/minidaq/MinidaqRoNode.cpp | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/apps/minidaq/MinidaqAroNode.cpp b/apps/minidaq/MinidaqAroNode.cpp index c6a83ca..f51fd7c 100644 --- a/apps/minidaq/MinidaqAroNode.cpp +++ b/apps/minidaq/MinidaqAroNode.cpp @@ -15,6 +15,7 @@ */ #include "MinidaqAroNode.h" +#include namespace DaqDB { @@ -35,7 +36,7 @@ void MinidaqAroNode::_Task(Key &&key, std::atomic &cnt, throw; } - memcpy(value.data(), _data_buffer, value.size()); + pmem_memcpy_nodrain(value.data(), _data_buffer, value.size()); #ifdef WITH_INTEGRITY_CHECK _FillBuffer(key, value.data(), value.size()); diff --git a/apps/minidaq/MinidaqRoNode.cpp b/apps/minidaq/MinidaqRoNode.cpp index 11c1a66..99654e6 100644 --- a/apps/minidaq/MinidaqRoNode.cpp +++ b/apps/minidaq/MinidaqRoNode.cpp @@ -15,6 +15,8 @@ */ #include "MinidaqRoNode.h" +#include +#include namespace DaqDB { @@ -51,7 +53,7 @@ void MinidaqRoNode::_Task(Key &&key, std::atomic &cnt, throw; } - memcpy(value.data(), _data_buffer, value.size()); + pmem_memcpy_nodrain(value.data(), _data_buffer, value.size()); #ifdef WITH_INTEGRITY_CHECK _FillBuffer(key, value.data(), value.size()); From 9769f3379e4eb711d983391c70ba52db24f11be0 Mon Sep 17 00:00:00 2001 From: Maciej Maciejewski Date: Sun, 3 Mar 2019 12:40:35 +0100 Subject: [PATCH 04/10] review comment --- apps/minidaq/MinidaqRoNode.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/apps/minidaq/MinidaqRoNode.cpp b/apps/minidaq/MinidaqRoNode.cpp index 99654e6..c65e5c7 100644 --- a/apps/minidaq/MinidaqRoNode.cpp +++ b/apps/minidaq/MinidaqRoNode.cpp @@ -15,7 +15,6 @@ */ #include "MinidaqRoNode.h" -#include #include namespace DaqDB { From 626fc47c35d68b01b3d3b3b4e59919c0c10bab7a Mon Sep 17 00:00:00 2001 From: Jakub Radtke Date: Tue, 5 Mar 2019 11:34:26 -0500 Subject: [PATCH 05/10] Update daqdb_thin and functest_thin to work with DPDK transport Signed-off-by: Jakub Radtke --- CMakeLists.txt | 29 +++++++++++++++++++++------ lib/core/KVStore.cpp | 23 +++++++++------------- lib/dht/DhtServer.cpp | 36 ++++++++++++++++++---------------- lib/thin/KVStoreThin.cpp | 6 +++++- lib/thin/KVStoreThin.h | 12 ++++++++++-- tests/functional_thin/main.cpp | 2 ++ 6 files changed, 68 insertions(+), 40 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b57ed1e..2263fc1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -74,15 +74,32 @@ add_subdirectory(${3RDPARTY}) include_directories(${LIBCONFIG_INCLUDES_EXPORT}) include_directories(${3RDPARTY}/eRPC-fork/src) include_directories(${3RDPARTY}/dpdk/build/include) - -file(GLOB_RECURSE DAQDB_THIN_SOURCES lib/thin/*.cpp lib/common/*.cpp - lib/dht/*.cpp lib/KVStoreBase.cpp) -list(FILTER DAQDB_THIN_SOURCES EXCLUDE REGEX "lib/dht/.*Server.*\.cpp$") include_directories(lib/common lib/dht lib/thin) include_directories(${HDRHISTOGRAM_INCLUDES_EXPORT}) -add_library(daqdb_thin SHARED ${DAQDB_THIN_SOURCES}) + +if(ERPC_RAW_TRANSPORT) + file(GLOB_RECURSE DAQDB_THIN_SOURCES lib/thin/*.cpp lib/common/*.cpp + lib/dht/*.cpp lib/KVStoreBase.cpp) + list(FILTER DAQDB_THIN_SOURCES EXCLUDE REGEX "lib/dht/.*Server.*\.cpp$") + + add_library(daqdb_thin SHARED ${DAQDB_THIN_SOURCES}) + target_link_libraries(daqdb_thin ${Boost_LIBRARIES} libconfig ${ERPC_LIBS}) + +else(ERPC_RAW_TRANSPORT) + include_directories(lib/offload lib/pmem lib/core lib/spdk lib/primary + lib/spdk ${3RDPARTY}/spdk/include) + + file(GLOB_RECURSE DAQDB_THIN_SOURCES lib/thin/*.cpp lib/common/*.cpp + lib/dht/*.cpp lib/KVStoreBase.cpp lib/spdk/*.cpp) + list(FILTER DAQDB_THIN_SOURCES EXCLUDE REGEX "lib/dht/.*Server.*\.cpp$") + set(Spdk_LIBRARIES -Wl,--whole-archive spdk -Wl,--no-whole-archive pthread + rt uuid) + + add_library(daqdb_thin SHARED ${DAQDB_THIN_SOURCES}) + target_link_libraries(daqdb_thin ${Spdk_LIBRARIES} ${Boost_LIBRARIES} libconfig ${ERPC_LIBS}) +endif(ERPC_RAW_TRANSPORT) + target_compile_definitions(daqdb_thin PRIVATE THIN_LIB=1) -target_link_libraries(daqdb_thin ${Boost_LIBRARIES} libconfig ${ERPC_LIBS}) if(NOT THIN_LIB) include_directories(${3RDPARTY}/pmdk/src/include) diff --git a/lib/core/KVStore.cpp b/lib/core/KVStore.cpp index 5ddbb62..3c7b165 100644 --- a/lib/core/KVStore.cpp +++ b/lib/core/KVStore.cpp @@ -11,7 +11,7 @@ * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and - * limitations under the License. + * limitations under the License. */ #include "KVStore.h" @@ -100,13 +100,14 @@ void KVStore::init() { new DhtServer(getDhtCore(), this, DHT_SERVER_WORKER_THREADS)); if (_spDhtServer->state == DhtServerState::DHT_SERVER_READY) { DAQ_DEBUG("DHT server started successfully"); + + if (_spDht->getLocalNode()->getPeerPort() > 0) { + _spDht->initNexus(_spDht->getLocalNode()->getPeerPort()); + _spDht->initClient(); + } } else { DAQ_DEBUG("Can not start DHT server"); } - if (_spDht->getLocalNode()->getPeerPort() > 0) { - _spDht->initNexus(_spDht->getLocalNode()->getPeerPort()); - _spDht->initClient(); - } if (isOffloadEnabled()) { _spOffloadPoller.reset(new DaqDB::OffloadPoller( @@ -623,16 +624,10 @@ std::string KVStore::getProperty(const std::string &name) { return ""; } -uint64_t KVStore::GetTreeSize() { - return pmem()->GetTreeSize(); -} +uint64_t KVStore::GetTreeSize() { return pmem()->GetTreeSize(); } -uint8_t KVStore::GetTreeDepth() { - return pmem()->GetTreeDepth(); -} +uint8_t KVStore::GetTreeDepth() { return pmem()->GetTreeDepth(); } -uint64_t KVStore::GetLeafCount() { - return pmem()->GetLeafCount(); -} +uint64_t KVStore::GetLeafCount() { return pmem()->GetLeafCount(); } } // namespace DaqDB diff --git a/lib/dht/DhtServer.cpp b/lib/dht/DhtServer.cpp index 1cadb4f..64ad836 100644 --- a/lib/dht/DhtServer.cpp +++ b/lib/dht/DhtServer.cpp @@ -11,7 +11,7 @@ * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and - * limitations under the License. + * limitations under the License. */ #include "DhtServer.h" @@ -30,9 +30,9 @@ #define DHT_SERVER_CPU_CORE_MAX 32 /** - * @TODO jradtke: not needed when eRPC implements configurable size of - * pre_resp_msgbuf - */ + * @TODO jradtke: not needed when eRPC implements configurable size of + * pre_resp_msgbuf + */ const unsigned int PRE_BUF_SIZE = 32 * 1024; namespace DaqDB { @@ -71,9 +71,9 @@ static void erpcReqGetAnyHandler(erpc::ReqHandle *req_handle, void *ctx) { erpc::MsgBuffer *resp; try { - resp = - erpcPrepareMsgbuf(rpc, req_handle, sizeof(DaqdbDhtResult) + - serverCtx->kvs->KeySize()); + resp = erpcPrepareMsgbuf(rpc, req_handle, + sizeof(DaqdbDhtResult) + + serverCtx->kvs->KeySize()); DaqdbDhtResult *result = reinterpret_cast(resp->buf); result->msgSize = serverCtx->kvs->KeySize(); serverCtx->kvs->GetAny(result->msg, serverCtx->kvs->KeySize()); @@ -182,11 +182,11 @@ DhtServer::~DhtServer() { _thread->join(); } -void DhtServer::_serveWorker(unsigned int workerId, cpu_set_t *cpuset, size_t size) { +void DhtServer::_serveWorker(unsigned int workerId, cpu_set_t *cpuset, + size_t size) { DhtServerCtx rpcCtx; - const int set_result = pthread_setaffinity_np(pthread_self(), - size, cpuset); + const int set_result = pthread_setaffinity_np(pthread_self(), size, cpuset); if (!set_result) { DAQ_DEBUG("Cannot set affinity for DHT server worker[" + to_string(workerId) + "]"); @@ -225,8 +225,7 @@ void DhtServer::_serve(void) { CPU_ZERO_S(size, cpuset); CPU_SET_S(DHT_SERVER_CPU_CORE_BASE, size, cpuset); - const int set_result = pthread_setaffinity_np(pthread_self(), - size, cpuset); + const int set_result = pthread_setaffinity_np(pthread_self(), size, cpuset); if (!set_result) { DAQ_DEBUG("Cannot set affinity for DHT server thread"); } @@ -259,7 +258,7 @@ void DhtServer::_serve(void) { CPU_ZERO_S(size, cpuset); CPU_SET_S(DHT_SERVER_CPU_CORE_BASE + threadIndex, size, cpuset); _workerThreads.push_back(thread(&DhtServer::_serveWorker, this, - threadIndex, cpuset, size)); + threadIndex, cpuset, size)); } state = DhtServerState::DHT_SERVER_READY; @@ -276,19 +275,22 @@ void DhtServer::_serve(void) { if (rpc) { delete rpc; } + CPU_FREE(cpuset); } catch (exception &e) { DAQ_DEBUG("DHT server exception: " + std::string(e.what())); - state = DhtServerState::DHT_SERVER_ERROR; + CPU_FREE(cpuset); - throw; + auto isInitFail = (state == DhtServerState::DHT_SERVER_INIT); + state = DhtServerState::DHT_SERVER_ERROR; + + if (!isInitFail) + throw; } catch (...) { DAQ_DEBUG("DHT server exception: unknown"); state = DhtServerState::DHT_SERVER_ERROR; CPU_FREE(cpuset); throw; } - - CPU_FREE(cpuset); } void DhtServer::serve(void) { diff --git a/lib/thin/KVStoreThin.cpp b/lib/thin/KVStoreThin.cpp index 86c8d76..fa458b0 100644 --- a/lib/thin/KVStoreThin.cpp +++ b/lib/thin/KVStoreThin.cpp @@ -11,7 +11,7 @@ * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and - * limitations under the License. + * limitations under the License. */ #include "KVStoreThin.h" @@ -40,6 +40,10 @@ void KVStoreThin::init() { if (getOptions().runtime.logFunc) gLog.setLogFunc(getOptions().runtime.logFunc); +#ifdef DPDK + _spSpdk.reset(new SpdkCore(getOptions().offload)); +#endif + _spDht.reset(new DhtCore(getOptions().dht)); _spDht->initNexus(); _spDht->initClient(); diff --git a/lib/thin/KVStoreThin.h b/lib/thin/KVStoreThin.h index 993f9ca..f7326e0 100644 --- a/lib/thin/KVStoreThin.h +++ b/lib/thin/KVStoreThin.h @@ -11,12 +11,16 @@ * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and - * limitations under the License. + * limitations under the License. */ #pragma once -#include +#ifdef DPDK +#include /* net/if.h (put before linux/if.h) */ +#endif + +#include /* include linux/if.h */ #include namespace DaqDB { @@ -83,6 +87,10 @@ class KVStoreThin : public KVStoreBase { size_t _keySize; Options _options; +#ifdef DPDK + std::unique_ptr _spSpdk; +#endif + std::unique_ptr _spDht; }; diff --git a/tests/functional_thin/main.cpp b/tests/functional_thin/main.cpp index 3ed2279..5a2f293 100644 --- a/tests/functional_thin/main.cpp +++ b/tests/functional_thin/main.cpp @@ -91,11 +91,13 @@ int main(int argc, const char *argv[]) { return -1; } +#ifndef DPDK if (!executeTest("testRemotePeerConnect", testRemotePeerConnect, nullptr, &options)) { DAQDB_INFO << "Cannot connect to peer node" << endl; return -1; } +#endif unique_ptr spKVStore; try { From 346c8fda0cdcaebd5a9854e148609c726c9cf44c Mon Sep 17 00:00:00 2001 From: Jakub Radtke Date: Thu, 28 Feb 2019 03:04:31 -0500 Subject: [PATCH 06/10] Minor fixes in functional tests Signed-off-by: Jakub Radtke --- tests/functional/tests/tests_base.cpp | 26 +++++++------- tests/functional/tests/tests_offload.cpp | 44 +++++++++++++----------- tests/functional_thin/main.cpp | 12 +++---- 3 files changed, 43 insertions(+), 39 deletions(-) diff --git a/tests/functional/tests/tests_base.cpp b/tests/functional/tests/tests_base.cpp index b52789b..09067d1 100644 --- a/tests/functional/tests/tests_base.cpp +++ b/tests/functional/tests/tests_base.cpp @@ -54,10 +54,10 @@ bool testAsyncOperations(KVStoreBase *kvs) { condition_variable cv; bool ready = false; - daqdb_async_put(kvs, keyId, val, - [&](KVStoreBase *kvs, Status status, const char *argKey, - const size_t keySize, const char *value, - const size_t valueSize) { + daqdb_async_put( + kvs, keyId, val, + [&](KVStoreBase *kvs, Status status, const char *argKey, + const size_t keySize, const char *value, const size_t valueSize) { unique_lock lck(mtx); if (status.ok()) { DAQDB_INFO << boost::format("PutAsync: [%1%]") % @@ -69,7 +69,7 @@ bool testAsyncOperations(KVStoreBase *kvs) { } ready = true; cv.notify_all(); - }); + }); // wait for completion { @@ -83,15 +83,17 @@ bool testAsyncOperations(KVStoreBase *kvs) { result = false; } - daqdb_async_get(kvs, keyId, - [&](KVStoreBase *kvs, Status status, const char *argKey, - size_t keySize, const char *value, size_t valueSize) { + daqdb_async_get( + kvs, keyId, + [&](KVStoreBase *kvs, Status status, const char *argKey, size_t keySize, + const char *value, size_t valueSize) { unique_lock lck(mtx); if (status.ok()) { - DAQDB_INFO << boost::format("GetAsync: [%1%] = %2%") % - keyToStr(argKey) % value; - if (!value || val.compare(value) != 0) { + DAQDB_INFO << boost::format("GetAsync: [%1%]") % + keyToStr(argKey); + + if (!value || memcmp(val.data(), value, val.size())) { DAQDB_INFO << "Error: wrong value returned" << flush; result = false; } @@ -103,7 +105,7 @@ bool testAsyncOperations(KVStoreBase *kvs) { ready = true; cv.notify_all(); - }); + }); // wait for completion { diff --git a/tests/functional/tests/tests_offload.cpp b/tests/functional/tests/tests_offload.cpp index 8ff785c..d450a03 100644 --- a/tests/functional/tests/tests_offload.cpp +++ b/tests/functional/tests/tests_offload.cpp @@ -74,10 +74,10 @@ bool testAsyncOffloadOperations(KVStoreBase *kvs) { condition_variable cv; bool ready = false; - daqdb_async_put(kvs, keyId, val, - [&](KVStoreBase *kvs, Status status, const char *argKey, - const size_t keySize, const char *value, - const size_t valueSize) { + daqdb_async_put( + kvs, keyId, val, + [&](KVStoreBase *kvs, Status status, const char *argKey, + const size_t keySize, const char *value, const size_t valueSize) { unique_lock lck(mtx); if (status.ok()) { DAQDB_INFO << boost::format("PutAsync: [%1%]") % @@ -89,7 +89,7 @@ bool testAsyncOffloadOperations(KVStoreBase *kvs) { } ready = true; cv.notify_all(); - }); + }); // wait for completion { @@ -109,15 +109,16 @@ bool testAsyncOffloadOperations(KVStoreBase *kvs) { result = false; } - daqdb_async_get(kvs, keyId, - [&](KVStoreBase *kvs, Status status, const char *argKey, - size_t keySize, const char *value, size_t valueSize) { + daqdb_async_get( + kvs, keyId, + [&](KVStoreBase *kvs, Status status, const char *argKey, size_t keySize, + const char *value, size_t valueSize) { unique_lock lck(mtx); if (status.ok()) { - DAQDB_INFO << boost::format("GetAsync: [%1%] = %2%") % - keyToStr(argKey) % value; - if (!value || val.compare(value) != 0) { + DAQDB_INFO << boost::format("GetAsync: [%1%]") % + keyToStr(argKey); + if (!value || memcmp(val.data(), value, val.size())) { DAQDB_INFO << "Error: wrong value returned" << flush; result = false; } @@ -129,7 +130,7 @@ bool testAsyncOffloadOperations(KVStoreBase *kvs) { ready = true; cv.notify_all(); - }); + }); // wait for completion { @@ -138,10 +139,10 @@ bool testAsyncOffloadOperations(KVStoreBase *kvs) { ready = false; } - daqdb_async_offload(kvs, keyId, - [&](KVStoreBase *kvs, Status status, const char *argKey, - const size_t keySize, const char *value, - const size_t valueSize) { + daqdb_async_offload( + kvs, keyId, + [&](KVStoreBase *kvs, Status status, const char *argKey, + const size_t keySize, const char *value, const size_t valueSize) { unique_lock lck(mtx); if (status.ok()) { DAQDB_INFO << boost::format("Offload: [%1%]") % @@ -153,7 +154,7 @@ bool testAsyncOffloadOperations(KVStoreBase *kvs) { } ready = true; cv.notify_all(); - }); + }); // wait for completion { @@ -172,9 +173,10 @@ bool testAsyncOffloadOperations(KVStoreBase *kvs) { result = false; } - daqdb_async_get(kvs, keyId, - [&](KVStoreBase *kvs, Status status, const char *argKey, - size_t keySize, const char *value, size_t valueSize) { + daqdb_async_get( + kvs, keyId, + [&](KVStoreBase *kvs, Status status, const char *argKey, size_t keySize, + const char *value, size_t valueSize) { unique_lock lck(mtx); if (status.ok()) { @@ -192,7 +194,7 @@ bool testAsyncOffloadOperations(KVStoreBase *kvs) { ready = true; cv.notify_all(); - }); + }); // wait for completion { diff --git a/tests/functional_thin/main.cpp b/tests/functional_thin/main.cpp index 5a2f293..f42b8f2 100644 --- a/tests/functional_thin/main.cpp +++ b/tests/functional_thin/main.cpp @@ -11,7 +11,7 @@ * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and - * limitations under the License. + * limitations under the License. */ #include @@ -117,11 +117,11 @@ int main(int argc, const char *argv[]) { if (!executeTest(test.first, test.second, spKVStore.get(), &options)) { failsCount++; } + } - if (failsCount > 0) { - DAQDB_INFO << format("Test(s) failed [%1%]") % failsCount << endl; - } else { - DAQDB_INFO << "All tests passed!" << endl; - } + if (failsCount > 0) { + DAQDB_INFO << format("Test(s) failed [%1%]") % failsCount << endl; + } else { + DAQDB_INFO << "All tests passed!" << endl; } } From 6e58a29265e93c48c8d11a158a078302bb936313 Mon Sep 17 00:00:00 2001 From: Grzegorz Jereczek Date: Wed, 6 Mar 2019 14:08:39 +0100 Subject: [PATCH 07/10] Adjusting minidaq key structure and adding key size checks (#279) * minidaq: chaning key strucutre * pmem: fixing key size check * artree: removing key from leaf node * minidaq: optimizing key structure for current artree implementation * tests: fixing key size * tests: fixing segfault * fixing clang formatting * pmem: adding checks for key size * fixing clang formatting * fixing clang formatting * fixing tests and examples --- apps/minidaq/MinidaqFfNode.cpp | 6 +-- apps/minidaq/MinidaqFfNodeSeq.cpp | 12 ++--- apps/minidaq/MinidaqFfNodeSeq.h | 6 ++- apps/minidaq/MinidaqNode.cpp | 17 ++++++-- apps/minidaq/MinidaqNode.h | 12 ++--- apps/minidaq/MinidaqRoNode.cpp | 12 ++--- apps/minidaq/MinidaqRoNode.h | 6 ++- apps/minidaq/minidaq.cpp | 8 ++-- examples/clinode/config.cpp | 6 +-- examples/clinode/config.h | 9 ++-- examples/clinode/nodeCli.cpp | 4 +- include/daqdb/Options.h | 9 +++- lib/core/KVStore.cpp | 21 ++++----- lib/pmem/ARTree.cpp | 65 ++++++++++++++-------------- lib/pmem/ARTree.h | 10 +++-- lib/pmem/RTree.cpp | 4 +- lib/pmem/RTree.h | 4 +- lib/pmem/RTreeEngine.h | 7 +-- tests/functional/base_operations.cpp | 21 ++++++--- tests/functional/config.cpp | 6 +-- tests/functional/config.h | 9 ++-- 21 files changed, 140 insertions(+), 114 deletions(-) diff --git a/apps/minidaq/MinidaqFfNode.cpp b/apps/minidaq/MinidaqFfNode.cpp index cb54df7..557309a 100644 --- a/apps/minidaq/MinidaqFfNode.cpp +++ b/apps/minidaq/MinidaqFfNode.cpp @@ -11,7 +11,7 @@ * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and - * limitations under the License. + * limitations under the License. */ #include "MinidaqFfNode.h" @@ -80,11 +80,9 @@ void MinidaqFfNode::_Task(Key &&key, std::atomic &cnt, int baseId = _PickSubdetector(); bool accept = _Accept(); - mKeyPtr->runId = _runId; - for (int i = 0; i < _PickNFragments(); i++) { /** @todo change to GetRange once implemented */ - mKeyPtr->subdetectorId = baseId + i; + mKeyPtr->componentId = baseId + i; DaqDB::Value value; try { value = _kvs->Get(key); diff --git a/apps/minidaq/MinidaqFfNodeSeq.cpp b/apps/minidaq/MinidaqFfNodeSeq.cpp index 08fafb3..ca520a3 100644 --- a/apps/minidaq/MinidaqFfNodeSeq.cpp +++ b/apps/minidaq/MinidaqFfNodeSeq.cpp @@ -11,14 +11,14 @@ * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and - * limitations under the License. + * limitations under the License. */ #include "MinidaqFfNodeSeq.h" namespace DaqDB { -thread_local int MinidaqFfNodeSeq::_eventId; +thread_local uint64_t MinidaqFfNodeSeq::_eventId; MinidaqFfNodeSeq::MinidaqFfNodeSeq(KVStoreBase *kvs) : MinidaqFfNode(kvs) {} @@ -35,9 +35,9 @@ Key MinidaqFfNodeSeq::_NextKey() { ? AllocOptions(KeyValAttribute::NOT_BUFFERED) : AllocOptions(KeyValAttribute::KVS_BUFFERED)); MinidaqKey *mKeyPtr = reinterpret_cast(key.data()); - mKeyPtr->runId = _runId; - mKeyPtr->subdetectorId = _baseId; - mKeyPtr->eventId = _eventId; + mKeyPtr->detectorId = 0; + mKeyPtr->componentId = _baseId; + memcpy(&mKeyPtr->eventId, &_eventId, sizeof(mKeyPtr->eventId)); _eventId += _nTh; return key; } @@ -51,7 +51,7 @@ void MinidaqFfNodeSeq::_Task(Key &&key, std::atomic &cnt, for (int i = 0; i < _PickNFragments(); i++) { /** @todo change to GetRange once implemented */ - mKeyPtr->subdetectorId = baseId + i; + mKeyPtr->componentId = baseId + i; DaqDB::Value value; nRetries = 0; while (nRetries < _maxRetries) { diff --git a/apps/minidaq/MinidaqFfNodeSeq.h b/apps/minidaq/MinidaqFfNodeSeq.h index 8e83bd5..68c33ec 100644 --- a/apps/minidaq/MinidaqFfNodeSeq.h +++ b/apps/minidaq/MinidaqFfNodeSeq.h @@ -11,7 +11,7 @@ * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and - * limitations under the License. + * limitations under the License. */ #pragma once @@ -32,6 +32,8 @@ class MinidaqFfNodeSeq : public MinidaqFfNode { Key _NextKey(); std::string _GetType(); - static thread_local int _eventId; + static thread_local uint64_t _eventId; + static_assert(sizeof(_eventId) >= sizeof(MinidaqKey().eventId), + "Event Id field of MinidaqKey is too big"); }; } diff --git a/apps/minidaq/MinidaqNode.cpp b/apps/minidaq/MinidaqNode.cpp index 8cda192..2c53040 100644 --- a/apps/minidaq/MinidaqNode.cpp +++ b/apps/minidaq/MinidaqNode.cpp @@ -11,7 +11,7 @@ * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and - * limitations under the License. + * limitations under the License. */ #include @@ -261,7 +261,9 @@ void MinidaqNode::SaveSummary(std::string &fs, std::string &tname) { char MinidaqNode::_GetBufferByte(const Key &key, size_t i) { const MinidaqKey *mKeyPtr = reinterpret_cast(key.data()); - return ((mKeyPtr->eventId + i) % 256); + const char *eventId = reinterpret_cast(&mKeyPtr->eventId); + return *(eventId + + (i % (sizeof(mKeyPtr->eventId) / sizeof(mKeyPtr->eventId[0])))); } void MinidaqNode::_FillBuffer(const Key &key, char *buf, size_t s) { @@ -286,8 +288,15 @@ bool MinidaqNode::_CheckBuffer(const Key &key, const char *buf, size_t s) { if (!err) { err = true; msg << "Integrity check failed (" << _GetType() - << ") EventId=" << mKeyPtr->eventId - << " SubdetectorId=" << mKeyPtr->subdetectorId << std::endl; + << ") EventId=0x"; + for (int j = (sizeof(mKeyPtr->eventId) / + sizeof(mKeyPtr->eventId[0])) - + 1; + j >= 0; j--) + msg << std::hex + << static_cast(mKeyPtr->eventId[j]); + msg << std::dec << " SubdetectorId=" << mKeyPtr->componentId + << std::endl; _nIntegrityErrors++; } msg << " buf[" << i << "] = " diff --git a/apps/minidaq/MinidaqNode.h b/apps/minidaq/MinidaqNode.h index 856f17a..4425d7f 100644 --- a/apps/minidaq/MinidaqNode.h +++ b/apps/minidaq/MinidaqNode.h @@ -11,7 +11,7 @@ * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and - * limitations under the License. + * limitations under the License. */ #pragma once @@ -27,12 +27,9 @@ namespace DaqDB { struct __attribute__((packed)) MinidaqKey { - MinidaqKey() : eventId(0), subdetectorId(0), runId(0){}; - MinidaqKey(uint64_t e, uint16_t s, uint16_t r) - : eventId(e), subdetectorId(s), runId(r) {} - uint16_t runId; - uint16_t subdetectorId; - uint64_t eventId; + uint8_t eventId[5]; + uint8_t detectorId; + uint16_t componentId; }; class MinidaqNode { @@ -72,7 +69,6 @@ class MinidaqNode { virtual std::string _GetType() = 0; KVStoreBase *_kvs; - int _runId = 599; int _nTh = 1; // number of worker threads bool _localOnly = false; // single-node benchmark #ifdef WITH_INTEGRITY_CHECK diff --git a/apps/minidaq/MinidaqRoNode.cpp b/apps/minidaq/MinidaqRoNode.cpp index c65e5c7..3c533d5 100644 --- a/apps/minidaq/MinidaqRoNode.cpp +++ b/apps/minidaq/MinidaqRoNode.cpp @@ -11,7 +11,7 @@ * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and - * limitations under the License. + * limitations under the License. */ #include "MinidaqRoNode.h" @@ -19,8 +19,8 @@ namespace DaqDB { -thread_local int MinidaqRoNode::_eventId; -thread_local constexpr char MinidaqRoNode::_data_buffer [100000]; +thread_local uint64_t MinidaqRoNode::_eventId; +thread_local constexpr char MinidaqRoNode::_data_buffer[100000]; MinidaqRoNode::MinidaqRoNode(KVStoreBase *kvs) : MinidaqNode(kvs) {} @@ -35,9 +35,9 @@ Key MinidaqRoNode::_NextKey() { ? AllocOptions(KeyValAttribute::NOT_BUFFERED) : AllocOptions(KeyValAttribute::KVS_BUFFERED)); MinidaqKey *mKeyPtr = reinterpret_cast(key.data()); - mKeyPtr->runId = _runId; - mKeyPtr->subdetectorId = _id; - mKeyPtr->eventId = _eventId; + mKeyPtr->detectorId = 0; + mKeyPtr->componentId = _id; + memcpy(&mKeyPtr->eventId, &_eventId, sizeof(mKeyPtr->eventId)); _eventId += _nTh; return key; } diff --git a/apps/minidaq/MinidaqRoNode.h b/apps/minidaq/MinidaqRoNode.h index 1e2103f..ac6e31d 100644 --- a/apps/minidaq/MinidaqRoNode.h +++ b/apps/minidaq/MinidaqRoNode.h @@ -11,7 +11,7 @@ * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and - * limitations under the License. + * limitations under the License. */ #pragma once @@ -37,7 +37,9 @@ class MinidaqRoNode : public MinidaqNode { size_t _fSize = 0; int _id = 0; - static thread_local int _eventId; + static thread_local uint64_t _eventId; + static_assert(sizeof(_eventId) >= sizeof(MinidaqKey().eventId), + "Event Id field of MinidaqKey is too big"); static thread_local constexpr char _data_buffer[100000] = " "; }; } diff --git a/apps/minidaq/minidaq.cpp b/apps/minidaq/minidaq.cpp index a438bf1..08dadd3 100644 --- a/apps/minidaq/minidaq.cpp +++ b/apps/minidaq/minidaq.cpp @@ -11,7 +11,7 @@ * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and - * limitations under the License. + * limitations under the License. */ #include @@ -108,9 +108,9 @@ static std::unique_ptr openKVS() { options.pmem.poolPath = pmem_path; options.pmem.totalSize = pmem_size; options.pmem.allocUnitSize = fSize; - options.key.field(0, sizeof(DaqDB::MinidaqKey::runId)); - options.key.field(1, sizeof(DaqDB::MinidaqKey::subdetectorId)); - options.key.field(2, sizeof(DaqDB::MinidaqKey::eventId), true); + options.key.field(0, sizeof(DaqDB::MinidaqKey::eventId), true); + options.key.field(1, sizeof(DaqDB::MinidaqKey::detectorId)); + options.key.field(2, sizeof(DaqDB::MinidaqKey::componentId)); options.runtime.numOfPollers = nPoolers; options.runtime.maxReadyKeys = maxReadyKeys; if (satellite) { diff --git a/examples/clinode/config.cpp b/examples/clinode/config.cpp index c7ed751..2eb3069 100644 --- a/examples/clinode/config.cpp +++ b/examples/clinode/config.cpp @@ -36,9 +36,9 @@ void initKvsOptions(DaqDB::Options &options, const std::string &configFile) { options.pmem.totalSize = DEFAULT_PMEM_POOL_SIZE; options.pmem.allocUnitSize = DEFAULT_PMEM_ALLOC_UNIT_SIZE; - options.key.field(0, sizeof(CliNodeKey::runId)); - options.key.field(1, sizeof(CliNodeKey::subdetectorId)); - options.key.field(2, sizeof(CliNodeKey::eventId), true); + options.key.field(0, sizeof(CliNodeKey::eventId), true); + options.key.field(1, sizeof(CliNodeKey::detectorId)); + options.key.field(2, sizeof(CliNodeKey::componentId)); options.offload.allocUnitSize = DEFAULT_OFFLOAD_ALLOC_UNIT_SIZE; diff --git a/examples/clinode/config.h b/examples/clinode/config.h index 8af02cf..6dc8aa6 100644 --- a/examples/clinode/config.h +++ b/examples/clinode/config.h @@ -24,12 +24,9 @@ #include struct __attribute__((packed)) CliNodeKey { - CliNodeKey() : eventId(0), subdetectorId(0), runId(0) {}; - CliNodeKey(uint64_t e, uint16_t s, uint16_t r) - : eventId(e), subdetectorId(s), runId(r) {} - uint16_t runId; - uint16_t subdetectorId; - uint64_t eventId; + uint8_t eventId[5]; + uint8_t detectorId; + uint16_t componentId; }; void initKvsOptions(DaqDB::Options &options, const std::string &configFile); diff --git a/examples/clinode/nodeCli.cpp b/examples/clinode/nodeCli.cpp index 9cef545..4eb8bab 100644 --- a/examples/clinode/nodeCli.cpp +++ b/examples/clinode/nodeCli.cpp @@ -264,9 +264,7 @@ DaqDB::Key nodeCli::_strToKey(const std::string &key) { DaqDB::Key keyBuff = _spKVStore->AllocKey(KeyValAttribute::NOT_BUFFERED); CliNodeKey *cliKeyPtr = reinterpret_cast(keyBuff.data()); - cliKeyPtr->runId = 0; - cliKeyPtr->subdetectorId = 0; - cliKeyPtr->eventId = 0; + memset(cliKeyPtr, 0, sizeof(CliNodeKey)); memcpy(&(cliKeyPtr->eventId), key.data(), key.size()); return keyBuff; diff --git a/include/daqdb/Options.h b/include/daqdb/Options.h index 3c3bd53..c277bfe 100644 --- a/include/daqdb/Options.h +++ b/include/daqdb/Options.h @@ -11,7 +11,7 @@ * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and - * limitations under the License. + * limitations under the License. */ #pragma once @@ -110,6 +110,13 @@ struct KeyFieldDescriptor { struct KeyDescriptor { size_t nfields() const { return _fields.size(); } + size_t size() const { + size_t size = 0; + for (auto &f : _fields) + size += f.size; + return size; + } + void field(size_t idx, size_t size, bool isPrimary = false) { if (nfields() <= idx) _fields.resize(idx + 1); diff --git a/lib/core/KVStore.cpp b/lib/core/KVStore.cpp index 5ddbb62..0d04f8f 100644 --- a/lib/core/KVStore.cpp +++ b/lib/core/KVStore.cpp @@ -11,7 +11,7 @@ * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and - * limitations under the License. + * limitations under the License. */ #include "KVStore.h" @@ -87,6 +87,13 @@ void KVStore::init() { getOptions().pmem.allocUnitSize)); if (_spRtree.get() == nullptr) throw OperationFailedException(errno, ::pmemobj_errormsg()); + size_t keySize = pmem()->SetKeySize(getOptions().key.size()); + if (keySize != getOptions().key.size()) { + DAQ_INFO("Requested key size of " + + std::to_string(getOptions().key.size()) + + +"B does not match expected " + std::to_string(keySize) + "B"); + throw OperationFailedException(Status(NOT_SUPPORTED)); + } _spSpdk.reset(new SpdkCore(getOptions().offload)); if (isOffloadEnabled()) { @@ -623,16 +630,10 @@ std::string KVStore::getProperty(const std::string &name) { return ""; } -uint64_t KVStore::GetTreeSize() { - return pmem()->GetTreeSize(); -} +uint64_t KVStore::GetTreeSize() { return pmem()->GetTreeSize(); } -uint8_t KVStore::GetTreeDepth() { - return pmem()->GetTreeDepth(); -} +uint8_t KVStore::GetTreeDepth() { return pmem()->GetTreeDepth(); } -uint64_t KVStore::GetLeafCount() { - return pmem()->GetLeafCount(); -} +uint64_t KVStore::GetLeafCount() { return pmem()->GetLeafCount(); } } // namespace DaqDB diff --git a/lib/pmem/ARTree.cpp b/lib/pmem/ARTree.cpp index c337d3c..00e73d4 100644 --- a/lib/pmem/ARTree.cpp +++ b/lib/pmem/ARTree.cpp @@ -48,10 +48,10 @@ void TreeImpl::_initAllocClasses(const size_t allocUnitSize) { if (rc) throw OperationFailedException(Status(ALLOCATION_ERROR)); setClassId(ALLOC_CLASS_VALUE, alloc_daqdb.class_id); - DAQ_DEBUG( - "ARTree alloc class (value) (" + std::to_string(alloc_daqdb.class_id) + - ") defined: unit_size=" + std::to_string(alloc_daqdb.unit_size) + - " units_per_block=" + std::to_string(alloc_daqdb.units_per_block)); + DAQ_DEBUG("ARTree alloc class (value) (" + + std::to_string(alloc_daqdb.class_id) + ") defined: unit_size=" + + std::to_string(alloc_daqdb.unit_size) + " units_per_block=" + + std::to_string(alloc_daqdb.units_per_block)); // value wrapper alloc_daqdb.header_type = POBJ_HEADER_NONE; @@ -213,6 +213,16 @@ uint8_t TreeImpl::getTreeDepth(persistent_ptr current) { return depth; } +size_t ARTree::SetKeySize(size_t req_size) { + tree->treeRoot->keySize = + (sizeof(LEVEL_TYPE) / sizeof(int) - 1) * LEVEL_BYTES; + /** @todo change to make it configurable at init time/ + if (!tree->treeRoot->initialized) + tree->treeRoot->keySize = req_size; + */ + return tree->treeRoot->keySize; +} + void ARTree::Get(const char *key, int32_t keybytes, void **value, size_t *size, uint8_t *location) { persistent_ptr valPrstPtr; @@ -326,10 +336,10 @@ void ARTree::Remove(const char *key) { pmemobj_cancel(tree->_pm_pool.get_handle(), actionValue, 1); #ifdef USE_ALLOCATION_CLASSES - // TODO: commented because of error in PMDK on free() of object - // reserved with xreserve - // pmemobj_defer_free(tree->_pm_pool.get_handle(),(*valPrstPtr.raw_ptr()),&actionsArray[actionsCounter]); - // pmemobj_free(valPrstPtr.raw_ptr()); +// TODO: commented because of error in PMDK on free() of object +// reserved with xreserve +// pmemobj_defer_free(tree->_pm_pool.get_handle(),(*valPrstPtr.raw_ptr()),&actionsArray[actionsCounter]); +// pmemobj_free(valPrstPtr.raw_ptr()); #else // @TODO Only one of (pmemobj_defer_free, pmemobj_free) could be // called. @@ -391,7 +401,7 @@ void TreeImpl::allocateFullLevels(persistent_ptr node, sizeof(Node256), VALUE); #endif if (OID_IS_NULL(*(node256_new).raw_ptr())) { - DAQ_DEBUG("reserve failed actionsCounter=" + + DAQ_DEBUG("reserve Node256 failed actionsCounter=" + std::to_string(actionsCounter)); alloc_err = true; break; @@ -416,8 +426,9 @@ void TreeImpl::allocateFullLevels(persistent_ptr node, #endif if (OID_IS_NULL(*(nodeLeafCompressed_new).raw_ptr())) { - DAQ_DEBUG("reserve failed actionsCounter=" + - std::to_string(actionsCounter)); + DAQ_DEBUG( + "reserve nodeLeafCompressed failed actionsCounter=" + + std::to_string(actionsCounter)); alloc_err = true; break; } @@ -425,8 +436,6 @@ void TreeImpl::allocateFullLevels(persistent_ptr node, nodeLeafCompressed_new->depth = depth; nodeLeafCompressed_new->type = TYPE_LEAF_COMPRESSED; nodeLeafCompressed_new->parent = node; - // Temporarily disable the node - nodeLeafCompressed_new->key = -1; children[i] = nodeLeafCompressed_new; } @@ -469,10 +478,8 @@ TreeImpl::findValueInNode(persistent_ptr current, const char *_key, // ValueWrapper *val; while (1) { - keyCalc = key[KEY_SIZE - current->depth - 1]; - std::bitset<8> x(keyCalc); DAQ_DEBUG("findValueInNode: current->depth= " + - std::to_string(current->depth) + " keyCalc=" + x.to_string()); + std::to_string(current->depth)); if (current->depth == ((sizeof(LEVEL_TYPE) / sizeof(int) - 1))) { // Node Compressed nodeLeafCompressed = current; @@ -493,8 +500,9 @@ TreeImpl::findValueInNode(persistent_ptr current, const char *_key, #endif if (OID_IS_NULL(*(nodeLeafCompressed->child).raw_ptr())) { - DAQ_DEBUG("reserve failed actionsCounter=" + - std::to_string(actionsCounter)); + DAQ_DEBUG( + "reserve NodeLeafCompressed failed actionsCounter=" + + std::to_string(actionsCounter)); throw OperationFailedException(Status(ALLOCATION_ERROR)); } // std::cout << "valueWrapper off=" @@ -513,24 +521,17 @@ TreeImpl::findValueInNode(persistent_ptr current, const char *_key, */ nodeLeafCompressed->child->location = EMPTY; // val->location = EMPTY; - // Enable the node - nodeLeafCompressed->key = keyCalc; return nodeLeafCompressed->child; } else { - DAQ_DEBUG("findValueInNode: not allocate, keyCalc=" + - std::to_string(keyCalc) + "NodeLeafCompressed->key=" + - std::to_string(nodeLeafCompressed->key)); - if (nodeLeafCompressed->key == keyCalc) { - /*val = reinterpret_cast( - (nodeLeafCompressed->child).raw_ptr());*/ - DAQ_DEBUG("findValueInNode: Found"); - return nodeLeafCompressed->child; - } else { - DAQ_DEBUG("findValueInNode: Not Found"); - return nullptr; - } + DAQ_DEBUG("findValueInNode: Found"); + return nodeLeafCompressed->child; } } + keyCalc = (treeRoot->keySize - current->depth - 1) < 0 + ? 0 + : key[treeRoot->keySize - current->depth - 1]; + std::bitset<8> x(keyCalc); + DAQ_DEBUG("findValueInNode: keyCalc=" + x.to_string()); if (current->type == TYPE256) { // TYPE256 node256 = current; if (!allocate && node256->children[keyCalc]) { diff --git a/lib/pmem/ARTree.h b/lib/pmem/ARTree.h index d7e6c0b..ce3bb37 100644 --- a/lib/pmem/ARTree.h +++ b/lib/pmem/ARTree.h @@ -54,12 +54,14 @@ const int LEVEL_TYPE[] = {TYPE256, TYPE256, TYPE256, TYPE256, TYPE256, TYPE_LEAF_COMPRESSED}; // how many levels will be created on ARTree initialization const int PREALLOC_LEVELS = 1; + +// Size of a single level in bytes +#define LEVEL_BYTES 1 + // size of table for actions for each Node #define ACTION_NUMBER_NODE256 (1 + 256) #define ACTION_NUMBER_COMPRESSED 1 -#define KEY_SIZE 12 - // Allocation class alignment #define ALLOC_CLASS_ALIGNMENT 0 // Units per allocation block. @@ -114,7 +116,6 @@ class Node { class NodeLeafCompressed : public Node { public: explicit NodeLeafCompressed(int _depth, int _type) : Node(_depth, _type) {} - uint32_t key; persistent_ptr child; // pointer to Value }; @@ -128,8 +129,8 @@ class Node256 : public Node { struct ARTreeRoot { persistent_ptr rootNode; pmem::obj::mutex mutex; - p level_bits; bool initialized = false; + size_t keySize; // bytes }; class TreeImpl { @@ -158,6 +159,7 @@ class ARTree : public DaqDB::RTreeEngine { ARTree(const string &path, const size_t size, const size_t allocUnitSize); virtual ~ARTree(); string Engine() final { return "ARTree"; } + size_t SetKeySize(size_t req_size); void Get(const char *key, int32_t keybytes, void **value, size_t *size, uint8_t *location) final; void Get(const char *key, void **value, size_t *size, diff --git a/lib/pmem/RTree.cpp b/lib/pmem/RTree.cpp index 81f387c..6801566 100644 --- a/lib/pmem/RTree.cpp +++ b/lib/pmem/RTree.cpp @@ -11,7 +11,7 @@ * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and - * limitations under the License. + * limitations under the License. */ #include "RTree.h" @@ -67,6 +67,8 @@ Tree::Tree(const string &path, const size_t size, const size_t allocUnitSize) { std::to_string(alloc_daqdb.units_per_block)); } +size_t RTree::SetKeySize(size_t req_size) { return KEY_SIZE / BITS_IN_BYTE; } + void RTree::Get(const char *key, int32_t keybytes, void **value, size_t *size, uint8_t *location) { ValueWrapper *val = tree->findValueInNode(tree->treeRoot->rootNode, key); diff --git a/lib/pmem/RTree.h b/lib/pmem/RTree.h index bcf2af3..3a29a2b 100644 --- a/lib/pmem/RTree.h +++ b/lib/pmem/RTree.h @@ -11,7 +11,7 @@ * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and - * limitations under the License. + * limitations under the License. */ #ifndef LIB_STORE_RTREE_H_ @@ -46,6 +46,7 @@ using namespace pmem::obj; #define ALLOC_CLASS_ALIGNMENT 0 // Units per allocation block. #define ALLOC_CLASS_UNITS_PER_BLOCK 1000 +#define BITS_IN_BYTE 8 enum OBJECT_TYPES { VALUE, IOV }; @@ -107,6 +108,7 @@ class RTree : public DaqDB::RTreeEngine { RTree(const string &path, const size_t size, const size_t allocUnitSize); virtual ~RTree(); string Engine() final { return "RTree"; } + size_t SetKeySize(size_t req_size); void Get(const char *key, int32_t keybytes, void **value, size_t *size, uint8_t *location) final; void Get(const char *key, void **value, size_t *size, diff --git a/lib/pmem/RTreeEngine.h b/lib/pmem/RTreeEngine.h index dfc67a2..7b1b1f4 100644 --- a/lib/pmem/RTreeEngine.h +++ b/lib/pmem/RTreeEngine.h @@ -11,7 +11,7 @@ * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and - * limitations under the License. + * limitations under the License. */ #pragma once @@ -37,10 +37,11 @@ class RTreeEngine { static RTreeEngine *Open(const string &path, // path to persistent pool size_t size, // size used when creating pool size_t allocUnitSize); // allocation unit size - virtual ~RTreeEngine() {}; - static void Close(RTreeEngine *kv); // close storage engine + virtual ~RTreeEngine(){}; + static void Close(RTreeEngine *kv); // close storage engine virtual string Engine() = 0; // engine identifier + virtual size_t SetKeySize(size_t req_size) = 0; virtual void Get(const char *key, int32_t keybytes, void **value, size_t *size, uint8_t *location) = 0; virtual void Get(const char *key, void **value, size_t *size, diff --git a/tests/functional/base_operations.cpp b/tests/functional/base_operations.cpp index a7aa36c..8152fc7 100644 --- a/tests/functional/base_operations.cpp +++ b/tests/functional/base_operations.cpp @@ -31,9 +31,8 @@ Value allocValue(KVStoreBase *kvs, const uint64_t keyId, const string &value) { Key allocKey(KVStoreBase *kvs, const uint64_t id) { Key keyBuff = kvs->AllocKey(KeyValAttribute::NOT_BUFFERED); FuncTestKey *fKeyPtr = reinterpret_cast(keyBuff.data()); - fKeyPtr->runId = 0; - fKeyPtr->subdetectorId = 0; - fKeyPtr->eventId = id; + memset(fKeyPtr, 0, sizeof(FuncTestKey)); + memcpy(&fKeyPtr->eventId, &id, sizeof(fKeyPtr->eventId)); return keyBuff; } @@ -58,12 +57,24 @@ bool checkValue(const string &expectedValue, Value *value) { const std::string keyToStr(Key &key) { FuncTestKey *fKeyPtr = reinterpret_cast(key.data()); - return std::to_string(fKeyPtr->eventId); + std::stringstream ss; + ss << "0x"; + for (int i = (sizeof(fKeyPtr->eventId) / sizeof(fKeyPtr->eventId[0])) - 1; + i >= 0; i--) + ss << std::hex << static_cast(fKeyPtr->eventId[i]); + + return ss.str(); } const std::string keyToStr(const char *key) { const FuncTestKey *fKeyPtr = reinterpret_cast(key); - return std::to_string(fKeyPtr->eventId); + std::stringstream ss; + ss << "0x"; + for (int i = (sizeof(fKeyPtr->eventId) / sizeof(fKeyPtr->eventId[0])) - 1; + i >= 0; i--) + ss << std::hex << static_cast(fKeyPtr->eventId[i]); + + return ss.str(); } Value daqdb_get(KVStoreBase *kvs, const uint64_t keyId) { diff --git a/tests/functional/config.cpp b/tests/functional/config.cpp index c83669e..34bcd21 100644 --- a/tests/functional/config.cpp +++ b/tests/functional/config.cpp @@ -36,9 +36,9 @@ bool initKvsOptions(DaqDB::Options &options, const std::string &configFile) { options.pmem.totalSize = DEFAULT_PMEM_POOL_SIZE; options.pmem.allocUnitSize = DEFAULT_PMEM_ALLOC_UNIT_SIZE; - options.key.field(0, sizeof(FuncTestKey::runId)); - options.key.field(1, sizeof(FuncTestKey::subdetectorId)); - options.key.field(2, sizeof(FuncTestKey::eventId), true); + options.key.field(0, sizeof(FuncTestKey::eventId), true); + options.key.field(1, sizeof(FuncTestKey::detectorId)); + options.key.field(2, sizeof(FuncTestKey::componentId)); options.offload.allocUnitSize = DEFAULT_OFFLOAD_ALLOC_UNIT_SIZE; diff --git a/tests/functional/config.h b/tests/functional/config.h index c9d650c..e1eaa2b 100644 --- a/tests/functional/config.h +++ b/tests/functional/config.h @@ -23,12 +23,9 @@ #include struct __attribute__((packed)) FuncTestKey { - FuncTestKey() : eventId(0), subdetectorId(0), runId(0) {}; - FuncTestKey(uint64_t e, uint16_t s, uint16_t r) - : eventId(e), subdetectorId(s), runId(r) {} - uint16_t runId; - uint16_t subdetectorId; - uint64_t eventId; + uint8_t eventId[5]; + uint8_t detectorId; + uint16_t componentId; }; bool initKvsOptions(DaqDB::Options &options, const std::string &configFile); From 7afa3d3b755ec7e281df1ea0f286fc5cb92d3d23 Mon Sep 17 00:00:00 2001 From: Jakub Radtke Date: Thu, 28 Feb 2019 02:36:58 -0500 Subject: [PATCH 08/10] Use separate DPDK submodule for compilation Required for DPDK transport in eRPC Signed-off-by: Jakub Radtke --- .gitmodules | 5 + CMakeLists.txt | 2 +- scripts/libdpdk.mri | 114 +++++++++++++++++++++ scripts/libspdk.mri | 15 --- scripts/prepare_dpdk_libs.sh | 11 ++ tests/functional_thin/tests/tests_base.cpp | 2 +- third-party/CMakeLists.txt | 2 + third-party/dpdk | 1 + third-party/dpdk.cmake | 21 ++++ third-party/erpc.cmake | 4 +- third-party/spdk.cmake | 8 +- 11 files changed, 161 insertions(+), 24 deletions(-) create mode 100644 scripts/libdpdk.mri create mode 100755 scripts/prepare_dpdk_libs.sh create mode 160000 third-party/dpdk create mode 100644 third-party/dpdk.cmake diff --git a/.gitmodules b/.gitmodules index ee09c6a..2a9f751 100644 --- a/.gitmodules +++ b/.gitmodules @@ -29,3 +29,8 @@ [submodule "third-party/eRPC-fork"] path = third-party/eRPC-fork url = https://github.com/daq-db/eRPC.git + ignore = dirty +[submodule "third-party/dpdk"] + path = third-party/dpdk + url = https://github.com/DPDK/dpdk.git + ignore = dirty diff --git a/CMakeLists.txt b/CMakeLists.txt index 35ff65f..b57ed1e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -73,7 +73,7 @@ add_subdirectory(${3RDPARTY}) include_directories(${LIBCONFIG_INCLUDES_EXPORT}) include_directories(${3RDPARTY}/eRPC-fork/src) -include_directories(${3RDPARTY}/spdk/dpdk/build/include) +include_directories(${3RDPARTY}/dpdk/build/include) file(GLOB_RECURSE DAQDB_THIN_SOURCES lib/thin/*.cpp lib/common/*.cpp lib/dht/*.cpp lib/KVStoreBase.cpp) diff --git a/scripts/libdpdk.mri b/scripts/libdpdk.mri new file mode 100644 index 0000000..b92c9b0 --- /dev/null +++ b/scripts/libdpdk.mri @@ -0,0 +1,114 @@ +verbose +create libdpdk.a +addlib build/lib/librte_bbdev.a +addlib build/lib/librte_bitratestats.a +addlib build/lib/librte_bpf.a +addlib build/lib/librte_bus_dpaa.a +addlib build/lib/librte_bus_fslmc.a +addlib build/lib/librte_bus_ifpga.a +addlib build/lib/librte_bus_pci.a +addlib build/lib/librte_bus_vdev.a +addlib build/lib/librte_bus_vmbus.a +addlib build/lib/librte_cfgfile.a +addlib build/lib/librte_cmdline.a +addlib build/lib/librte_common_cpt.a +addlib build/lib/librte_common_dpaax.a +addlib build/lib/librte_common_octeontx.a +addlib build/lib/librte_compressdev.a +addlib build/lib/librte_cryptodev.a +addlib build/lib/librte_distributor.a +addlib build/lib/librte_eal.a +addlib build/lib/librte_efd.a +addlib build/lib/librte_ethdev.a +addlib build/lib/librte_eventdev.a +addlib build/lib/librte_flow_classify.a +addlib build/lib/librte_gro.a +addlib build/lib/librte_gso.a +addlib build/lib/librte_hash.a +addlib build/lib/librte_ip_frag.a +addlib build/lib/librte_ipsec.a +addlib build/lib/librte_jobstats.a +addlib build/lib/librte_kni.a +addlib build/lib/librte_kvargs.a +addlib build/lib/librte_latencystats.a +addlib build/lib/librte_lpm.a +addlib build/lib/librte_mbuf.a +addlib build/lib/librte_member.a +addlib build/lib/librte_mempool.a +addlib build/lib/librte_mempool_bucket.a +addlib build/lib/librte_mempool_dpaa2.a +addlib build/lib/librte_mempool_dpaa.a +addlib build/lib/librte_mempool_octeontx.a +addlib build/lib/librte_mempool_ring.a +addlib build/lib/librte_mempool_stack.a +addlib build/lib/librte_meter.a +addlib build/lib/librte_metrics.a +addlib build/lib/librte_net.a +addlib build/lib/librte_pci.a +addlib build/lib/librte_pdump.a +addlib build/lib/librte_pipeline.a +addlib build/lib/librte_pmd_af_packet.a +addlib build/lib/librte_pmd_ark.a +addlib build/lib/librte_pmd_atlantic.a +addlib build/lib/librte_pmd_avf.a +addlib build/lib/librte_pmd_avp.a +addlib build/lib/librte_pmd_axgbe.a +addlib build/lib/librte_pmd_bbdev_null.a +addlib build/lib/librte_pmd_bnxt.a +addlib build/lib/librte_pmd_bond.a +addlib build/lib/librte_pmd_caam_jr.a +addlib build/lib/librte_pmd_crypto_scheduler.a +addlib build/lib/librte_pmd_cxgbe.a +addlib build/lib/librte_pmd_dpaa2.a +addlib build/lib/librte_pmd_dpaa2_cmdif.a +addlib build/lib/librte_pmd_dpaa2_event.a +addlib build/lib/librte_pmd_dpaa2_qdma.a +addlib build/lib/librte_pmd_dpaa2_sec.a +addlib build/lib/librte_pmd_dpaa.a +addlib build/lib/librte_pmd_dpaa_event.a +addlib build/lib/librte_pmd_dpaa_sec.a +addlib build/lib/librte_pmd_dsw_event.a +addlib build/lib/librte_pmd_e1000.a +addlib build/lib/librte_pmd_ena.a +addlib build/lib/librte_pmd_enetc.a +addlib build/lib/librte_pmd_enic.a +addlib build/lib/librte_pmd_failsafe.a +addlib build/lib/librte_pmd_fm10k.a +addlib build/lib/librte_pmd_i40e.a +addlib build/lib/librte_pmd_ice.a +addlib build/lib/librte_pmd_ifpga_rawdev.a +addlib build/lib/librte_pmd_ixgbe.a +addlib build/lib/librte_pmd_kni.a +addlib build/lib/librte_pmd_lio.a +addlib build/lib/librte_pmd_netvsc.a +addlib build/lib/librte_pmd_nfp.a +addlib build/lib/librte_pmd_null.a +addlib build/lib/librte_pmd_null_crypto.a +addlib build/lib/librte_pmd_octeontx.a +addlib build/lib/librte_pmd_octeontx_crypto.a +addlib build/lib/librte_pmd_octeontx_ssovf.a +addlib build/lib/librte_pmd_octeontx_zip.a +addlib build/lib/librte_pmd_opdl_event.a +addlib build/lib/librte_pmd_qat.a +addlib build/lib/librte_pmd_qede.a +addlib build/lib/librte_pmd_ring.a +addlib build/lib/librte_pmd_sfc_efx.a +addlib build/lib/librte_pmd_skeleton_event.a +addlib build/lib/librte_pmd_skeleton_rawdev.a +addlib build/lib/librte_pmd_sw_event.a +addlib build/lib/librte_pmd_tap.a +addlib build/lib/librte_pmd_thunderx_nicvf.a +addlib build/lib/librte_pmd_vdev_netvsc.a +addlib build/lib/librte_pmd_virtio.a +addlib build/lib/librte_pmd_virtio_crypto.a +addlib build/lib/librte_pmd_vmxnet3_uio.a +addlib build/lib/librte_port.a +addlib build/lib/librte_power.a +addlib build/lib/librte_rawdev.a +addlib build/lib/librte_reorder.a +addlib build/lib/librte_ring.a +addlib build/lib/librte_sched.a +addlib build/lib/librte_security.a +addlib build/lib/librte_timer.a +save +end diff --git a/scripts/libspdk.mri b/scripts/libspdk.mri index 9bc449c..8ff3a4c 100644 --- a/scripts/libspdk.mri +++ b/scripts/libspdk.mri @@ -46,18 +46,3 @@ addlib build/lib/libspdk_util.a addlib build/lib/libspdk_vhost.a addlib build/lib/libspdk_virtio.a save -create libdpdk.a -addlib dpdk/build/lib/librte_bus_pci.a -addlib dpdk/build/lib/librte_bus_vdev.a -addlib dpdk/build/lib/librte_eal.a -addlib dpdk/build/lib/librte_ethdev.a -addlib dpdk/build/lib/librte_kvargs.a -addlib dpdk/build/lib/librte_mbuf.a -addlib dpdk/build/lib/librte_mempool.a -addlib dpdk/build/lib/librte_mempool_bucket.a -addlib dpdk/build/lib/librte_mempool_ring.a -addlib dpdk/build/lib/librte_net.a -addlib dpdk/build/lib/librte_pci.a -addlib dpdk/build/lib/librte_ring.a -save -end diff --git a/scripts/prepare_dpdk_libs.sh b/scripts/prepare_dpdk_libs.sh new file mode 100755 index 0000000..301073f --- /dev/null +++ b/scripts/prepare_dpdk_libs.sh @@ -0,0 +1,11 @@ +#!/bin/sh +set -x +SCRIPT=$(readlink -f $0) +SCRIPTPATH=`dirname $SCRIPT` +DPDKPATH=third-party/dpdk + +cd $SCRIPTPATH/../$DPDKPATH +rm -f x86_64-native-linuxapp-gcc/lib/libdpdk.a +ar -M <$SCRIPTPATH/libdpdk.mri +cd - +set +x diff --git a/tests/functional_thin/tests/tests_base.cpp b/tests/functional_thin/tests/tests_base.cpp index 8c7da70..979e244 100644 --- a/tests/functional_thin/tests/tests_base.cpp +++ b/tests/functional_thin/tests/tests_base.cpp @@ -18,9 +18,9 @@ #include "tests.h" +#include "../base_operations.h" #include #include -#include "../base_operations.h" using namespace std; using namespace DaqDB; diff --git a/third-party/CMakeLists.txt b/third-party/CMakeLists.txt index e9f2f51..2515fd0 100644 --- a/third-party/CMakeLists.txt +++ b/third-party/CMakeLists.txt @@ -13,6 +13,7 @@ set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${DAQDB_BIN_DIR}) if(NOT THIN_LIB) include(pmdk.cmake) + include(dpdk.cmake) include(spdk.cmake) else(NOT THIN_LIB) if(NOT ERPC_RAW_TRANSPORT) @@ -44,4 +45,5 @@ add_custom_target(clean-dep) if(NOT THIN_LIB) add_dependencies(clean-dep libpmdk_clean) add_dependencies(clean-dep libspdk_clean) + add_dependencies(clean-dep libdpdk_clean) endif() diff --git a/third-party/dpdk b/third-party/dpdk new file mode 160000 index 0000000..8d47a75 --- /dev/null +++ b/third-party/dpdk @@ -0,0 +1 @@ +Subproject commit 8d47a753b7cb05e0ef62360324c6a28505038d09 diff --git a/third-party/dpdk.cmake b/third-party/dpdk.cmake new file mode 100644 index 0000000..f954faa --- /dev/null +++ b/third-party/dpdk.cmake @@ -0,0 +1,21 @@ +cmake_minimum_required(VERSION 3.5) + +include(ExternalProject) + +ExternalProject_Add(project_dpdk + PREFIX ${PROJECT_SOURCE_DIR}/dpdk + SOURCE_DIR ${PROJECT_SOURCE_DIR}/dpdk + BUILD_IN_SOURCE ${PROJECT_SOURCE_DIR}/dpdk + CONFIGURE_COMMAND ${CMAKE_MAKE_PROGRAM} config T=x86_64-native-linuxapp-gcc + BUILD_COMMAND ${CMAKE_MAKE_PROGRAM} + INSTALL_COMMAND ${ROOT_DAQDB_DIR}/scripts/prepare_dpdk_libs.sh +) +add_library(dpdk STATIC IMPORTED GLOBAL) +set_property(TARGET dpdk PROPERTY IMPORTED_LOCATION ${PROJECT_SOURCE_DIR}/dpdk/libdpdk.a) +add_dependencies(dpdk project_dpdk) + +add_custom_target(libdpdk_clean + COMMAND ${CMAKE_MAKE_PROGRAM} clean + COMMAND rm -f ${PROJECT_SOURCE_DIR}/dpdk/libdpdk.a + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/dpdk +) diff --git a/third-party/erpc.cmake b/third-party/erpc.cmake index 5af0eb2..10fe82a 100644 --- a/third-party/erpc.cmake +++ b/third-party/erpc.cmake @@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.5) include(ExternalProject) -set(DPDK_PATH ${PROJECT_SOURCE_DIR}/spdk/dpdk/build) +set(DPDK_PATH ${PROJECT_SOURCE_DIR}/dpdk/build) ExternalProject_Add(project_erpc PREFIX ${PROJECT_SOURCE_DIR}/eRPC-fork @@ -11,7 +11,7 @@ ExternalProject_Add(project_erpc CMAKE_ARGS -DPERF=ON -DTRANSPORT=${ERPC_TRANSPORT_MODE} -DLTO=OFF -DCMAKE_PREFIX_PATH=${DPDK_PATH} -DCMAKE_POSITION_INDEPENDENT_CODE=ON BUILD_COMMAND ${CMAKE_MAKE_PROGRAM} INSTALL_COMMAND "" - DEPENDS project_spdk + DEPENDS project_dpdk ) add_library(liberpc STATIC IMPORTED GLOBAL) set_target_properties(liberpc PROPERTIES IMPORTED_LOCATION diff --git a/third-party/spdk.cmake b/third-party/spdk.cmake index f5f2d73..74441c9 100644 --- a/third-party/spdk.cmake +++ b/third-party/spdk.cmake @@ -6,20 +6,18 @@ ExternalProject_Add(project_spdk PREFIX ${PROJECT_SOURCE_DIR}/spdk SOURCE_DIR ${PROJECT_SOURCE_DIR}/spdk BUILD_IN_SOURCE ${PROJECT_SOURCE_DIR}/spdk - CONFIGURE_COMMAND "./configure" + CONFIGURE_COMMAND "./configure" --with-dpdk=${PROJECT_SOURCE_DIR}/dpdk/build --with-igb-uio-driver BUILD_COMMAND ${CMAKE_MAKE_PROGRAM} INSTALL_COMMAND ${ROOT_DAQDB_DIR}/scripts/prepare_spdk_libs.sh ) +add_dependencies(project_spdk project_dpdk) + add_library(spdk STATIC IMPORTED GLOBAL) set_property(TARGET spdk PROPERTY IMPORTED_LOCATION ${PROJECT_SOURCE_DIR}/spdk/libspdk.a) add_dependencies(spdk project_spdk) -add_library(dpdk STATIC IMPORTED GLOBAL) -set_property(TARGET dpdk PROPERTY IMPORTED_LOCATION ${PROJECT_SOURCE_DIR}/spdk/libdpdk.a) -add_dependencies(dpdk project_spdk) add_custom_target(libspdk_clean COMMAND ${CMAKE_MAKE_PROGRAM} clean COMMAND rm -f ${PROJECT_SOURCE_DIR}/spdk/libspdk.a - COMMAND rm -f ${PROJECT_SOURCE_DIR}/spdk/libdpdk.a WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/spdk ) From df497ac981f4423f6c3d53801538718558857edf Mon Sep 17 00:00:00 2001 From: Jakub Radtke Date: Tue, 5 Mar 2019 11:34:26 -0500 Subject: [PATCH 09/10] Update daqdb_thin and functest_thin to work with DPDK transport Signed-off-by: Jakub Radtke --- CMakeLists.txt | 29 +++++++++++++++++++++------ lib/core/KVStore.cpp | 9 +++++---- lib/dht/DhtServer.cpp | 36 ++++++++++++++++++---------------- lib/thin/KVStoreThin.cpp | 6 +++++- lib/thin/KVStoreThin.h | 12 ++++++++++-- tests/functional_thin/main.cpp | 2 ++ 6 files changed, 64 insertions(+), 30 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b57ed1e..2263fc1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -74,15 +74,32 @@ add_subdirectory(${3RDPARTY}) include_directories(${LIBCONFIG_INCLUDES_EXPORT}) include_directories(${3RDPARTY}/eRPC-fork/src) include_directories(${3RDPARTY}/dpdk/build/include) - -file(GLOB_RECURSE DAQDB_THIN_SOURCES lib/thin/*.cpp lib/common/*.cpp - lib/dht/*.cpp lib/KVStoreBase.cpp) -list(FILTER DAQDB_THIN_SOURCES EXCLUDE REGEX "lib/dht/.*Server.*\.cpp$") include_directories(lib/common lib/dht lib/thin) include_directories(${HDRHISTOGRAM_INCLUDES_EXPORT}) -add_library(daqdb_thin SHARED ${DAQDB_THIN_SOURCES}) + +if(ERPC_RAW_TRANSPORT) + file(GLOB_RECURSE DAQDB_THIN_SOURCES lib/thin/*.cpp lib/common/*.cpp + lib/dht/*.cpp lib/KVStoreBase.cpp) + list(FILTER DAQDB_THIN_SOURCES EXCLUDE REGEX "lib/dht/.*Server.*\.cpp$") + + add_library(daqdb_thin SHARED ${DAQDB_THIN_SOURCES}) + target_link_libraries(daqdb_thin ${Boost_LIBRARIES} libconfig ${ERPC_LIBS}) + +else(ERPC_RAW_TRANSPORT) + include_directories(lib/offload lib/pmem lib/core lib/spdk lib/primary + lib/spdk ${3RDPARTY}/spdk/include) + + file(GLOB_RECURSE DAQDB_THIN_SOURCES lib/thin/*.cpp lib/common/*.cpp + lib/dht/*.cpp lib/KVStoreBase.cpp lib/spdk/*.cpp) + list(FILTER DAQDB_THIN_SOURCES EXCLUDE REGEX "lib/dht/.*Server.*\.cpp$") + set(Spdk_LIBRARIES -Wl,--whole-archive spdk -Wl,--no-whole-archive pthread + rt uuid) + + add_library(daqdb_thin SHARED ${DAQDB_THIN_SOURCES}) + target_link_libraries(daqdb_thin ${Spdk_LIBRARIES} ${Boost_LIBRARIES} libconfig ${ERPC_LIBS}) +endif(ERPC_RAW_TRANSPORT) + target_compile_definitions(daqdb_thin PRIVATE THIN_LIB=1) -target_link_libraries(daqdb_thin ${Boost_LIBRARIES} libconfig ${ERPC_LIBS}) if(NOT THIN_LIB) include_directories(${3RDPARTY}/pmdk/src/include) diff --git a/lib/core/KVStore.cpp b/lib/core/KVStore.cpp index 0d04f8f..d63348c 100644 --- a/lib/core/KVStore.cpp +++ b/lib/core/KVStore.cpp @@ -107,13 +107,14 @@ void KVStore::init() { new DhtServer(getDhtCore(), this, DHT_SERVER_WORKER_THREADS)); if (_spDhtServer->state == DhtServerState::DHT_SERVER_READY) { DAQ_DEBUG("DHT server started successfully"); + + if (_spDht->getLocalNode()->getPeerPort() > 0) { + _spDht->initNexus(_spDht->getLocalNode()->getPeerPort()); + _spDht->initClient(); + } } else { DAQ_DEBUG("Can not start DHT server"); } - if (_spDht->getLocalNode()->getPeerPort() > 0) { - _spDht->initNexus(_spDht->getLocalNode()->getPeerPort()); - _spDht->initClient(); - } if (isOffloadEnabled()) { _spOffloadPoller.reset(new DaqDB::OffloadPoller( diff --git a/lib/dht/DhtServer.cpp b/lib/dht/DhtServer.cpp index 1cadb4f..64ad836 100644 --- a/lib/dht/DhtServer.cpp +++ b/lib/dht/DhtServer.cpp @@ -11,7 +11,7 @@ * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and - * limitations under the License. + * limitations under the License. */ #include "DhtServer.h" @@ -30,9 +30,9 @@ #define DHT_SERVER_CPU_CORE_MAX 32 /** - * @TODO jradtke: not needed when eRPC implements configurable size of - * pre_resp_msgbuf - */ + * @TODO jradtke: not needed when eRPC implements configurable size of + * pre_resp_msgbuf + */ const unsigned int PRE_BUF_SIZE = 32 * 1024; namespace DaqDB { @@ -71,9 +71,9 @@ static void erpcReqGetAnyHandler(erpc::ReqHandle *req_handle, void *ctx) { erpc::MsgBuffer *resp; try { - resp = - erpcPrepareMsgbuf(rpc, req_handle, sizeof(DaqdbDhtResult) + - serverCtx->kvs->KeySize()); + resp = erpcPrepareMsgbuf(rpc, req_handle, + sizeof(DaqdbDhtResult) + + serverCtx->kvs->KeySize()); DaqdbDhtResult *result = reinterpret_cast(resp->buf); result->msgSize = serverCtx->kvs->KeySize(); serverCtx->kvs->GetAny(result->msg, serverCtx->kvs->KeySize()); @@ -182,11 +182,11 @@ DhtServer::~DhtServer() { _thread->join(); } -void DhtServer::_serveWorker(unsigned int workerId, cpu_set_t *cpuset, size_t size) { +void DhtServer::_serveWorker(unsigned int workerId, cpu_set_t *cpuset, + size_t size) { DhtServerCtx rpcCtx; - const int set_result = pthread_setaffinity_np(pthread_self(), - size, cpuset); + const int set_result = pthread_setaffinity_np(pthread_self(), size, cpuset); if (!set_result) { DAQ_DEBUG("Cannot set affinity for DHT server worker[" + to_string(workerId) + "]"); @@ -225,8 +225,7 @@ void DhtServer::_serve(void) { CPU_ZERO_S(size, cpuset); CPU_SET_S(DHT_SERVER_CPU_CORE_BASE, size, cpuset); - const int set_result = pthread_setaffinity_np(pthread_self(), - size, cpuset); + const int set_result = pthread_setaffinity_np(pthread_self(), size, cpuset); if (!set_result) { DAQ_DEBUG("Cannot set affinity for DHT server thread"); } @@ -259,7 +258,7 @@ void DhtServer::_serve(void) { CPU_ZERO_S(size, cpuset); CPU_SET_S(DHT_SERVER_CPU_CORE_BASE + threadIndex, size, cpuset); _workerThreads.push_back(thread(&DhtServer::_serveWorker, this, - threadIndex, cpuset, size)); + threadIndex, cpuset, size)); } state = DhtServerState::DHT_SERVER_READY; @@ -276,19 +275,22 @@ void DhtServer::_serve(void) { if (rpc) { delete rpc; } + CPU_FREE(cpuset); } catch (exception &e) { DAQ_DEBUG("DHT server exception: " + std::string(e.what())); - state = DhtServerState::DHT_SERVER_ERROR; + CPU_FREE(cpuset); - throw; + auto isInitFail = (state == DhtServerState::DHT_SERVER_INIT); + state = DhtServerState::DHT_SERVER_ERROR; + + if (!isInitFail) + throw; } catch (...) { DAQ_DEBUG("DHT server exception: unknown"); state = DhtServerState::DHT_SERVER_ERROR; CPU_FREE(cpuset); throw; } - - CPU_FREE(cpuset); } void DhtServer::serve(void) { diff --git a/lib/thin/KVStoreThin.cpp b/lib/thin/KVStoreThin.cpp index 86c8d76..fa458b0 100644 --- a/lib/thin/KVStoreThin.cpp +++ b/lib/thin/KVStoreThin.cpp @@ -11,7 +11,7 @@ * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and - * limitations under the License. + * limitations under the License. */ #include "KVStoreThin.h" @@ -40,6 +40,10 @@ void KVStoreThin::init() { if (getOptions().runtime.logFunc) gLog.setLogFunc(getOptions().runtime.logFunc); +#ifdef DPDK + _spSpdk.reset(new SpdkCore(getOptions().offload)); +#endif + _spDht.reset(new DhtCore(getOptions().dht)); _spDht->initNexus(); _spDht->initClient(); diff --git a/lib/thin/KVStoreThin.h b/lib/thin/KVStoreThin.h index 993f9ca..f7326e0 100644 --- a/lib/thin/KVStoreThin.h +++ b/lib/thin/KVStoreThin.h @@ -11,12 +11,16 @@ * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and - * limitations under the License. + * limitations under the License. */ #pragma once -#include +#ifdef DPDK +#include /* net/if.h (put before linux/if.h) */ +#endif + +#include /* include linux/if.h */ #include namespace DaqDB { @@ -83,6 +87,10 @@ class KVStoreThin : public KVStoreBase { size_t _keySize; Options _options; +#ifdef DPDK + std::unique_ptr _spSpdk; +#endif + std::unique_ptr _spDht; }; diff --git a/tests/functional_thin/main.cpp b/tests/functional_thin/main.cpp index 49e6017..f42b8f2 100644 --- a/tests/functional_thin/main.cpp +++ b/tests/functional_thin/main.cpp @@ -91,11 +91,13 @@ int main(int argc, const char *argv[]) { return -1; } +#ifndef DPDK if (!executeTest("testRemotePeerConnect", testRemotePeerConnect, nullptr, &options)) { DAQDB_INFO << "Cannot connect to peer node" << endl; return -1; } +#endif unique_ptr spKVStore; try { From 8bbe5e4a7a99c53965a9cd6335046ad6566707dd Mon Sep 17 00:00:00 2001 From: Jakub Radtke Date: Wed, 6 Mar 2019 08:25:45 -0500 Subject: [PATCH 10/10] Update functional test configuration Signed-off-by: Jakub Radtke --- .../functests_thin_clinode.cfg.in | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/tests/functional_thin/functests_thin_clinode.cfg.in b/tests/functional_thin/functests_thin_clinode.cfg.in index c90391a..ab93a87 100644 --- a/tests/functional_thin/functests_thin_clinode.cfg.in +++ b/tests/functional_thin/functests_thin_clinode.cfg.in @@ -27,23 +27,6 @@ alloc_unit_size = 16384; */ logging_level = "DEBUG"; -//***************************************************************************** -//******************************* DATA SECTION ******************************** -//***************************************************************************** - -/** - * keys_structure - defines how compound structure of a key looks like - * format: /1/8/4/16, representing 29 bytes total key - * with four parts - * primaryKey - primary key is being used to derive positioning of - * data. It is defined by operation on key parts. - * described by consecutive numbers starting with 1, e.g. - * 1*3 - * Result has to be unique for all the key stored. - */ -keys_structure = [8]; -primaryKey = "0"; - //***************************************************************************** //******************************* DHT SECTION ******************************** //*****************************************************************************