diff --git a/operators/advanced_network/Dockerfile b/operators/advanced_network/Dockerfile index 8229525c1d..760f0481d7 100644 --- a/operators/advanced_network/Dockerfile +++ b/operators/advanced_network/Dockerfile @@ -75,18 +75,91 @@ RUN --mount=type=bind,source=${HOLOSCAN_DEB_LOCAL_PATH},target=/tmp/holoscan.deb FROM hsdk-${HOLOSCAN_DEB_SRC} AS hsdk-installed # ============================================================== -# common-deps: common dependencies for the advanced network lib +# base-deps: base dependencies for the advanced network lib # ============================================================== -FROM hsdk-installed AS common-deps +FROM hsdk-installed AS base-deps ARG TARGETARCH -ARG DOCA_VERSION=2.8.0 ARG CACHEBUST=1 ARG DEBIAN_FRONTEND=noninteractive WORKDIR /opt -# Configure the DOCA APT repository +# APT installs for base dependencies (no DOCA/GPUNetIO packages) +# - libibverbs-dev, librdmacm-dev: RDMA/ibverbs support for Mellanox NICs +# - libmlx5-1: Mellanox ConnectX driver +# - ibverbs-utils: utilities +# - python3-dev: for building python bindings +RUN apt-get update && apt-get install -y --no-install-recommends \ + libibverbs-dev \ + librdmacm-dev \ + libmlx5-1 \ + ibverbs-utils \ + python3-dev \ + && rm -rf /var/lib/apt/lists/* + +# PIP installs +# - pytest: test harness +# - pyyaml: to parse yaml configs in tests +# - scapy: for debugging and mocking network packets for tests +RUN python3 -m pip install --no-cache-dir \ + pytest \ + pyyaml \ + scapy + +# ============================================================== +# dpdk-deps: Build upstream DPDK from source +# ============================================================== +FROM base-deps AS dpdk-deps + +ARG TARGETARCH +# DPDK version to download and build +ARG DPDK_VERSION=24.11.3 +ARG DPDK_BUILD_DIR=/opt/dpdk-build +ARG DPDK_INSTALL_PREFIX=/usr/local +ARG DOCA_VERSION=2.8.0 +ARG DEBIAN_FRONTEND=noninteractive + + +# - infiniband-diags for IB diagnostics +# - mft for Mellanox Flex Transport +# - ninja-build: for cmake build +# - pkgconf: to import dpdk in CMake +# - meson: for building DPDK +# - python3-pyelftools: required for DPDK build +# - libnuma-dev: NUMA support for DPDK +RUN apt-get update && apt-get install -y --no-install-recommends \ + infiniband-diags \ + mft \ + ninja-build \ + pkgconf \ + meson \ + python3-pyelftools \ + libnuma-dev \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /tmp + +# Download DPDK source from official repository +ADD https://fast.dpdk.org/rel/dpdk-${DPDK_VERSION}.tar.xz /tmp/ +RUN tar xf dpdk-${DPDK_VERSION}.tar.xz + +# Build and install DPDK using meson/ninja +WORKDIR /tmp/dpdk-stable-${DPDK_VERSION} +RUN meson setup ${DPDK_BUILD_DIR} \ + --prefix=${DPDK_INSTALL_PREFIX} \ + -Denable_kmods=false \ + -Dtests=false \ + -Denable_docs=false \ + -Ddisable_drivers=baseband/*,bus/ifpga/*,common/cpt,common/dpaax,common/iavf,common/octeontx,common/octeontx2,crypto/nitrox,net/ark,net/atlantic,net/avp,net/axgbe,net/bnx2x,net/bnxt,net/cxgbe,net/e1000,net/ena,net/enic,net/fm10k,net/hinic,net/hns3,net/i40e,net/ixgbe,vdpa/ifc,net/igc,net/liquidio,net/netvsc,net/nfp,net/qede,net/sfc,net/thunderx,net/vdev_netvsc,net/vmxnet3,regex/octeontx2 \ + && cd ${DPDK_BUILD_DIR} \ + && ninja \ + && meson install \ + && ldconfig \ + && rm -rf /tmp/dpdk-stable-${DPDK_VERSION} /tmp/dpdk-stable-${DPDK_VERSION}.tar.xz ${DPDK_BUILD_DIR} /tmp/dpdk_patches + +# Configure the DOCA APT repository for mlnx-ofed-kernel-utils +# (needed for ibdev2netdev utility used by tune_system.py) RUN if [ "${TARGETARCH}" = "amd64" ]; then \ DOCA_ARCH="x86_64"; \ elif [ "$TARGETARCH" = "arm64" ]; then \ @@ -102,65 +175,47 @@ RUN if [ "${TARGETARCH}" = "amd64" ]; then \ && curl -fsSL ${DOCA_REPO_LINK}/GPG-KEY-Mellanox.pub | gpg --dearmor | tee ${LOCAL_GPG_KEY_PATH} > /dev/null \ && echo "deb [signed-by=${LOCAL_GPG_KEY_PATH}] ${DOCA_REPO_LINK} ./" | tee /etc/apt/sources.list.d/mellanox.list -# APT installs -# - cublas, cufft, cusolver, curand dev libs: for matx -# - ninja-build: for cmake build -# - pkgconf: to import dpdk and doca in CMake -# - mlnx-dpdk-dev: for dpdk and gpunetio backends +# - mlnx-ofed-kernel-utils for utilities including ibdev2netdev used by tune_system.py +RUN apt-get update && apt-get install -y --no-install-recommends \ + mlnx-ofed-kernel-utils + + +WORKDIR /opt + +# ============================================================== +# gpunetio-deps: Add DOCA SDK packages for GPUNetIO support +# ============================================================== +FROM dpdk-deps AS gpunetio-deps + +# Install DOCA SDK packages required for GPUNetIO +# (DOCA repo is already configured in dpdk-deps stage) # - libdoca-sdk-dma-dev: for gpunetio backend (dependency of doca-gpunetio module) # - libdoca-sdk-gpunetio-dev: for gpunetio backend (doca-gpunetio module) # - libdoca-sdk-eth-dev: for gpunetio backend (doca-eth module) # - libdoca-sdk-flow-dev: for gpunetio backend (doca-flow module) -# - mlnx-ofed-kernel-utils: utilities, including ibdev2netdev used by tune_system.py -# - ibverbs-utils: utilities -# - python3-pyelftools: used by some mlnx tools -# - python3-dev: for building python bindings RUN apt-get update && apt-get install -y --no-install-recommends \ - libcublas-dev-12-6 \ - libcufft-dev-12-6 \ - libcusolver-dev-12-6 \ - libcurand-dev-12-6 \ - ninja-build \ - pkgconf \ - mlnx-dpdk-dev \ libdoca-sdk-dma-dev \ libdoca-sdk-gpunetio-dev \ libdoca-sdk-eth-dev \ libdoca-sdk-flow-dev \ - mlnx-ofed-kernel-utils \ - ibverbs-utils \ - python3-pyelftools \ - python3-dev \ && rm -rf /var/lib/apt/lists/* -# PIP installs -# - pytest: test harness -# - pyyaml: to parse yaml configs in tests -# - scapy: for debugging and mocking network packets for tests -RUN python3 -m pip install --no-cache-dir \ - pytest \ - pyyaml \ - scapy - -# ============================== -# DOCA Target -# This stage is only built when --target doca is specified. It contains any DOCA-specific configurations. -# ============================== -FROM common-deps AS doca -# DOCA-specific installation or steps (if needed) - # ============================== -# DPDK Target -# This stage is only built when --target dpdk is specified. It contains any DPDK-specific configurations. +# GPUNetIO Target +# This stage is built when --target gpunetio is specified. +# It includes DOCA packages required for GPUNetIO support. # ============================== -FROM common-deps AS dpdk -# DPDK-specific installation or steps (if needed) +FROM gpunetio-deps AS gpunetio +# GPUNetIO is ready to use with DOCA support # ============================== # Rivermax Target # This stage is only built when --target rivermax is specified. It installs and configures Rivermax SDK. +# Note: Rivermax does not require DPDK - extends from base-deps directly. # ============================== -FROM common-deps AS rivermax +FROM base-deps AS rivermax + +ARG TARGETARCH # Define Rivermax-specific build arguments and environment variables ARG RIVERMAX_VERSION=1.70.32 @@ -227,7 +282,9 @@ RUN cd rivermax-dev-kit && \ cmake --build build -j $(nproc) # ============================== -# Default stage: common-deps -# If no target is specified, the common-deps stage will be built by default. +# Default stage: dpdk +# If no target is specified, the dpdk stage will be built by default. +# This provides upstream DPDK support without DOCA dependencies. +# Use --target gpunetio if you need DOCA GPUNetIO support. # ============================== -FROM common-deps +FROM dpdk-deps diff --git a/operators/advanced_network/README.md b/operators/advanced_network/README.md index 1fc1bc023c..aa254d122d 100644 --- a/operators/advanced_network/README.md +++ b/operators/advanced_network/README.md @@ -26,9 +26,16 @@ is available in userspace, thus bypassing the kernel's networking stack entirely - Linux - An NVIDIA NIC with a ConnectX-6 or later chip - System tuning as described [here](/tutorials/high_performance_networking/README.md) -- DPDK 22.11 -- MOFED 5.8-1.0.1.1 or later -- DOCA 2.7 or later +- DPDK 24.11.3 or higher +- MOFED 5.8-1.0.1.1 or later (included with DOCA package) +- MLNX5/IB drivers with peermem support - either through: + - Inbox drivers (ubuntu kernel >= 5.4 and [< 6.8](https://discourse.ubuntu.com/t/nvidia-gpudirect-over-infiniband-migration-paths/44425)) + - NVIDIA optimized kernels (IGX OS, DGX BaseOS) + - MLNX-OFED drivers, either from: + - [DOCA-Host](https://developer.nvidia.com/doca-archive) 2.8 or later (install `mlnx-ofed-kernel-dkms` package or the `doca-ofed` meta-package for extra tooling) + - _(deprecated)_ [MOFED](https://network.nvidia.com/products/infiniband-drivers/linux/mlnx_ofed/) 23.10 or later (`sudo ./mlnxofedinstall --kernel-only`) + +> User-space libraries are included in the [Dockerfile](./Dockerfile) for each networking backend. Inspect this file if you wish to know what is needed to build and run on baremetal instead. #### Features diff --git a/operators/advanced_network/advanced_network/CMakeLists.txt b/operators/advanced_network/advanced_network/CMakeLists.txt index 2a921d9b81..7cc23fe82d 100644 --- a/operators/advanced_network/advanced_network/CMakeLists.txt +++ b/operators/advanced_network/advanced_network/CMakeLists.txt @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,6 +20,7 @@ find_package(holoscan 2.6 REQUIRED CONFIG PATHS "/opt/nvidia/holoscan" "/workspace/holoscan-sdk/install") # Remove if holoscan logger dependency removed find_package(PkgConfig) +find_package(CUDAToolkit REQUIRED) enable_language(CUDA) @@ -31,9 +32,13 @@ add_compile_definitions(DOCA_ALLOW_EXPERIMENTAL_API) execute_process(COMMAND arch OUTPUT_VARIABLE ARCH) string(STRIP ${ARCH} ARCH) -set(DPDK_PATH /opt/mellanox/dpdk) + +# Path configurations for DPDK and DOCA +# Upstream DPDK (installed via meson) uses /usr/local by default +# DOCA/mlnx-dpdk uses /opt/mellanox paths +set(DPDK_PATH_UPSTREAM /usr/local) +set(DPDK_PATH_MLNX /opt/mellanox/dpdk) set(DOCA_PATH /opt/mellanox/doca) -set(ENV{PKG_CONFIG_PATH} "$ENV{PKG_CONFIG_PATH}:${DPDK_PATH}/lib/${ARCH}-linux-gnu/pkgconfig:${DOCA_PATH}/lib/${ARCH}-linux-gnu/pkgconfig:/opt/mellanox/flexio/lib/pkgconfig:/opt/mellanox/collectx/lib/${ARCH}-linux-gnu/pkgconfig") # Common library add_library(advanced_network_common SHARED @@ -71,8 +76,10 @@ install( # Backend libraries if(NOT DEFINED ANO_MGR) # Initialize ANO_MGR with a default if not provided - # Note: rivermax not added to the default list yet since it requires licensing - set(ANO_MGR "dpdk gpunetio" CACHE STRING "Manager type(s) list") + # Default: dpdk only (no DOCA dependency) + # Add "gpunetio" to enable GPU-accelerated networking (requires DOCA) + # Note: rivermax not added to the default list since it requires licensing + set(ANO_MGR "dpdk" CACHE STRING "Manager type(s) list") endif() separate_arguments(ANO_MGR_LIST UNIX_COMMAND ${ANO_MGR}) diff --git a/operators/advanced_network/advanced_network/managers/dpdk/CMakeLists.txt b/operators/advanced_network/advanced_network/managers/dpdk/CMakeLists.txt index 8c61445135..bf4ac71671 100644 --- a/operators/advanced_network/advanced_network/managers/dpdk/CMakeLists.txt +++ b/operators/advanced_network/advanced_network/managers/dpdk/CMakeLists.txt @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,38 +20,39 @@ target_include_directories(${PROJECT_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) target_sources(${PROJECT_NAME} PRIVATE adv_network_dpdk_mgr.cpp adv_network_dpdk_stats.cpp) +# Try to find DPDK via pkg-config +# This will find upstream DPDK (installed to /usr/local) or mlnx-dpdk (via PKG_CONFIG_PATH set in parent CMakeLists.txt) pkg_check_modules(DPDK QUIET libdpdk) if(NOT DPDK_FOUND) - message(STATUS "Looking for DPDK in alternative directories") - set(ENV{PKG_CONFIG_PATH} /opt/mellanox/dpdk/lib/x86_64-linux-gnu/pkgconfig/) + # Fallback: try to find mlnx-dpdk in standard Mellanox installation paths + message(STATUS "Looking for DPDK in Mellanox installation directories") + execute_process(COMMAND arch OUTPUT_VARIABLE ARCH) + string(STRIP ${ARCH} ARCH) + list(APPEND CMAKE_PREFIX_PATH "/opt/mellanox/dpdk/lib/${ARCH}-linux-gnu/pkgconfig/") pkg_check_modules(DPDK REQUIRED libdpdk) - - target_link_directories(${PROJECT_NAME} PUBLIC ${DPDK_LIBRARY_DIRS}) - target_include_directories(${PROJECT_NAME} PUBLIC ${DPDK_INCLUDE_DIRS}) - target_compile_options(${PROJECT_NAME} PUBLIC ${DPDK_CFLAGS_OTHER}) - target_link_libraries(${PROJECT_NAME} - PUBLIC - ${DPDK_LDFLAGS_OTHER} - ${DPDK_LIBRARIES} - PRIVATE - holoscan::core - CUDA::cudart - ) -else() # Upstream DPDK - set(DPDK_EXTRA_LIBS -Wl,--no-whole-archive -lmlx5 -libverbs -pthread -lnuma -ldl) - target_link_directories(${PROJECT_NAME} PUBLIC ${DPDK_LIBRARY_DIRS}) - target_link_libraries(${PROJECT_NAME} - PUBLIC - ${DPDK_LDFLAGS_OTHER} - ${DPDK_LIBRARIES} - ${DPDK_EXTRA_LIBS} - PRIVATE - holoscan::core - CUDA::cudart - ) endif() +message(STATUS "Found DPDK: ${DPDK_VERSION} in ${DPDK_LIBRARY_DIRS}") + +# Common DPDK configuration +target_link_directories(${PROJECT_NAME} PUBLIC ${DPDK_LIBRARY_DIRS}) +target_include_directories(${PROJECT_NAME} PUBLIC ${DPDK_INCLUDE_DIRS}) +target_compile_options(${PROJECT_NAME} PUBLIC ${DPDK_CFLAGS_OTHER}) + +# Link libraries - add extra libs for Mellanox NIC support +set(DPDK_EXTRA_LIBS -Wl,--no-whole-archive -lmlx5 -libverbs -pthread -lnuma -ldl) +target_link_libraries(${PROJECT_NAME} + PUBLIC + ${DPDK_LDFLAGS_OTHER} + ${DPDK_LIBRARIES} + ${DPDK_EXTRA_LIBS} + PRIVATE + holoscan::core + CUDA::cudart + CUDA::cuda_driver +) + set_target_properties(${PROJECT_NAME} PROPERTIES CUDA_SEPARABLE_COMPILATION ON) set_target_properties(${PROJECT_NAME} PROPERTIES CUDA_RESOLVE_DEVICE_SYMBOLS ON) set_target_properties(${PROJECT_NAME} PROPERTIES CUDA_ARCHITECTURES "80;90") diff --git a/operators/advanced_network/advanced_network/managers/gpunetio/CMakeLists.txt b/operators/advanced_network/advanced_network/managers/gpunetio/CMakeLists.txt index 4b4c6fa2e6..e892b562d0 100644 --- a/operators/advanced_network/advanced_network/managers/gpunetio/CMakeLists.txt +++ b/operators/advanced_network/advanced_network/managers/gpunetio/CMakeLists.txt @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -64,6 +64,8 @@ target_link_directories(${PROJECT_NAME} PUBLIC ${DOCA_LIBRARY_DIRS}) target_link_libraries(${PROJECT_NAME} PRIVATE holoscan::core + CUDA::cudart + CUDA::cuda_driver -ldoca_gpunetio -ldoca_gpunetio_device # static -ldoca_eth