Support upstream DPDK in Dockerfile

cliffburdick · cliffburdick · commit 2a3e1e60e1b1 · 2025-11-26T17:32:23.000Z
diff --git a/operators/advanced_network/Dockerfile b/operators/advanced_network/Dockerfile
@@ -75,45 +75,26 @@ RUN --mount=type=bind,source=${HOLOSCAN_DEB_LOCAL_PATH},target=/tmp/holoscan.deb
 FROM hsdk-${HOLOSCAN_DEB_SRC} AS hsdk-installed
 
 # ==============================================================
-# common-deps: common dependencies for the advanced network lib
+# base-deps: base dependencies for the advanced network lib
 # ==============================================================
-FROM hsdk-installed AS common-deps
+FROM hsdk-installed AS base-deps
 
 ARG TARGETARCH
-ARG DOCA_VERSION=2.8.0
 ARG CACHEBUST=1
 ARG DEBIAN_FRONTEND=noninteractive
 
 WORKDIR /opt
 
-# Configure the DOCA APT repository
-RUN if [ "${TARGETARCH}" = "amd64" ]; then \
-        DOCA_ARCH="x86_64"; \
-    elif [ "$TARGETARCH" = "arm64" ]; then \
-        DOCA_ARCH="arm64-sbsa"; \
-    else \
-        echo "Unknown architecture: $TARGETARCH"; \
-        exit 1; \
-    fi \
-    && DISTRO=$(. /etc/os-release && echo ${ID}${VERSION_ID}) \
-    && DOCA_REPO_LINK=https://linux.mellanox.com/public/repo/doca/${DOCA_VERSION}/${DISTRO}/${DOCA_ARCH} \
-    && echo "Using DOCA_REPO_LINK=${DOCA_REPO_LINK}" \
-    && LOCAL_GPG_KEY_PATH="/usr/share/keyrings/mellanox-archive-keyring.gpg" \
-    && curl -fsSL ${DOCA_REPO_LINK}/GPG-KEY-Mellanox.pub | gpg --dearmor | tee ${LOCAL_GPG_KEY_PATH} > /dev/null \
-    && echo "deb [signed-by=${LOCAL_GPG_KEY_PATH}] ${DOCA_REPO_LINK} ./" | tee /etc/apt/sources.list.d/mellanox.list
-
-# APT installs
+# APT installs for base dependencies (no DOCA/GPUNetIO packages)
 # - cublas, cufft, cusolver, curand dev libs: for matx
 # - ninja-build: for cmake build
-# - pkgconf: to import dpdk and doca in CMake
-# - mlnx-dpdk-dev: for dpdk and gpunetio backends
-# - libdoca-sdk-dma-dev: for gpunetio backend (dependency of doca-gpunetio module)
-# - libdoca-sdk-gpunetio-dev: for gpunetio backend (doca-gpunetio module)
-# - libdoca-sdk-eth-dev: for gpunetio backend (doca-eth module)
-# - libdoca-sdk-flow-dev: for gpunetio backend (doca-flow module)
-# - mlnx-ofed-kernel-utils: utilities, including ibdev2netdev used by tune_system.py
+# - pkgconf: to import dpdk in CMake
+# - meson: for building DPDK
+# - python3-pyelftools: required for DPDK build
+# - libnuma-dev: NUMA support for DPDK
+# - libibverbs-dev, librdmacm-dev: RDMA/ibverbs support for Mellanox NICs
+# - libmlx5-1: Mellanox ConnectX driver
 # - ibverbs-utils: utilities
-# - python3-pyelftools: used by some mlnx tools
 # - python3-dev: for building python bindings
 RUN apt-get update && apt-get install -y --no-install-recommends \
         libcublas-dev-12-6 \
@@ -122,14 +103,13 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         libcurand-dev-12-6 \
         ninja-build \
         pkgconf \
-        mlnx-dpdk-dev \
-        libdoca-sdk-dma-dev \
-        libdoca-sdk-gpunetio-dev \
-        libdoca-sdk-eth-dev \
-        libdoca-sdk-flow-dev \
-        mlnx-ofed-kernel-utils \
-        ibverbs-utils \
+        meson \
         python3-pyelftools \
+        libnuma-dev \
+        libibverbs-dev \
+        librdmacm-dev \
+        libmlx5-1 \
+        ibverbs-utils \
         python3-dev \
     && rm -rf /var/lib/apt/lists/*
 
@@ -142,25 +122,107 @@ RUN python3 -m pip install --no-cache-dir \
         pyyaml \
         scapy
 
+# ==============================================================
+# dpdk-build: Build upstream DPDK from source
+# ==============================================================
+FROM base-deps AS dpdk-build
+
+ARG TARGETARCH
+# DPDK version to download and build
+ARG DPDK_VERSION=24.11.3
+ARG DPDK_BUILD_DIR=/opt/dpdk-build
+ARG DPDK_INSTALL_PREFIX=/usr/local
+ARG DOCA_VERSION=2.8.0
+ARG DEBIAN_FRONTEND=noninteractive
+
+WORKDIR /tmp
+
+# Download DPDK source from official repository
+ADD https://fast.dpdk.org/rel/dpdk-${DPDK_VERSION}.tar.xz /tmp/
+RUN tar xf dpdk-${DPDK_VERSION}.tar.xz
+
+# Build and install DPDK using meson/ninja
+WORKDIR /tmp/dpdk-stable-${DPDK_VERSION}
+RUN meson setup ${DPDK_BUILD_DIR} \
+        --prefix=${DPDK_INSTALL_PREFIX} \
+        -Denable_kmods=false \
+        -Dtests=false \
+        -Denable_docs=false \
+        -Ddisable_drivers=baseband/*,bus/ifpga/*,common/cpt,common/dpaax,common/iavf,common/octeontx,common/octeontx2,crypto/nitrox,net/ark,net/atlantic,net/avp,net/axgbe,net/bnx2x,net/bnxt,net/cxgbe,net/e1000,net/ena,net/enic,net/fm10k,net/hinic,net/hns3,net/i40e,net/ixgbe,vdpa/ifc,net/igc,net/liquidio,net/netvsc,net/nfp,net/qede,net/sfc,net/thunderx,net/vdev_netvsc,net/vmxnet3,regex/octeontx2 \
+    && cd ${DPDK_BUILD_DIR} \
+    && ninja \
+    && meson install \
+    && ldconfig \
+    && rm -rf /tmp/dpdk-stable-${DPDK_VERSION} /tmp/dpdk-stable-${DPDK_VERSION}.tar.xz ${DPDK_BUILD_DIR} /tmp/dpdk_patches
+
+# Configure the DOCA APT repository for mlnx-ofed-kernel-utils
+# (needed for ibdev2netdev utility used by tune_system.py)
+RUN if [ "${TARGETARCH}" = "amd64" ]; then \
+        DOCA_ARCH="x86_64"; \
+    elif [ "$TARGETARCH" = "arm64" ]; then \
+        DOCA_ARCH="arm64-sbsa"; \
+    else \
+        echo "Unknown architecture: $TARGETARCH"; \
+        exit 1; \
+    fi \
+    && DISTRO=$(. /etc/os-release && echo ${ID}${VERSION_ID}) \
+    && DOCA_REPO_LINK=https://linux.mellanox.com/public/repo/doca/${DOCA_VERSION}/${DISTRO}/${DOCA_ARCH} \
+    && echo "Using DOCA_REPO_LINK=${DOCA_REPO_LINK}" \
+    && LOCAL_GPG_KEY_PATH="/usr/share/keyrings/mellanox-archive-keyring.gpg" \
+    && curl -fsSL ${DOCA_REPO_LINK}/GPG-KEY-Mellanox.pub | gpg --dearmor | tee ${LOCAL_GPG_KEY_PATH} > /dev/null \
+    && echo "deb [signed-by=${LOCAL_GPG_KEY_PATH}] ${DOCA_REPO_LINK} ./" | tee /etc/apt/sources.list.d/mellanox.list
+
+# Install mlnx-ofed-kernel-utils (utilities including ibdev2netdev used by tune_system.py)
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        mlnx-ofed-kernel-utils \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /opt
+
 # ==============================
-# DOCA Target
-# This stage is only built when --target doca is specified. It contains any DOCA-specific configurations.
+# DPDK Target (Default)
+# This is the default target for users who only need DPDK support.
+# GPUNetIO/DOCA is NOT included.
 # ==============================
-FROM common-deps AS doca
-# DOCA-specific installation or steps (if needed)
+FROM dpdk-build AS dpdk
+# DPDK is ready to use, no additional steps needed
+
+# ==============================================================
+# gpunetio-deps: Add DOCA SDK packages for GPUNetIO support
+# ==============================================================
+FROM dpdk-build AS gpunetio-deps
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+# Install DOCA SDK packages required for GPUNetIO
+# (DOCA repo is already configured in dpdk-build stage)
+# - libdoca-sdk-dma-dev: for gpunetio backend (dependency of doca-gpunetio module)
+# - libdoca-sdk-gpunetio-dev: for gpunetio backend (doca-gpunetio module)
+# - libdoca-sdk-eth-dev: for gpunetio backend (doca-eth module)
+# - libdoca-sdk-flow-dev: for gpunetio backend (doca-flow module)
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        libdoca-sdk-dma-dev \
+        libdoca-sdk-gpunetio-dev \
+        libdoca-sdk-eth-dev \
+        libdoca-sdk-flow-dev \
+    && rm -rf /var/lib/apt/lists/*
 
 # ==============================
-# DPDK Target
-# This stage is only built when --target dpdk is specified. It contains any DPDK-specific configurations.
+# GPUNetIO Target
+# This stage is built when --target gpunetio is specified.
+# It includes DOCA packages required for GPUNetIO support.
 # ==============================
-FROM common-deps AS dpdk
-# DPDK-specific installation or steps (if needed)
+FROM gpunetio-deps AS gpunetio
+# GPUNetIO is ready to use with DOCA support
 
 # ==============================
 # Rivermax Target
 # This stage is only built when --target rivermax is specified. It installs and configures Rivermax SDK.
+# Note: Rivermax does not require DPDK - extends from base-deps directly.
 # ==============================
-FROM common-deps AS rivermax
+FROM base-deps AS rivermax
+
+ARG TARGETARCH
 
 # Define Rivermax-specific build arguments and environment variables
 ARG RIVERMAX_VERSION=1.70.32
@@ -227,7 +289,9 @@ RUN cd rivermax-dev-kit && \
     cmake --build build -j $(nproc)
 
 # ==============================
-# Default stage: common-deps
-# If no target is specified, the common-deps stage will be built by default.
+# Default stage: dpdk
+# If no target is specified, the dpdk stage will be built by default.
+# This provides upstream DPDK support without DOCA dependencies.
+# Use --target gpunetio if you need DOCA GPUNetIO support.
 # ==============================
-FROM common-deps
+FROM dpdk
diff --git a/operators/advanced_network/README.md b/operators/advanced_network/README.md
@@ -26,9 +26,9 @@ is available in userspace, thus bypassing the kernel's networking stack entirely
 - Linux
 - An NVIDIA NIC with a ConnectX-6 or later chip
 - System tuning as described [here](/tutorials/high_performance_networking/README.md)
-- DPDK 22.11
-- MOFED 5.8-1.0.1.1 or later
-- DOCA 2.7 or later
+- DPDK 24.11.3 or higher
+- MOFED 5.8-1.0.1.1 or later (included with DOCA package)
+- DOCA 2.8 or later if using GPUNetIO
 
 #### Features
 
diff --git a/operators/advanced_network/advanced_network/CMakeLists.txt b/operators/advanced_network/advanced_network/CMakeLists.txt
@@ -20,6 +20,7 @@ find_package(holoscan 2.6 REQUIRED CONFIG
              PATHS "/opt/nvidia/holoscan" "/workspace/holoscan-sdk/install") # Remove if holoscan logger dependency removed
 
 find_package(PkgConfig)
+find_package(CUDAToolkit REQUIRED)
 
 enable_language(CUDA)
 
@@ -31,9 +32,16 @@ add_compile_definitions(DOCA_ALLOW_EXPERIMENTAL_API)
 
 execute_process(COMMAND arch OUTPUT_VARIABLE ARCH)
 string(STRIP ${ARCH} ARCH)
-set(DPDK_PATH /opt/mellanox/dpdk)
+
+# Path configurations for DPDK and DOCA
+# Upstream DPDK (installed via meson) uses /usr/local by default
+# DOCA/mlnx-dpdk uses /opt/mellanox paths
+set(DPDK_PATH_UPSTREAM /usr/local)
+set(DPDK_PATH_MLNX /opt/mellanox/dpdk)
 set(DOCA_PATH /opt/mellanox/doca)
-set(ENV{PKG_CONFIG_PATH} "$ENV{PKG_CONFIG_PATH}:${DPDK_PATH}/lib/${ARCH}-linux-gnu/pkgconfig:${DOCA_PATH}/lib/${ARCH}-linux-gnu/pkgconfig:/opt/mellanox/flexio/lib/pkgconfig:/opt/mellanox/collectx/lib/${ARCH}-linux-gnu/pkgconfig")
+
+# Set PKG_CONFIG_PATH to find DPDK (upstream first, then mlnx) and DOCA
+set(ENV{PKG_CONFIG_PATH} "$ENV{PKG_CONFIG_PATH}:${DPDK_PATH_UPSTREAM}/lib/${ARCH}-linux-gnu/pkgconfig:${DPDK_PATH_UPSTREAM}/lib/pkgconfig:${DPDK_PATH_MLNX}/lib/${ARCH}-linux-gnu/pkgconfig:${DOCA_PATH}/lib/${ARCH}-linux-gnu/pkgconfig:/opt/mellanox/flexio/lib/pkgconfig:/opt/mellanox/collectx/lib/${ARCH}-linux-gnu/pkgconfig")
 
 # Common library
 add_library(advanced_network_common SHARED
@@ -71,8 +79,10 @@ install(
 # Backend libraries
 if(NOT DEFINED ANO_MGR)
     # Initialize ANO_MGR with a default if not provided
-    # Note: rivermax not added to the default list yet since it requires licensing
-    set(ANO_MGR "dpdk gpunetio" CACHE STRING "Manager type(s) list")
+    # Default: dpdk only (no DOCA dependency)
+    # Add "gpunetio" to enable GPU-accelerated networking (requires DOCA)
+    # Note: rivermax not added to the default list since it requires licensing
+    set(ANO_MGR "dpdk" CACHE STRING "Manager type(s) list")
 endif()
 separate_arguments(ANO_MGR_LIST UNIX_COMMAND ${ANO_MGR})
 
diff --git a/operators/advanced_network/advanced_network/managers/dpdk/CMakeLists.txt b/operators/advanced_network/advanced_network/managers/dpdk/CMakeLists.txt
@@ -20,38 +20,39 @@ target_include_directories(${PROJECT_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
 
 target_sources(${PROJECT_NAME} PRIVATE adv_network_dpdk_mgr.cpp adv_network_dpdk_stats.cpp)
 
+# Try to find DPDK via pkg-config
+# This will find upstream DPDK (installed to /usr/local) or mlnx-dpdk (via PKG_CONFIG_PATH set in parent CMakeLists.txt)
 pkg_check_modules(DPDK QUIET libdpdk)
 
 if(NOT DPDK_FOUND)
-  message(STATUS "Looking for DPDK in alternative directories")
-  set(ENV{PKG_CONFIG_PATH} /opt/mellanox/dpdk/lib/x86_64-linux-gnu/pkgconfig/)
+  # Fallback: try to find mlnx-dpdk in standard Mellanox installation paths
+  message(STATUS "Looking for DPDK in Mellanox installation directories")
+  execute_process(COMMAND arch OUTPUT_VARIABLE ARCH)
+  string(STRIP ${ARCH} ARCH)
+  set(ENV{PKG_CONFIG_PATH} "/opt/mellanox/dpdk/lib/${ARCH}-linux-gnu/pkgconfig/")
   pkg_check_modules(DPDK REQUIRED libdpdk)
-
-  target_link_directories(${PROJECT_NAME} PUBLIC ${DPDK_LIBRARY_DIRS})
-  target_include_directories(${PROJECT_NAME} PUBLIC ${DPDK_INCLUDE_DIRS})
-  target_compile_options(${PROJECT_NAME} PUBLIC ${DPDK_CFLAGS_OTHER})
-  target_link_libraries(${PROJECT_NAME}
-    PUBLIC
-      ${DPDK_LDFLAGS_OTHER}
-      ${DPDK_LIBRARIES}
-    PRIVATE
-      holoscan::core
-      CUDA::cudart
-  )
-else() # Upstream DPDK
-  set(DPDK_EXTRA_LIBS -Wl,--no-whole-archive -lmlx5 -libverbs -pthread -lnuma -ldl)
-  target_link_directories(${PROJECT_NAME} PUBLIC ${DPDK_LIBRARY_DIRS})
-  target_link_libraries(${PROJECT_NAME}
-    PUBLIC
-      ${DPDK_LDFLAGS_OTHER}
-      ${DPDK_LIBRARIES}
-      ${DPDK_EXTRA_LIBS}
-    PRIVATE
-      holoscan::core
-      CUDA::cudart
-  )
 endif()
 
+message(STATUS "Found DPDK: ${DPDK_VERSION} in ${DPDK_LIBRARY_DIRS}")
+
+# Common DPDK configuration
+target_link_directories(${PROJECT_NAME} PUBLIC ${DPDK_LIBRARY_DIRS})
+target_include_directories(${PROJECT_NAME} PUBLIC ${DPDK_INCLUDE_DIRS})
+target_compile_options(${PROJECT_NAME} PUBLIC ${DPDK_CFLAGS_OTHER})
+
+# Link libraries - add extra libs for Mellanox NIC support
+set(DPDK_EXTRA_LIBS -Wl,--no-whole-archive -lmlx5 -libverbs -pthread -lnuma -ldl)
+target_link_libraries(${PROJECT_NAME}
+  PUBLIC
+    ${DPDK_LDFLAGS_OTHER}
+    ${DPDK_LIBRARIES}
+    ${DPDK_EXTRA_LIBS}
+  PRIVATE
+    holoscan::core
+    CUDA::cudart
+    CUDA::cuda_driver
+)
+
 set_target_properties(${PROJECT_NAME} PROPERTIES CUDA_SEPARABLE_COMPILATION ON)
 set_target_properties(${PROJECT_NAME} PROPERTIES CUDA_RESOLVE_DEVICE_SYMBOLS ON)
 set_target_properties(${PROJECT_NAME} PROPERTIES CUDA_ARCHITECTURES "80;90")
diff --git a/operators/advanced_network/advanced_network/managers/gpunetio/CMakeLists.txt b/operators/advanced_network/advanced_network/managers/gpunetio/CMakeLists.txt
@@ -64,6 +64,8 @@ target_link_directories(${PROJECT_NAME} PUBLIC ${DOCA_LIBRARY_DIRS})
 target_link_libraries(${PROJECT_NAME}
   PRIVATE
     holoscan::core
+    CUDA::cudart
+    CUDA::cuda_driver
     -ldoca_gpunetio
     -ldoca_gpunetio_device # static
     -ldoca_eth
