Update to DPDK v21.11, tested on Azure

jiangxiaosheng · Apr 17, 2023 · 51abae1 · 51abae1
1 parent 1ef5d5f
commit 51abae1
Show file tree

Hide file tree

Showing 4 changed files with 66 additions and 45 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -331,24 +331,39 @@ if(DPDK_NEEDED STREQUAL "true")
       "-lshell32" "-lole32" "-loleaut32" "-luuid" "-lcomdlg32" "-liphlpapi")
 
   else(WIN32)
-    message(STATUS "DPDK for Linux")
-    find_library(DPDK_LIB dpdk)
-    if(NOT DPDK_LIB)
-      message(FATAL_ERROR "DPDK library not found")
+    # Linux
+    if(NOT DEFINED ENV{RTE_SDK})
+      message(FATAL "RTE_SDK is not defined")
+    else()
+      set(RTE_SDK $ENV{RTE_SDK})
+    endif()
+    set(DPDK_INSTALL_DIR "${RTE_SDK}/build/install/usr/local")
+
+    # DPDK installs libraries into local/lib on Mariner, but
+    # local/lib/x86_64-linux-gnu on Ubuntu
+    set(DPDK_LIB_DIR "${DPDK_INSTALL_DIR}/lib/x86_64-linux-gnu")
+    if(NOT EXISTS ${DPDK_LIB_DIR})
+      set(DPDK_LIB_DIR "${DPDK_INSTALL_DIR}/lib")
+      if(NOT EXISTS ${DPDK_LIB_DIR})
+        message(FATAL_ERROR "${DPDK_LIB_DIR} not found. Did you run `ninja install`?")
+      endif()
     endif()
 
-    set(LIBRARIES ${LIBRARIES} -Wl,--whole-archive dpdk -Wl,--no-whole-archive numa dl ibverbs mlx4 mlx5)
-
-    # DPDK include directory. Locating rte_config.h does not work on some systems.
-    # Example: it may be kept in /usr/include/x86_64-linux-gnu/, and symlinked
-    # from the real DPDK include directory (/usr/include/dpdk/).
-    find_path(DPDK_INCLUDE_DIR NAMES rte_ethdev.h PATH_SUFFIXES dpdk)
-    if (DPDK_INCLUDE_DIR)
-      message(STATUS "DPDK include directory = ${DPDK_INCLUDE_DIR}")
-    else()
-      message(FATAL_ERROR "DPDK include directory not found")
+    # We use some hacky command-line and CMake magic to construct DPDK library list
+    execute_process(
+      COMMAND bash -c "PKG_CONFIG_PATH=${DPDK_LIB_DIR}/pkgconfig pkg-config --static --libs-only-l libdpdk"
+      OUTPUT_VARIABLE DPDK_PKGCONFIG_OUT RESULT_VARIABLE pkgconfig_ret)
+    if(pkgconfig_ret EQUAL "1")
+      message( FATAL_ERROR "Failed to run pkgconfig on DPDK (in ${DPDK_LIB_DIR}/pkgconfig). See error above.")
     endif()
-    include_directories(SYSTEM ${DPDK_INCLUDE_DIR})
+
+    string(STRIP ${DPDK_PKGCONFIG_OUT} DPDK_PKGCONFIG_OUT) # Remove newline from pkg-config output
+    set(LIBDPDK_LIBRARIES
+      "-Wl,--whole-archive -Wl,--as-needed -L${DPDK_LIB_DIR} ${DPDK_PKGCONFIG_OUT} -lpthread -Wl,--no-whole-archive")
+    set(LIBRARIES ${LIBRARIES} ${LIBDPDK_LIBRARIES})
+
+    link_directories(${DPDK_LIB_DIR})
+    include_directories(SYSTEM ${DPDK_INSTALL_DIR}/include)
   endif(WIN32)
 else()
   message(STATUS "DPDK not needed to build eRPC")

diff --git a/README.md b/README.md
@@ -16,13 +16,7 @@ Some highlights:
  * A port of [Raft](https://github.com/willemt/raft) as an example. Our 3-way
    replication latency is 5.3 microseconds with traditional UDP over Ethernet.
 
-## Requirements
- * Toolchain: A C++11 compiler and CMake 2.8+
- * See `scripts/packages/` for required software packages for your distro.
- * The latest `rdma_core`, preferably installed from source
- * For non-Mellanox DPDK-compatible NICs, a system-wide installation from DPDK
-   19.11.5 LTS sources (i.e., `sudo make install T=x86_64-native-linuxapp-gcc
-   DESTDIR=/usr`). Other DPDK versions are not supported.
+## System requirements
  * NICs: Fast (10 GbE+) NICs are needed for good performance. eRPC works best
    with Mellanox Ethernet and InfiniBand NICs. Any DPDK-capable NICs
    also work well.
@@ -37,7 +31,7 @@ Some highlights:
  * Build and run the test suite:
    `cmake . -DPERF=OFF -DTRANSPORT=dpdk; make -j; sudo ctest`.
    * `DPERF=OFF` enables debugging, which greatly reduces performance. Set
-     `DPERF=ON` for performance measurements.
+     `DPERF=ON` for good performance.
    * Here, `dpdk` should be replaced with `infiniband` for InfiniBand NICs.
    * A machine with two ports is needed to run the unit tests if DPDK is chosen.
      Run `scripts/run-tests-dpdk.sh` instead of `ctest`.
@@ -51,8 +45,7 @@ Some highlights:
 
 ## Supported bare-metal NICs:
  * Ethernet/UDP mode:
-   * DPDK-enabled NICs: Use `DTRANSPORT=dpdk`
-     * We have primarily tested Mellanox CX3--CX5 NICs.
+   * DPDK-enabled bare-metal NICs: Use `DTRANSPORT=dpdk`. We have primarily tested Mellanox CX3--CX5 NICs.
    * DPDK-enabled NICs on Microsoft Azure: Use `-DTRANSPORT=dpdk -DAZURE=on`
  * RDMA (InfiniBand/RoCE) NICs: Use `DTRANSPORT=infiniband`. Add `DROCE=on`
    if using RoCE.
@@ -77,32 +70,40 @@ Some highlights:
     * Re-start the VM. It should have a new interface called `eth1`, which eRPC
       will use for DPDK traffic.
 
-  * Prepare DPDK 19.11.5:
+  * Prepare DPDK 21.11
     * [rdma-core](https://github.com/linux-rdma/rdma-core) must be installed
-      from source. First, install its dependencies listed in rdma-core's README.
+      from source. We recommend the tag `stable-v40. First, install its
+      dependencies listed in rdma-core's README.
       Then, in the `rdma-core` directory:
        * `cmake .`
        * `sudo make install`
     * Install upstream pre-requisite libraries and modules:
        * `sudo apt install make cmake g++ gcc libnuma-dev libgflags-dev numactl`
        * `sudo modprobe ib_uverbs`
        * `sudo modprobe mlx4_ib`
-    * Download the [DPDK 19.11.5 tarball](https://core.dpdk.org/download/) and
+    * Download the [DPDK  tarball](https://core.dpdk.org/download/) and
       extract it. Other DPDK versions are not supported.
     * Edit `config/common_base` by changing `CONFIG_RTE_LIBRTE_MLX5_PMD` and
       `CONFIG_RTE_LIBRTE_MLX4_PMD` to `y` instead of `n`.
-    * Build and install DPDK: `sudo make install T=x86_64-native-linuxapp-gcc
-      DESTDIR=/usr`
-
-    * Create hugepages:
+    * Build and locally install DPDK: 
+```bash
+export RTE_SDK=<some dpdk directory>
+git clone --depth 1 --branch 'v21.11' https://github.com/DPDK/dpdk.git "${RTE_SDK}"
+cd "${RTE_SDK}"
+meson build -Dexamples='' -Denable_kmods=false -Dtests=false -Ddisable_drivers='raw/*,crypto/*,baseband/*,dma/*'
+cd build/
+DESTDIR="${RTE_SDK}/build/install" ninja install
 ```
+
+  *  Create hugepages:
+```bash
 sudo bash -c "echo 2048 > /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages"
 sudo mkdir /mnt/huge
 sudo mount -t hugetlbfs nodev /mnt/huge
 ```
 
   * Build eRPC's library and latency benchmark:
-```
+```bash
 cmake . -DTRANSPORT=dpdk -DAZURE=on
 make
 make latency

diff --git a/apps/latency/latency.cc b/apps/latency/latency.cc
@@ -1,7 +1,10 @@
 #include "util/latency.h"
+
 #include <gflags/gflags.h>
 #include <signal.h>
+
 #include <cstring>
+
 #include "../apps_common.h"
 #include "HdrHistogram_c/src/hdr_histogram.h"
 #include "rpc.h"
@@ -11,8 +14,8 @@
 static constexpr size_t kAppEvLoopMs = 1000;  // Duration of event loop
 static constexpr bool kAppVerbose = false;    // Print debug info on datapath
 static constexpr size_t kAppReqType = 1;      // eRPC request type
-static constexpr size_t kAppMinReqSize = 64;
-static constexpr size_t kAppMaxReqSize = 1024;
+static constexpr size_t kAppStartReqSize = 64;
+static constexpr size_t kAppEndReqSize = 1024;
 
 // Precision factor for latency measurement
 static constexpr double kAppLatFac = erpc::kIsAzure ? 1.0 : 10.0;
@@ -35,13 +38,13 @@ class ClientContext : public BasicAppContext {
 
  public:
   size_t start_tsc_;
-  size_t req_size_;  // Between kAppMinReqSize and kAppMaxReqSize
+  size_t req_size_;  // Between kAppStartReqSize and kAppEndReqSize
   erpc::MsgBuffer req_msgbuf_, resp_msgbuf_;
   hdr_histogram *latency_hist_;
   size_t latency_samples_ = 0;
   size_t latency_samples_prev_ = 0;
 
-  // If true, the client doubles its request size (up to kAppMaxReqSize) when
+  // If true, the client doubles its request size (up to kAppEndReqSize) when
   // issuing the next request, and resets this flag to false
   bool double_req_size_ = false;
 
@@ -58,7 +61,6 @@ void req_handler(erpc::ReqHandle *req_handle, void *_context) {
   auto *c = static_cast<ServerContext *>(_context);
   erpc::Rpc<erpc::CTransport>::resize_msg_buffer(&req_handle->pre_resp_msgbuf_,
                                                  FLAGS_resp_size);
-  // erpc::nano_sleep((c->fast_rand_.next_u32() % 5) * 1000 * 1000, 3.0);
   c->rpc_->enqueue_response(req_handle, &req_handle->pre_resp_msgbuf_);
 }
 
@@ -102,7 +104,7 @@ inline void send_req(ClientContext &c) {
   if (c.double_req_size_) {
     c.double_req_size_ = false;
     c.req_size_ *= 2;
-    if (c.req_size_ > kAppMaxReqSize) c.req_size_ = kAppMinReqSize;
+    if (c.req_size_ > kAppEndReqSize) c.req_size_ = kAppStartReqSize;
 
     c.rpc_->resize_msg_buffer(&c.req_msgbuf_, c.req_size_);
     c.rpc_->resize_msg_buffer(&c.resp_msgbuf_, FLAGS_resp_size);
@@ -149,9 +151,9 @@ void client_func(erpc::Nexus *nexus) {
 
   rpc.retry_connect_on_invalid_rpc_id_ = true;
   c.rpc_ = &rpc;
-  c.req_size_ = kAppMinReqSize;
+  c.req_size_ = kAppStartReqSize;
 
-  c.req_msgbuf_ = rpc.alloc_msg_buffer_or_die(kAppMaxReqSize);
+  c.req_msgbuf_ = rpc.alloc_msg_buffer_or_die(kAppEndReqSize);
   c.resp_msgbuf_ = rpc.alloc_msg_buffer_or_die(FLAGS_resp_size);
 
   connect_sessions(c);
@@ -205,10 +207,11 @@ void client_func(erpc::Nexus *nexus) {
 }
 
 int main(int argc, char **argv) {
-  printf("Latency: Welcome!");
   signal(SIGINT, ctrl_c_handler);
   gflags::ParseCommandLineFlags(&argc, &argv, true);
   erpc::rt_assert(FLAGS_numa_node <= 1, "Invalid NUMA node");
+  printf("Latency: Starting latency test. Response size = %zu bytes\n",
+         FLAGS_resp_size);
 
   erpc::Nexus nexus(erpc::get_uri_for_process(FLAGS_process_id),
                     FLAGS_numa_node, 0);

diff --git a/src/transport_impl/eth_common.h b/src/transport_impl/eth_common.h
@@ -10,13 +10,14 @@
 #endif
 
 #define ASIO_STANDLONE
-#include <asio/ts/internet.hpp>
-
 #include <assert.h>
 #include <stdint.h>
 #include <string.h>
+
+#include <asio/ts/internet.hpp>
 #include <sstream>
 #include <string>
+
 #include "common.h"
 
 namespace erpc {
@@ -180,7 +181,8 @@ static void gen_ipv4_header(ipv4_hdr_t* ipv4_hdr, uint32_t src_ip,
 
 // Compute IP header checksum (copied from DPDK testpmd)
 static uint16_t get_ipv4_checksum(const ipv4_hdr_t* ipv4_hdr) {
-  auto* ptr16 = reinterpret_cast<const uint16_t*>(ipv4_hdr);
+  auto* void_ptr16  = reinterpret_cast<const void*>(ipv4_hdr);
+  auto* ptr16 = reinterpret_cast<const uint16_t*>(void_ptr16);
   uint32_t ip_cksum = 0;
   ip_cksum += ptr16[0];
   ip_cksum += ptr16[1];