Drop libpciaccess and switch to DPDK PCI

This patch also drops support for automatically unbinding devices from the kernel - run scripts/setup.sh first. Our generic pci interface is now hidden behind include/spdk/pci.h and implemented in lib/util/pci.c. We no longer wrap the calls in nvme_impl.h or ioat_impl.h. The implementation now only uses DPDK and the libpciaccess dependency has been removed. If using a version of DPDK earlier than 16.07, enumerating devices by class code isn't available and only Intel SSDs will be discovered. DPDK 16.07 adds enumeration by class code and all NVMe devices will be correctly discovered. Change-Id: I0e8bac36b5ca57df604a2b310c47342c67dc9f3c Signed-off-by: Ben Walker <[email protected]>
gampel · Oct 4, 2016 · 6b1e4e7 · 6b1e4e7
1 parent f4140ad
commit 6b1e4e7
Show file tree

Hide file tree

Showing 65 changed files with 358 additions and 1,239 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -10,6 +10,11 @@ updated by simply removing the initialization of `request_mempool`.  Since the N
 library user no longer needs to know the size of the internal NVMe request
 structure to create the pool, the `spdk_nvme_request_size()` function was also removed.
 
+Libpciaccess has been removed as a dependency and DPDK PCI enumeration is
+used instead. Prior to DPDK 16.07 enumeration by class code was not supported,
+so for earlier DPDK versions only Intel SSDs will be discovered. Starting with
+DPDK 16.07 all devices will be discovered correctly by class code.
+
 v16.08: iSCSI target, NVMe over Fabrics maturity
 ------------------------------------------------
 

diff --git a/CONFIG b/CONFIG
@@ -56,11 +56,6 @@ CONFIG_NVME_IMPL?=nvme_impl.h
 # Defaults to depending on DPDK.
 CONFIG_IOAT_IMPL?=ioat_impl.h
 
-# This item indicates using libpciaccess library or not. If enabled with y,
-# libpciaccess library is used to map pci devices; else DPDK library is used to
-# map pci devices.
-CONFIG_PCIACCESS?=y
-
 # Build SPDK FIO plugin. Requires FIO_SOURCE_DIR set to a valid
 # fio source code directory.
 CONFIG_FIO_PLUGIN?=n

diff --git a/PORTING.md b/PORTING.md
@@ -8,10 +8,9 @@ operations (config cycles and mapping BARs), virtual to physical
 address translation and allocating per I/O data structures.
 
 SPDK includes a default implementation of the nvme_impl API based
-on the Data Plane Development Kit ([DPDK](dpdk.org)) and
-libpciaccess.  This DPDK implementation can be found in
-lib/nvme/nvme_impl.h.  DPDK is currently supported on Linux and
-FreeBSD only.
+on the Data Plane Development Kit ([DPDK](dpdk.org)).  This DPDK
+implementation can be found in lib/nvme/nvme_impl.h.  DPDK is
+currently supported on Linux and FreeBSD only.
 
 Users who want to use SPDK on other operating system, or in
 userspace driver frameworks other than DPDK, will need to implement

diff --git a/README.md b/README.md
@@ -38,20 +38,19 @@ To build SPDK, some dependencies must be installed.
 
 Fedora/CentOS:
 
-    sudo dnf install -y gcc gcc-c++ libpciaccess-devel CUnit-devel libaio-devel openssl-devel
+    sudo dnf install -y gcc gcc-c++ CUnit-devel libaio-devel openssl-devel
     # Additional dependencies for NVMe over Fabrics:
     sudo dnf install -y libibverbs-devel librdmacm-devel
 
 Ubuntu/Debian:
 
-    sudo apt-get install -y gcc g++ libpciaccess-dev make libcunit1-dev libaio-dev libssl-dev
+    sudo apt-get install -y gcc g++ make libcunit1-dev libaio-dev libssl-dev
     # Additional dependencies for NVMe over Fabrics:
     sudo apt-get install -y libibverbs-dev librdmacm-dev
 
 FreeBSD:
 
 - gcc
-- libpciaccess
 - gmake
 - cunit
 - openssl

diff --git a/app/iscsi_tgt/Makefile b/app/iscsi_tgt/Makefile
@@ -64,7 +64,7 @@ SPDK_LIBS = \
 	$(SPDK_ROOT_DIR)/lib/event/rpc/libspdk_app_rpc.a \
 
 LIBS += -Wl,--whole-archive $(SPDK_LIBS) -Wl,--no-whole-archive
-LIBS += -lcrypto $(PCIACCESS_LIB) $(DPDK_LIB)
+LIBS += -lcrypto $(DPDK_LIB)
 LIBS += $(BLOCKDEV_MODULES_LINKER_ARGS) \
 	$(COPY_MODULES_LINKER_ARGS)
 

diff --git a/app/nvmf_tgt/Makefile b/app/nvmf_tgt/Makefile
@@ -65,7 +65,7 @@ SPDK_LIBS = \
 LIBS += $(BLOCKDEV_MODULES_LINKER_ARGS) \
         $(COPY_MODULES_LINKER_ARGS)
 
-LIBS += $(SPDK_LIBS) $(PCIACCESS_LIB)
+LIBS += $(SPDK_LIBS)
 
 ifeq ($(CONFIG_RDMA),y)
 LIBS += -libverbs -lrdmacm

diff --git a/app/nvmf_tgt/conf.c b/app/nvmf_tgt/conf.c
@@ -341,11 +341,8 @@ probe_cb(void *cb_ctx, struct spdk_pci_device *dev, struct spdk_nvme_ctrlr_opts
 	    found_bus == ctx->bus &&
 	    found_dev == ctx->device &&
 	    found_func == ctx->function) {
-		if (!spdk_pci_device_has_non_uio_driver(dev)) {
-			ctx->found = true;
-			return true;
-		}
-		SPDK_ERRLOG("Requested device is still bound to the kernel. Unbind your NVMe devices first.\n");
+		ctx->found = true;
+		return true;
 	}
 
 	return false;

diff --git a/app/nvmf_tgt/nvmf_rpc.c b/app/nvmf_tgt/nvmf_rpc.c
@@ -33,7 +33,7 @@
 
 #include <stdlib.h>
 #include <stdio.h>
-#include <pciaccess.h>
+
 #include "nvmf_tgt.h"
 #include "nvmf/subsystem.h"
 #include "nvmf/transport.h"

diff --git a/doc/iscsi/getting_started.txt b/doc/iscsi/getting_started.txt
@@ -149,12 +149,6 @@ the kernel to avoid interrupts and context switching.
   BDF 0000:00:00.0 Nvme0
   BDF 0000:01:00.0 Nvme1
 
-  # Users can set the option "UnbindFromKernel Yes", SPDK will unbind all NVMe
-  #  devices from kernel driver and bind them to the uio_pci_generic driver.
-  #  The purpose is to prevent the case where NVMe driver is loaded while iscsi
-  #  is running.
-  UnbindFromKernel Yes
-
   # SPDK supports partitioning each nvme card into multiple LUNs
   #  through the NvmeLunsPerNs parameter. If NvmeLunsPerNs is specified,
   #  then the size of the nvme card is split up equally only if LunSizeinMB

diff --git a/etc/spdk/iscsi.conf.in b/etc/spdk/iscsi.conf.in
@@ -96,9 +96,6 @@
   BDF 0000:00:00.0 Nvme0
   BDF 0000:01:00.0 Nvme1
 
-  # If 'Yes', iscsi will automatically unbind the kernel NVMe driver from
-  # discovered devices and rebind it to the uio driver.
-  UnbindFromKernel Yes
   # The following two arguments allow the user to partition NVMe namespaces
   # into multiple LUNs
   NvmeLunsPerNs 1
@@ -126,13 +123,10 @@
 # Users may not want to use offload even it is available.
 # Users may use the whitelist to initialize specified devices, IDS
 #  uses BUS:DEVICE.FUNCTION to identify each Ioat channel.
-# Users can set the option "UnbindFromKernel Yes", iscsi will automatically
-#  unbind from the Ioat driver and bind the device to uio driver.
 [Ioat]
   Disable Yes
   Whitelist 00:04.0
   Whitelist 00:04.1
-  UnbindFromKernel Yes
 
 # Users must change this section to match the /dev/sdX devices to be
 #  exported as iSCSI LUNs. The devices are accessed using Linux AIO.

diff --git a/examples/ioat/perf/Makefile b/examples/ioat/perf/Makefile
@@ -45,7 +45,7 @@ SPDK_LIBS += $(SPDK_ROOT_DIR)/lib/ioat/libspdk_ioat.a \
 	     $(SPDK_ROOT_DIR)/lib/memory/libspdk_memory.a \
 	     $(SPDK_ROOT_DIR)/lib/log/libspdk_log.a \
 
-LIBS += $(SPDK_LIBS) $(PCIACCESS_LIB) $(DPDK_LIB)
+LIBS += $(SPDK_LIBS) $(DPDK_LIB)
 
 all: $(APP)
 

diff --git a/examples/ioat/perf/perf.c b/examples/ioat/perf/perf.c
@@ -154,11 +154,6 @@ probe_cb(void *cb_ctx, struct spdk_pci_device *pci_dev)
 	       spdk_pci_device_get_vendor_id(pci_dev), spdk_pci_device_get_device_id(pci_dev),
 	       spdk_pci_device_get_device_name(pci_dev));
 
-	if (spdk_pci_device_has_non_uio_driver(pci_dev)) {
-		printf("Device has non-uio kernel driver, skipping...\n");
-		return false;
-	}
-
 	return true;
 }
 
@@ -329,7 +324,7 @@ init(void)
 		return 1;
 	}
 
-	char *ealargs[] = {"perf", core_mask_conf, "-n 4", "--no-pci"};
+	char *ealargs[] = {"perf", core_mask_conf, "-n 4"};
 
 	if (rte_eal_init(sizeof(ealargs) / sizeof(ealargs[0]), ealargs) < 0) {
 		free(core_mask_conf);

diff --git a/examples/ioat/verify/Makefile b/examples/ioat/verify/Makefile
@@ -45,7 +45,7 @@ SPDK_LIBS += $(SPDK_ROOT_DIR)/lib/ioat/libspdk_ioat.a \
 	     $(SPDK_ROOT_DIR)/lib/memory/libspdk_memory.a \
 	     $(SPDK_ROOT_DIR)/lib/log/libspdk_log.a \
 
-LIBS += $(SPDK_LIBS) $(PCIACCESS_LIB) $(DPDK_LIB)
+LIBS += $(SPDK_LIBS) $(DPDK_LIB)
 
 all: $(APP)
 

diff --git a/examples/ioat/verify/verify.c b/examples/ioat/verify/verify.c
@@ -210,11 +210,6 @@ probe_cb(void *cb_ctx, struct spdk_pci_device *pci_dev)
 	       spdk_pci_device_get_vendor_id(pci_dev), spdk_pci_device_get_device_id(pci_dev),
 	       spdk_pci_device_get_device_name(pci_dev));
 
-	if (spdk_pci_device_has_non_uio_driver(pci_dev)) {
-		printf("Device has non-uio kernel driver, skipping...\n");
-		return false;
-	}
-
 	return true;
 }
 
@@ -395,7 +390,7 @@ init(void)
 		return 1;
 	}
 
-	char *ealargs[] = {"verify", core_mask_conf, "-n 4", "--no-pci"};
+	char *ealargs[] = {"verify", core_mask_conf, "-n 4"};
 	if (rte_eal_init(sizeof(ealargs) / sizeof(ealargs[0]), ealargs) < 0) {
 		free(core_mask_conf);
 		fprintf(stderr, "Could not init eal\n");

diff --git a/examples/nvme/arbitration/Makefile b/examples/nvme/arbitration/Makefile
@@ -45,7 +45,7 @@ SPDK_LIBS += $(SPDK_ROOT_DIR)/lib/nvme/libspdk_nvme.a \
 	     $(SPDK_ROOT_DIR)/lib/memory/libspdk_memory.a \
 	     $(SPDK_ROOT_DIR)/lib/log/libspdk_log.a \
 
-LIBS += $(SPDK_LIBS) $(PCIACCESS_LIB) $(DPDK_LIB)
+LIBS += $(SPDK_LIBS) $(DPDK_LIB)
 
 all : $(APP)
 

diff --git a/examples/nvme/arbitration/arbitration.c b/examples/nvme/arbitration/arbitration.c
@@ -869,17 +869,6 @@ register_workers(void)
 static bool
 probe_cb(void *cb_ctx, struct spdk_pci_device *dev, struct spdk_nvme_ctrlr_opts *opts)
 {
-	if (spdk_pci_device_has_non_uio_driver(dev)) {
-		fprintf(stderr, "non-uio kernel driver attached to NVMe\n");
-		fprintf(stderr, " controller at PCI address %04x:%02x:%02x.%02x\n",
-			spdk_pci_device_get_domain(dev),
-			spdk_pci_device_get_bus(dev),
-			spdk_pci_device_get_dev(dev),
-			spdk_pci_device_get_func(dev));
-		fprintf(stderr, " skipping...\n");
-		return false;
-	}
-
 	/* Update with user specified arbitration configuration */
 	opts->arb_mechanism = g_arbitration.arbitration_mechanism;
 

diff --git a/examples/nvme/fio_plugin/Makefile b/examples/nvme/fio_plugin/Makefile
@@ -49,7 +49,7 @@ SPDK_LIBS += $(SPDK_ROOT_DIR)/lib/nvme/libspdk_nvme.a \
 	     $(SPDK_ROOT_DIR)/lib/memory/libspdk_memory.a \
 	     $(SPDK_ROOT_DIR)/lib/log/libspdk_log.a \
 
-LIBS += $(SPDK_LIBS) $(PCIACCESS_LIB) $(DPDK_LIB)
+LIBS += $(SPDK_LIBS) $(DPDK_LIB)
 
 all : $(APP)
 

diff --git a/examples/nvme/fio_plugin/fio_plugin.c b/examples/nvme/fio_plugin/fio_plugin.c
@@ -36,7 +36,6 @@
 #include <unistd.h>
 #include <assert.h>
 #include <pthread.h>
-#include <pciaccess.h>
 
 #include "rte_config.h"
 #include "rte_mempool.h"
@@ -109,13 +108,6 @@ probe_cb(void *cb_ctx, struct spdk_pci_device *dev, struct spdk_nvme_ctrlr_opts
 			continue;
 		}
 		if (bus == found_bus && slot == found_slot && func == found_func) {
-			/* We do want to claim this device */
-			if (spdk_pci_device_has_non_uio_driver(dev)) {
-				fprintf(stderr,
-					"Requested to attach to %02x:%02x.%02x but that device is not unbound from the kernel\n",
-					bus, slot, func);
-				return false;
-			}
 			return true;
 		}
 	}

diff --git a/examples/nvme/hello_world/Makefile b/examples/nvme/hello_world/Makefile
@@ -45,7 +45,7 @@ SPDK_LIBS += $(SPDK_ROOT_DIR)/lib/nvme/libspdk_nvme.a \
 	     $(SPDK_ROOT_DIR)/lib/memory/libspdk_memory.a \
 	     $(SPDK_ROOT_DIR)/lib/log/libspdk_log.a \
 
-LIBS += $(SPDK_LIBS) $(PCIACCESS_LIB) $(DPDK_LIB)
+LIBS += $(SPDK_LIBS) $(DPDK_LIB)
 
 all : $(APP)
 

diff --git a/examples/nvme/hello_world/hello_world.c b/examples/nvme/hello_world/hello_world.c
@@ -240,22 +240,6 @@ hello_world(void)
 static bool
 probe_cb(void *cb_ctx, struct spdk_pci_device *dev, struct spdk_nvme_ctrlr_opts *opts)
 {
-	if (spdk_pci_device_has_non_uio_driver(dev)) {
-		/*
-		 * If an NVMe controller is found, but it is attached to a non-uio
-		 *  driver (i.e. the kernel NVMe driver), we will not try to attach
-		 *  to it.
-		 */
-		fprintf(stderr, "non-uio kernel driver attached to NVMe\n");
-		fprintf(stderr, " controller at PCI address %04x:%02x:%02x.%02x\n",
-			spdk_pci_device_get_domain(dev),
-			spdk_pci_device_get_bus(dev),
-			spdk_pci_device_get_dev(dev),
-			spdk_pci_device_get_func(dev));
-		fprintf(stderr, " skipping...\n");
-		return false;
-	}
-
 	printf("Attaching to %04x:%02x:%02x.%02x\n",
 	       spdk_pci_device_get_domain(dev),
 	       spdk_pci_device_get_bus(dev),

diff --git a/examples/nvme/identify/Makefile b/examples/nvme/identify/Makefile
@@ -45,7 +45,7 @@ SPDK_LIBS += $(SPDK_ROOT_DIR)/lib/nvme/libspdk_nvme.a \
 	     $(SPDK_ROOT_DIR)/lib/memory/libspdk_memory.a \
 	     $(SPDK_ROOT_DIR)/lib/log/libspdk_log.a \
 
-LIBS += $(SPDK_LIBS) $(PCIACCESS_LIB) $(DPDK_LIB)
+LIBS += $(SPDK_LIBS) $(DPDK_LIB)
 
 all : $(APP)
 

diff --git a/examples/nvme/identify/identify.c b/examples/nvme/identify/identify.c
@@ -872,17 +872,6 @@ parse_args(int argc, char **argv)
 static bool
 probe_cb(void *cb_ctx, struct spdk_pci_device *dev, struct spdk_nvme_ctrlr_opts *opts)
 {
-	if (spdk_pci_device_has_non_uio_driver(dev)) {
-		fprintf(stderr, "non-uio kernel driver attached to NVMe\n");
-		fprintf(stderr, " controller at PCI address %04x:%02x:%02x.%02x\n",
-			spdk_pci_device_get_domain(dev),
-			spdk_pci_device_get_bus(dev),
-			spdk_pci_device_get_dev(dev),
-			spdk_pci_device_get_func(dev));
-		fprintf(stderr, " skipping...\n");
-		return false;
-	}
-
 	return true;
 }
 

diff --git a/examples/nvme/nvme_manage/Makefile b/examples/nvme/nvme_manage/Makefile
@@ -45,7 +45,7 @@ SPDK_LIBS += $(SPDK_ROOT_DIR)/lib/nvme/libspdk_nvme.a \
 	     $(SPDK_ROOT_DIR)/lib/memory/libspdk_memory.a \
 	     $(SPDK_ROOT_DIR)/lib/log/libspdk_log.a \
 
-LIBS += $(SPDK_LIBS) $(PCIACCESS_LIB) $(DPDK_LIB)
+LIBS += $(SPDK_LIBS) $(DPDK_LIB)
 
 all : $(APP)
 

diff --git a/examples/nvme/nvme_manage/nvme_manage.c b/examples/nvme/nvme_manage/nvme_manage.c
@@ -101,17 +101,6 @@ cmp_devs(const void *ap, const void *bp)
 static bool
 probe_cb(void *cb_ctx, struct spdk_pci_device *dev, struct spdk_nvme_ctrlr_opts *opts)
 {
-	if (spdk_pci_device_has_non_uio_driver(dev)) {
-		fprintf(stderr, "non-uio kernel driver attached to NVMe\n");
-		fprintf(stderr, " controller at PCI address %04x:%02x:%02x.%02x\n",
-			spdk_pci_device_get_domain(dev),
-			spdk_pci_device_get_bus(dev),
-			spdk_pci_device_get_dev(dev),
-			spdk_pci_device_get_func(dev));
-		fprintf(stderr, " skipping...\n");
-		return false;
-	}
-
 	return true;
 }
 

diff --git a/examples/nvme/perf/Makefile b/examples/nvme/perf/Makefile
@@ -45,7 +45,7 @@ SPDK_LIBS += $(SPDK_ROOT_DIR)/lib/nvme/libspdk_nvme.a \
 	     $(SPDK_ROOT_DIR)/lib/memory/libspdk_memory.a \
 	     $(SPDK_ROOT_DIR)/lib/log/libspdk_log.a \
 
-LIBS += $(SPDK_LIBS) $(PCIACCESS_LIB) $(DPDK_LIB)
+LIBS += $(SPDK_LIBS) $(DPDK_LIB)
 
 ifeq ($(OS),Linux)
 LIBS += -laio

diff --git a/examples/nvme/perf/perf.c b/examples/nvme/perf/perf.c
@@ -966,17 +966,6 @@ unregister_workers(void)
 static bool
 probe_cb(void *cb_ctx, struct spdk_pci_device *dev, struct spdk_nvme_ctrlr_opts *opts)
 {
-	if (spdk_pci_device_has_non_uio_driver(dev)) {
-		fprintf(stderr, "non-uio kernel driver attached to NVMe\n");
-		fprintf(stderr, " controller at PCI address %04x:%02x:%02x.%02x\n",
-			spdk_pci_device_get_domain(dev),
-			spdk_pci_device_get_bus(dev),
-			spdk_pci_device_get_dev(dev),
-			spdk_pci_device_get_func(dev));
-		fprintf(stderr, " skipping...\n");
-		return false;
-	}
-
 	printf("Attaching to %04x:%02x:%02x.%02x\n",
 	       spdk_pci_device_get_domain(dev),
 	       spdk_pci_device_get_bus(dev),

diff --git a/examples/nvme/reserve/Makefile b/examples/nvme/reserve/Makefile
@@ -45,7 +45,7 @@ SPDK_LIBS += $(SPDK_ROOT_DIR)/lib/nvme/libspdk_nvme.a \
 	     $(SPDK_ROOT_DIR)/lib/memory/libspdk_memory.a \
 	     $(SPDK_ROOT_DIR)/lib/log/libspdk_log.a \
 
-LIBS += $(SPDK_LIBS) $(PCIACCESS_LIB) $(DPDK_LIB)
+LIBS += $(SPDK_LIBS) $(DPDK_LIB)
 
 all : $(APP)
 

diff --git a/examples/nvme/reserve/reservation.c b/examples/nvme/reserve/reservation.c
@@ -364,17 +364,6 @@ reserve_controller(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair,
 static bool
 probe_cb(void *cb_ctx, struct spdk_pci_device *dev, struct spdk_nvme_ctrlr_opts *opts)
 {
-	if (spdk_pci_device_has_non_uio_driver(dev)) {
-		fprintf(stderr, "non-uio kernel driver attached to NVMe\n");
-		fprintf(stderr, " controller at PCI address %04x:%02x:%02x.%02x\n",
-			spdk_pci_device_get_domain(dev),
-			spdk_pci_device_get_bus(dev),
-			spdk_pci_device_get_dev(dev),
-			spdk_pci_device_get_func(dev));
-		fprintf(stderr, " skipping...\n");
-		return false;
-	}
-
 	return true;
 }