test/iscsi: utilize network namespaces in iSCSI tests

Network namespaces are used to assure that kernel
is not routing packets within host stack,
but they go through veth interfaces.

This patch serves as a base for future VPP test changes,
where namespaces are used as well.

Change-Id: Ic7b82b0a0837bca2e16774fde244348a691fe056
Signed-off-by: Tomasz Zawadzki <[email protected]>
Reviewed-on: https://review.gerrithub.io/405641
Reviewed-by: Jim Harris <[email protected]>
Reviewed-by: Daniel Verkamp <[email protected]>
Tested-by: SPDK Automated Test System <[email protected]>

test/common/autotest_common.sh

@@ -236,13 +236,19 @@ function waitforlisten() {
 		if ! kill -s 0 $1; then
 			exit 1
 		fi
+
+		namespace=$(ip netns identify $1)
+		if [ -n "$namespace" ]; then
+			ns_cmd="ip netns exec $namespace"
+		fi
+
 		if hash ss; then
-			if ss -lx | grep -q $rpc_addr; then
+			if $ns_cmd ss -lx | grep -q $rpc_addr; then
 				ret=0
 			fi
 		else
 			# if system doesn't have ss, just assume it has netstat
-			if netstat -an -x | grep -iw LISTENING | grep -q $rpc_addr; then
+			if $ns_cmd netstat -an -x | grep -iw LISTENING | grep -q $rpc_addr; then
 				ret=0
 			fi
 		fi
@@ -315,20 +321,29 @@ function start_iscsi_service() {
 
 function rbd_setup() {
 	# $1 = monitor ip address
+	# $2 = name of the namespace
 	if [ -z "$1" ]; then
 		echo "No monitor IP address provided for ceph"
 		exit 1
 	fi
+	if [ -n "$2" ]; then
+		if ip netns list | grep "$2"; then
+			NS_CMD="ip netns exec $2"
+		else
+			echo "No namespace $2 exists"
+			exit 1
+		fi
+	fi
 
 	if hash ceph; then
 		export PG_NUM=128
 		export RBD_POOL=rbd
 		export RBD_NAME=foo
-		$rootdir/scripts/ceph/start.sh $1
+		$NS_CMD $rootdir/scripts/ceph/start.sh $1
 
-		ceph osd pool create $RBD_POOL $PG_NUM || true
-		rbd pool init $RBD_POOL || true
-		rbd create $RBD_NAME --size 1000
+		$NS_CMD ceph osd pool create $RBD_POOL $PG_NUM || true
+		$NS_CMD rbd pool init $RBD_POOL || true
+		$NS_CMD rbd create $RBD_NAME --size 1000
 	fi
 }
 

test/iscsi_tgt/common.sh

@@ -1,34 +1,43 @@
 # Network configuration
 TARGET_INTERFACE="spdk_tgt_int"
 INITIATOR_INTERFACE="spdk_init_int"
+TARGET_NAMESPACE="spdk_iscsi_ns"
+TARGET_NS_CMD="ip netns exec $TARGET_NAMESPACE"
 
 # iSCSI target configuration
 TARGET_IP=10.0.0.1
 INITIATOR_IP=10.0.0.2
 ISCSI_PORT=3260
-NETMASK=$INITIATOR_IP/30
+NETMASK=$INITIATOR_IP/32
 INITIATOR_TAG=2
 INITIATOR_NAME=ANY
 PORTAL_TAG=1
-ISCSI_APP="./app/iscsi_tgt/iscsi_tgt -i 0"
+ISCSI_APP="$TARGET_NS_CMD ./app/iscsi_tgt/iscsi_tgt -i 0"
 ISCSI_TEST_CORE_MASK=0xFF
 
 function create_veth_interfaces() {
+	ip netns del $TARGET_NAMESPACE || true
 	ip link delete $INITIATOR_INTERFACE || true
 
 	# Create veth (Virtual ethernet) interface pair
 	ip link add $INITIATOR_INTERFACE type veth peer name $TARGET_INTERFACE
 	ip addr add $INITIATOR_IP/24 dev $INITIATOR_INTERFACE
 	ip link set $INITIATOR_INTERFACE up
 
-	ip addr add $TARGET_IP/24 dev $TARGET_INTERFACE
-	ip link set $TARGET_INTERFACE up
+	# Create and add interface for target to network namespace
+	ip netns add $TARGET_NAMESPACE
+	ip link set $TARGET_INTERFACE netns $TARGET_NAMESPACE
+
+	$TARGET_NS_CMD ip link set lo up
+	$TARGET_NS_CMD ip addr add $TARGET_IP/24 dev $TARGET_INTERFACE
+	$TARGET_NS_CMD ip link set $TARGET_INTERFACE up
 
 	trap "cleanup_veth_interfaces; exit 1" SIGINT SIGTERM EXIT
 }
 
 function cleanup_veth_interfaces() {
-	# Cleanup veth interfaces
+	# Cleanup veth interfaces and network namespace
 	# Note: removing one veth, removes the pair
 	ip link delete $INITIATOR_INTERFACE
+	ip netns del $TARGET_NAMESPACE
 }

test/iscsi_tgt/initiator/initiator.sh

@@ -16,7 +16,7 @@ timing_enter start_iscsi_tgt
 
 # Start the iSCSI target without using stub
 # Reason: Two SPDK processes will be started
-$rootdir/app/iscsi_tgt/iscsi_tgt -c $testdir/iscsi.conf -m 0x2 -p 1 -s 512 &
+$ISCSI_APP -c $testdir/iscsi.conf -m 0x2 -p 1 -s 512 &
 pid=$!
 echo "iSCSI target launched. pid: $pid"
 trap "killprocess $pid;exit 1" SIGINT SIGTERM EXIT

test/iscsi_tgt/ip_migration/ip_migration.sh

@@ -8,6 +8,9 @@ source $rootdir/test/iscsi_tgt/common.sh
 rpc_py="python $rootdir/scripts/rpc.py"
 fio_py="python $rootdir/scripts/fio.py"
 
+# Namespaces are NOT used here on purpose. This test requires changes to detect
+# ifc_index for interface that was put into namespace. Needed for add_ip_address.
+ISCSI_APP="$rootdir/app/iscsi_tgt/iscsi_tgt"
 NETMASK=127.0.0.0/24
 MIGRATION_ADDRESS=127.0.0.2
 

test/iscsi_tgt/multiconnection/multiconnection.sh

@@ -48,7 +48,7 @@ $rootdir/scripts/gen_nvme.sh >> $testdir/iscsi.conf
 
 timing_enter start_iscsi_tgt
 # Start the iSCSI target without using stub.
-$rootdir/app/iscsi_tgt/iscsi_tgt -c $testdir/iscsi.conf &
+$ISCSI_APP -c $testdir/iscsi.conf &
 iscsipid=$!
 echo "iSCSI target launched. pid: $iscsipid"
 trap "remove_backends; iscsicleanup; killprocess $iscsipid; exit 1" SIGINT SIGTERM EXIT

test/iscsi_tgt/nvme_remote/fio_remote_nvme.sh

@@ -20,6 +20,11 @@ fio_py="python $rootdir/scripts/fio.py"
 
 NVMF_PORT=4420
 
+# Namespaces are NOT used here on purpose. Rxe_cfg utilility used for NVMf tests do not support namespaces.
+TARGET_IP=127.0.0.1
+INITIATOR_IP=127.0.0.1
+NETMASK=$INITIATOR_IP/32
+
 function run_nvme_remote() {
 	echo "now use $1 method to run iscsi tgt."
 	cp $testdir/iscsi.conf $testdir/iscsi.conf.tmp
@@ -30,7 +35,8 @@ function run_nvme_remote() {
 	fi
 	# Start the iSCSI target without using stub
 	iscsi_rpc_addr="/var/tmp/spdk-iscsi.sock"
-	$rootdir/app/iscsi_tgt/iscsi_tgt -r "$iscsi_rpc_addr" -c $testdir/iscsi.conf.tmp -m 0x1 -p 0 -s 512 &
+	ISCSI_APP="$rootdir/app/iscsi_tgt/iscsi_tgt"
+	$ISCSI_APP -r "$iscsi_rpc_addr" -c $testdir/iscsi.conf.tmp -m 0x1 -p 0 -s 512 &
 	iscsipid=$!
 	echo "iSCSI target launched. pid: $iscsipid"
 	trap "killprocess $iscsipid; killprocess $nvmfpid; exit 1" SIGINT SIGTERM EXIT
@@ -56,7 +62,8 @@ function run_nvme_remote() {
 timing_enter nvme_remote
 
 # Start the NVMf target
-$rootdir/app/nvmf_tgt/nvmf_tgt -c $rootdir/test/nvmf/nvmf.conf -m 0x2 -p 1 -s 512 &
+NVMF_APP="$rootdir/app/nvmf_tgt/nvmf_tgt"
+$NVMF_APP -c $rootdir/test/nvmf/nvmf.conf -m 0x2 -p 1 -s 512 &
 nvmfpid=$!
 echo "NVMf target launched. pid: $nvmfpid"
 trap "killprocess $nvmfpid; exit 1" SIGINT SIGTERM EXIT

test/iscsi_tgt/rbd/rbd.sh

@@ -11,7 +11,7 @@ if ! hash ceph; then
 fi
 
 timing_enter rbd_setup
-rbd_setup $TARGET_IP
+rbd_setup $TARGET_IP $TARGET_NAMESPACE
 trap "rbd_cleanup; exit 1" SIGINT SIGTERM EXIT
 timing_exit rbd_setup
 

test/iscsi_tgt/rpc_config/rpc_config.py

@@ -10,12 +10,14 @@
 import random
 from subprocess import check_call, call, check_output, Popen, PIPE, CalledProcessError
 
-if (len(sys.argv) == 6):
+if (len(sys.argv) == 7):
     target_ip = sys.argv[2]
     initiator_ip = sys.argv[3]
     port = sys.argv[4]
     netmask = sys.argv[5]
+    namespace = sys.argv[6]
 
+ns_cmd = 'ip netns exec ' + namespace
 other_ip = '127.0.0.6'
 initiator_name = 'ANY'
 portal_tag = '1'
@@ -402,7 +404,8 @@ def verify_get_interfaces(rpc_py):
     nics = json.loads(rpc.get_interfaces())
     nics_names = set(x["name"].encode('ascii', 'ignore') for x in nics)
     # parse ip link show to verify the get_interfaces result
-    ifcfg_nics = set(re.findall("\S+:\s(\S+?)(?:@\S+){0,1}:\s<.*", check_output(["ip", "link", "show"])))
+    ip_show = ns_cmd + " ip link show"
+    ifcfg_nics = set(re.findall("\S+:\s(\S+?)(?:@\S+){0,1}:\s<.*", check_output(ip_show.split())))
     verify(nics_names == ifcfg_nics, 1, "get_interfaces returned {}".format(nics))
     print "verify_get_interfaces passed."
 
@@ -422,11 +425,12 @@ def verify_add_delete_ip_address(rpc_py):
     # add ip on up to first 2 nics
     for x in nics[:2]:
         faked_ip = "123.123.{}.{}".format(random.randint(1, 254), random.randint(1, 254))
+        ping_cmd = ns_cmd + " ping -c 1 -W 1 " + faked_ip
         rpc.add_ip_address(x["ifc_index"], faked_ip)
         verify(faked_ip in help_get_interface_ip_list(rpc_py, x["name"]), 1,
                "add ip {} to nic {} failed.".format(faked_ip, x["name"]))
         try:
-            check_call(["ping", "-c", "1", "-W", "1", faked_ip])
+            check_call(ping_cmd.split())
         except:
             verify(False, 1,
                    "ping ip {} for {} was failed(adding was successful)".format
@@ -437,7 +441,7 @@ def verify_add_delete_ip_address(rpc_py):
                (faked_ip, x["name"]))
         # ping should be failed and throw an CalledProcessError exception
         try:
-            check_call(["ping", "-c", "1", "-W", "1", faked_ip])
+            check_call(ping_cmd.split())
         except CalledProcessError as _:
             pass
         except Exception as e:
@@ -486,10 +490,7 @@ def verify_add_nvme_bdev_rpc_methods(rpc_py):
         verify_initiator_groups_rpc_methods(rpc_py, rpc_param)
         verify_target_nodes_rpc_methods(rpc_py, rpc_param)
         verify_scsi_devices_rpc_methods(rpc_py)
-        # This test is removed due to kernel routing packets in host stack
-        # when handling connection between interfaces on same host.
-        # It is enabled back in next patch in series adding namespaces.
-        # verify_iscsi_connection_rpc_methods(rpc_py)
+        verify_iscsi_connection_rpc_methods(rpc_py)
         verify_add_nvme_bdev_rpc_methods(rpc_py)
     except RpcException as e:
         print "{}. Exiting with status {}".format(e.message, e.retval)

test/iscsi_tgt/rpc_config/rpc_config.sh

@@ -26,7 +26,7 @@ echo "iscsi_tgt is listening. Running tests..."
 
 timing_exit start_iscsi_tgt
 
-$rpc_config_py $rpc_py $TARGET_IP $INITIATOR_IP $ISCSI_PORT $NETMASK
+$rpc_config_py $rpc_py $TARGET_IP $INITIATOR_IP $ISCSI_PORT $NETMASK $TARGET_NAMESPACE
 
 $rpc_py get_bdevs