Merge pull request ceph#34789 from trociny/wip-rbd-nbd-quiesce

rbd-nbd: add quiesce/unquiesce hooks Reviewed-by: Jason Dillaman <[email protected]>
JiYou · May 15, 2020 · 7f1df0d · 7f1df0d
2 parents 04a430d + 2f70046
commit 7f1df0d
Show file tree

Hide file tree

Showing 15 changed files with 329 additions and 16 deletions.
diff --git a/ceph.spec.in b/ceph.spec.in
@@ -1284,6 +1284,7 @@ install -m 0644 -D src/logrotate.conf %{buildroot}%{_sysconfdir}/logrotate.d/cep
 chmod 0644 %{buildroot}%{_docdir}/ceph/sample.ceph.conf
 install -m 0644 -D COPYING %{buildroot}%{_docdir}/ceph/COPYING
 install -m 0644 -D etc/sysctl/90-ceph-osd.conf %{buildroot}%{_sysctldir}/90-ceph-osd.conf
+install -m 0755 -D src/tools/rbd_nbd/rbd-nbd_quiesce %{buildroot}%{_libexecdir}/rbd-nbd/rbd-nbd_quiesce
 
 install -m 0755 src/cephadm/cephadm %{buildroot}%{_sbindir}/cephadm
 mkdir -p %{buildroot}%{_sharedstatedir}/cephadm
@@ -1926,6 +1927,8 @@ fi
 %files -n rbd-nbd
 %{_bindir}/rbd-nbd
 %{_mandir}/man8/rbd-nbd.8*
+%dir %{_libexecdir}/rbd-nbd
+%{_libexecdir}/rbd-nbd/rbd-nbd_quiesce
 
 %files radosgw
 %{_bindir}/ceph-diff-sorted

diff --git a/debian/rbd-nbd.install b/debian/rbd-nbd.install
@@ -1,2 +1,3 @@
 usr/bin/rbd-nbd
 usr/share/man/man8/rbd-nbd.8
+usr/lib/rbd-nbd/rbd-nbd_quiesce
diff --git a/debian/rules b/debian/rules
@@ -63,6 +63,7 @@ override_dh_auto_install:
 	install -D -m 644 etc/sysctl/90-ceph-osd.conf $(DESTDIR)/etc/sysctl.d/30-ceph-osd.conf
 	install -D -m 600 sudoers.d/ceph-osd-smartctl $(DESTDIR)/etc/sudoers.d/ceph-osd-smartctl
 	install -D -m 600 sudoers.d/cephadm $(DESTDIR)/etc/sudoers.d/cephadm
+	install -D -m 755 src/tools/rbd_nbd/rbd-nbd_quiesce $(DESTDIR)/usr/lib/rbd-nbd/rbd-nbd_quiesce
 
 	install -m 755 src/cephadm/cephadm $(DESTDIR)/usr/sbin/cephadm
 

diff --git a/qa/suites/rbd/nbd/supported-random-distro$ b/qa/suites/rbd/nbd/supported-random-distro$
@@ -0,0 +1 @@
+.qa/distros/supported-random-distro$
diff --git a/qa/suites/rbd/nbd/workloads/rbd_fsx_nbd.yaml b/qa/suites/rbd/nbd/workloads/rbd_fsx_nbd.yaml
@@ -1,4 +1,3 @@
-os_type: ubuntu
 overrides:
   install:
     ceph:

diff --git a/qa/suites/rbd/nbd/workloads/rbd_nbd.yaml b/qa/suites/rbd/nbd/workloads/rbd_nbd.yaml
@@ -1,4 +1,3 @@
-os_type: ubuntu
 overrides:
   install:
     ceph:

diff --git a/qa/workunits/rbd/rbd-nbd.sh b/qa/workunits/rbd/rbd-nbd.sh
@@ -63,6 +63,9 @@ function cleanup()
     local ns s
 
     set +e
+
+    mount | fgrep ${TEMPDIR}/mnt && umount ${TEMPDIR}/mnt
+
     rm -Rf ${TEMPDIR}
     if [ -n "${DEV}" ]
     then
@@ -238,4 +241,66 @@ for i in `seq 10`; do
 done
 rbd-nbd list-mapped | expect_false grep "^${PID} *${POOL} *${IMAGE}"
 
+# quiesce test
+QUIESCE_HOOK=${TEMPDIR}/quiesce.sh
+DEV=`_sudo rbd-nbd map --quiesce --quiesce-hook ${QUIESCE_HOOK} ${POOL}/${IMAGE}`
+
+# test it does not fail if the hook does not exists
+test ! -e ${QUIESCE_HOOK}
+rbd snap create ${POOL}/${IMAGE}@quiesce1
+_sudo dd if=${DATA} of=${DEV} bs=1M count=1 oflag=direct
+
+# test the hook is executed
+touch ${QUIESCE_HOOK}
+chmod +x ${QUIESCE_HOOK}
+cat > ${QUIESCE_HOOK} <<EOF
+#/bin/sh
+echo "test the hook is executed" >&2
+echo \$1 > ${TEMPDIR}/\$2
+EOF
+rbd snap create ${POOL}/${IMAGE}@quiesce2
+_sudo dd if=${DATA} of=${DEV} bs=1M count=1 oflag=direct
+test "$(cat ${TEMPDIR}/quiesce)" = ${DEV}
+test "$(cat ${TEMPDIR}/unquiesce)" = ${DEV}
+
+# test it does not fail if the hook fails
+touch ${QUIESCE_HOOK}
+chmod +x ${QUIESCE_HOOK}
+cat > ${QUIESCE_HOOK} <<EOF
+#/bin/sh
+echo "test it does not fail if the hook fails" >&2
+exit 1
+EOF
+rbd snap create ${POOL}/${IMAGE}@quiesce3
+_sudo dd if=${DATA} of=${DEV} bs=1M count=1 oflag=direct
+
+# test the hook is slow
+cat > ${QUIESCE_HOOK} <<EOF
+#/bin/sh
+echo "test the hook is slow" >&2
+sleep 7
+EOF
+rbd snap create ${POOL}/${IMAGE}@quiesce4
+_sudo dd if=${DATA} of=${DEV} bs=1M count=1 oflag=direct
+
+# test rbd-nbd_quiesce hook that comes with distribution
+unmap_device ${DEV}
+LOG_FILE=${TEMPDIR}/rbd-nbd.log
+if [ -n "${CEPH_SRC}" ]; then
+    QUIESCE_HOOK=${CEPH_SRC}/tools/rbd_nbd/rbd-nbd_quiesce
+    DEV=`_sudo rbd-nbd map --quiesce --quiesce-hook ${QUIESCE_HOOK} \
+               ${POOL}/${IMAGE} --log-file=${LOG_FILE}`
+else
+    DEV=`_sudo rbd-nbd map --quiesce ${POOL}/${IMAGE} --log-file=${LOG_FILE}`
+fi
+_sudo mkfs ${DEV}
+mkdir ${TEMPDIR}/mnt
+_sudo mount ${DEV} ${TEMPDIR}/mnt
+rbd snap create ${POOL}/${IMAGE}@quiesce5
+_sudo dd if=${DATA} of=${TEMPDIR}/mnt/test bs=1M count=1 oflag=direct
+_sudo umount ${TEMPDIR}/mnt
+unmap_device ${DEV}
+cat ${LOG_FILE}
+expect_false grep 'quiesce failed' ${LOG_FILE}
+
 echo OK
diff --git a/src/include/config-h.in.cmake b/src/include/config-h.in.cmake
@@ -351,4 +351,7 @@
 /* Shared library extension, such as .so, .dll or .dylib */
 #cmakedefine CMAKE_SHARED_LIBRARY_SUFFIX "@CMAKE_SHARED_LIBRARY_SUFFIX@"
 
+/* libexec directory path */
+#cmakedefine CMAKE_INSTALL_LIBEXECDIR "@CMAKE_INSTALL_LIBEXECDIR@"
+
 #endif /* CONFIG_H */
diff --git a/src/test/cli/rbd/help.t b/src/test/cli/rbd/help.t
@@ -582,8 +582,8 @@
   rbd help device map
   usage: rbd device map [--device-type <device-type>] [--pool <pool>] 
                         [--namespace <namespace>] [--image <image>] 
-                        [--snap <snap>] [--read-only] [--exclusive] 
-                        [--options <options>] 
+                        [--snap <snap>] [--read-only] [--exclusive] [--quiesce] 
+                        [--quiesce-hook <quiesce-hook>] [--options <options>] 
                         <image-or-snap-spec> 
 
   Map an image to a block device.
@@ -602,6 +602,8 @@
     --snap arg               snapshot name
     --read-only              map read-only
     --exclusive              disable automatic exclusive lock transitions
+    --quiesce                use quiesce callbacks
+    --quiesce-hook arg       quiesce hook path
     -o [ --options ] arg     device specific options
 
   rbd help device unmap

diff --git a/src/tools/rbd/action/Device.cc b/src/tools/rbd/action/Device.cc
@@ -136,7 +136,9 @@ void get_map_arguments(po::options_description *positional,
                                      at::ARGUMENT_MODIFIER_NONE);
   options->add_options()
     ("read-only", po::bool_switch(), "map read-only")
-    ("exclusive", po::bool_switch(), "disable automatic exclusive lock transitions");
+    ("exclusive", po::bool_switch(), "disable automatic exclusive lock transitions")
+    ("quiesce", po::bool_switch(), "use quiesce callbacks")
+    ("quiesce-hook", po::value<std::string>(), "quiesce hook path");
   add_device_specific_options(options);
 }
 

diff --git a/src/tools/rbd/action/Ggate.cc b/src/tools/rbd/action/Ggate.cc
@@ -130,6 +130,10 @@ int execute_map(const po::variables_map &vm,
   }
   args.push_back(img);
 
+  if (vm["quiesce"].as<bool>()) {
+    std::cerr << "rbd: warning: quiesce is not supported" << std::endl;
+  }
+
   if (vm["read-only"].as<bool>()) {
     args.push_back("--read-only");
   }
@@ -138,6 +142,10 @@ int execute_map(const po::variables_map &vm,
     args.push_back("--exclusive");
   }
 
+  if (vm.count("quiesce-hook")) {
+    std::cerr << "rbd: warning: quiesce-hook is not supported" << std::endl;
+  }
+
   if (vm.count("options")) {
     r = parse_options(vm["options"].as<std::vector<std::string>>(), &args);
     if (r < 0) {

diff --git a/src/tools/rbd/action/Kernel.cc b/src/tools/rbd/action/Kernel.cc
@@ -434,13 +434,20 @@ int execute_map(const po::variables_map &vm,
     return r;
   }
 
+  if (vm["quiesce"].as<bool>()) {
+    std::cerr << "rbd: warning: quiesce is not supported" << std::endl;
+  }
   if (vm["read-only"].as<bool>()) {
     put_map_option("rw", "ro");
   }
   if (vm["exclusive"].as<bool>()) {
     put_map_option("exclusive", "exclusive");
   }
 
+  if (vm.count("quiesce-hook")) {
+    std::cerr << "rbd: warning: quiesce-hook is not supported" << std::endl;
+  }
+
   // parse default options first so they can be overwritten by cli options
   r = parse_map_options(
       g_conf().get_val<std::string>("rbd_default_map_options"));

diff --git a/src/tools/rbd/action/Nbd.cc b/src/tools/rbd/action/Nbd.cc
@@ -134,6 +134,10 @@ int execute_map(const po::variables_map &vm,
   }
   args.push_back(img);
 
+  if (vm["quiesce"].as<bool>()) {
+    args.push_back("--quiesce");
+  }
+
   if (vm["read-only"].as<bool>()) {
     args.push_back("--read-only");
   }
@@ -142,6 +146,11 @@ int execute_map(const po::variables_map &vm,
     args.push_back("--exclusive");
   }
 
+  if (vm.count("quiesce-hook")) {
+    args.push_back("--quiesce-hook");
+    args.push_back(vm["quiesce-hook"].as<std::string>());
+  }
+
   if (vm.count("options")) {
     r = parse_options(vm["options"].as<std::vector<std::string>>(), &args);
     if (r < 0) {