Merge remote-tracking branch 'gh/hammer'

Conflicts:
	src/gmock

debian/control

@@ -53,7 +53,8 @@ Build-Depends: autoconf,
                xfslibs-dev,
                xfsprogs,
                xmlstarlet,
-               yasm [amd64]
+               yasm [amd64],
+               zlib1g-dev
 Standards-Version: 3.9.3
 
 Package: ceph

doc/man/8/rbd.rst

@@ -162,7 +162,8 @@ Commands
 :command:`clone` [*parent-snapname*] [*image-name*]
   Will create a clone (copy-on-write child) of the parent snapshot.
   Object order will be identical to that of the parent image unless
-  specified. Size will be the same as the parent snapshot.
+  specified. Size will be the same as the parent snapshot. The --stripe-unit
+  and --stripe-count arguments are optional, but must be used together.
 
   The parent snapshot must be protected (see `rbd snap protect`).
   This requires image format 2.
@@ -198,6 +199,9 @@ Commands
   if possible.  For import from stdin, the sparsification unit is
   the data block size of the destination image (1 << order).
 
+  The --stripe-unit and --stripe-count arguments are optional, but must be
+  used together.
+
 :command:`export-diff` [*image-name*] [*dest-path*] [--from-snap *snapname*]
   Exports an incremental diff for an image to dest path (use - for stdout).  If
   an initial snapshot is specified, only changes since that snapshot are included; otherwise,

doc/start/quick-start-preflight.rst

@@ -63,7 +63,7 @@ following steps:
    Paste the following example code. Replace ``{ceph-release}`` with
    the recent major release of Ceph (e.g., ``firefly``). Replace ``{distro}``
    with your Linux distribution (e.g., ``el6`` for CentOS 6, 
-   ``el7`` for CentOS 7, ``rhel6.5`` for
+   ``el7`` for CentOS 7, ``rhel6`` for
    Red Hat 6.5, ``rhel7`` for Red Hat 7, and ``fc19`` or ``fc20`` for Fedora 19
    or Fedora 20. Finally, save the contents to the 
    ``/etc/yum.repos.d/ceph.repo`` file. ::

install-deps.sh

@@ -38,14 +38,16 @@ Ubuntu|Debian|Devuan)
         packages=$(dpkg-checkbuilddeps --admindir=$DIR debian/control 2>&1 | \
             perl -p -e 's/.*Unmet build dependencies: *//;' \
             -e 's/build-essential:native/build-essential/;' \
+            -e 's/\|//g;' \
             -e 's/\(.*?\)//g;' \
             -e 's/ +/\n/g;' | sort)
         case $(lsb_release -sc) in
             squeeze|wheezy)
                 packages=$(echo $packages | perl -pe 's/[-\w]*babeltrace[-\w]*//g')
                 ;;
         esac
-        $SUDO apt-get install -y $packages
+        packages=$(echo $packages) # change newlines into spaces
+        $SUDO bash -c "DEBIAN_FRONTEND=noninteractive apt-get install -y $packages"
         ;;
 CentOS|Fedora|SUSE*|RedHatEnterpriseServer)
         case $(lsb_release -si) in

qa/workunits/cephtool/test.sh

@@ -1397,6 +1397,24 @@ function test_mon_tell()
   ceph_watch_wait 'mon.1 \[DBG\] from.*cmd=\[{"prefix": "version"}\]: dispatch'
 }
 
+function test_mon_crushmap_validation()
+{
+  local map=$TMPDIR/map
+  ceph osd getcrushmap -o $map
+  # crushtool validation timesout and is ignored
+  cat > $TMPDIR/crushtool <<EOF
+#!/bin/sh
+sleep 1000
+exit 0 # success
+EOF
+  chmod +x $TMPDIR/crushtool
+  ceph tell mon.* injectargs --crushtool $TMPDIR/crushtool
+  ceph osd setcrushmap -i $map 2>&1 | grep 'took too long'
+  ceph tell mon.* injectargs --crushtool crushtool
+  # crushtool validation succeeds
+  ceph osd setcrushmap -i $map
+}
+
 #
 # New tests should be added to the TESTS array below
 #
@@ -1431,6 +1449,7 @@ MON_TESTS+=" mon_osd_erasure_code"
 MON_TESTS+=" mon_osd_misc"
 MON_TESTS+=" mon_heap_profiler"
 MON_TESTS+=" mon_tell"
+MON_TESTS+=" mon_crushmap_validation"
 
 OSD_TESTS+=" osd_bench"
 

qa/workunits/rbd/copy.sh

@@ -233,11 +233,15 @@ test_locking() {
     rbd lock list test1 | grep ' 2 '
     rbd lock add test1 id2 --shared tag
     rbd lock list test1 | grep ' 3 '
-    LOCKER=$(rbd lock list test1 | tail -n 1 | awk '{print $1;}')
-    ID=$(rbd lock list test1 | tail -n 1 | awk '{print $2;}')
-    rbd lock remove test1 $ID $LOCKER
-    # locks don't prevent you from removing an image,
-    # just from taking a lock
+    rbd lock list test1 | tail -n 1 | awk '{print $2, $1;}' | xargs rbd lock remove test1
+    if rbd info test1 | grep -qE "features:.*exclusive"
+    then
+      # new locking functionality requires all locks to be released
+      while [ -n "$(rbd lock list test1)" ]
+      do
+        rbd lock list test1 | tail -n 1 | awk '{print $2, $1;}' | xargs rbd lock remove test1
+      done
+    fi
     rbd rm test1
 }
 

qa/workunits/rbd/permissions.sh

@@ -12,11 +12,11 @@ delete_pools() {
 }
 
 recreate_pools() {
-    create_pools
     delete_pools
+    create_pools
 }
 
-delete_uers() {
+delete_users() {
     (ceph auth del client.volumes || true) >/dev/null 2>&1
     (ceph auth del client.images || true) >/dev/null 2>&1
 }
@@ -26,31 +26,54 @@ create_users() {
     ceph auth get-or-create client.images mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool images' >> $KEYRING
 }
 
+expect() {
+
+  set +e
+
+  local expected_ret=$1
+  local ret
+
+  shift
+  cmd=$@
+
+  eval $cmd
+  ret=$?
+
+  set -e
+
+  if [[ $ret -ne $expected_ret ]]; then
+    echo "ERROR: running \'$cmd\': expected $expected_ret got $ret"
+    return 1
+  fi
+
+  return 0
+}
+
 test_images_access() {
-    rbd -k $KEYRING --id images create --format 2 -s 1 images/foo
+    rbd -k $KEYRING --id images create --image-format 2 -s 1 images/foo
     rbd -k $KEYRING --id images snap create images/foo@snap
     rbd -k $KEYRING --id images snap protect images/foo@snap
     rbd -k $KEYRING --id images snap unprotect images/foo@snap
     rbd -k $KEYRING --id images snap protect images/foo@snap
     rbd -k $KEYRING --id images export images/foo@snap - >/dev/null
-    ! rbd -k $KEYRING --id images snap rm images/foo@snap
+    expect 16 rbd -k $KEYRING --id images snap rm images/foo@snap
 
     rbd -k $KEYRING --id volumes clone images/foo@snap volumes/child
-    ! rbd -k $KEYRING --id images snap unprotect images/foo@snap
-    ! rbd -k $KEYRING --id volumes snap unprotect images/foo@snap
-    ! rbd -k $KEYRING --id images flatten volumes/child
+    expect 16 rbd -k $KEYRING --id images snap unprotect images/foo@snap
+    expect 1 rbd -k $KEYRING --id volumes snap unprotect images/foo@snap
+    expect 1 rbd -k $KEYRING --id images flatten volumes/child
     rbd -k $KEYRING --id volumes flatten volumes/child
-    ! rbd -k $KEYRING --id volumes snap unprotect images/foo@snap
+    expect 1 rbd -k $KEYRING --id volumes snap unprotect images/foo@snap
     rbd -k $KEYRING --id images snap unprotect images/foo@snap
 
-    ! rbd -k $KEYRING --id images rm images/foo
+    expect 39 rbd -k $KEYRING --id images rm images/foo
     rbd -k $KEYRING --id images snap rm images/foo@snap
     rbd -k $KEYRING --id images rm images/foo
     rbd -k $KEYRING --id volumes rm volumes/child
 }
 
 test_volumes_access() {
-    rbd -k $KEYRING --id images create --format 2 -s 1 images/foo
+    rbd -k $KEYRING --id images create --image-format 2 -s 1 images/foo
     rbd -k $KEYRING --id images snap create images/foo@snap
     rbd -k $KEYRING --id images snap protect images/foo@snap
 
@@ -64,16 +87,16 @@ test_volumes_access() {
     rbd -k $KEYRING --id volumes lock list images/foo
 
     # commands that fail with read-only access
-    ! rbd -k $KEYRING --id volumes resize -s 2 images/foo --allow-shrink
-    ! rbd -k $KEYRING --id volumes snap create images/foo@2
-    ! rbd -k $KEYRING --id volumes snap rollback images/foo@snap
-    ! rbd -k $KEYRING --id volumes snap remove images/foo@snap
-    ! rbd -k $KEYRING --id volumes snap purge images/foo
-    ! rbd -k $KEYRING --id volumes snap unprotect images/foo@snap
-    ! rbd -k $KEYRING --id volumes flatten images/foo
-    ! rbd -k $KEYRING --id volumes lock add images/foo test
-    ! rbd -k $KEYRING --id volumes lock remove images/foo test locker
-    ! rbd -k $KEYRING --id volumes ls rbd
+    expect 1 rbd -k $KEYRING --id volumes resize -s 2 images/foo --allow-shrink
+    expect 1 rbd -k $KEYRING --id volumes snap create images/foo@2
+    expect 1 rbd -k $KEYRING --id volumes snap rollback images/foo@snap
+    expect 1 rbd -k $KEYRING --id volumes snap remove images/foo@snap
+    expect 1 rbd -k $KEYRING --id volumes snap purge images/foo
+    expect 1 rbd -k $KEYRING --id volumes snap unprotect images/foo@snap
+    expect 1 rbd -k $KEYRING --id volumes flatten images/foo
+    expect 1 rbd -k $KEYRING --id volumes lock add images/foo test
+    expect 1 rbd -k $KEYRING --id volumes lock remove images/foo test locker
+    expect 1 rbd -k $KEYRING --id volumes ls rbd
 
     # create clone and snapshot
     rbd -k $KEYRING --id volumes clone images/foo@snap volumes/child
@@ -82,14 +105,14 @@ test_volumes_access() {
     rbd -k $KEYRING --id volumes snap create volumes/child@snap2
 
     # make sure original snapshot stays protected
-    ! rbd -k $KEYRING --id images snap unprotect images/foo@snap
+    expect 16 rbd -k $KEYRING --id images snap unprotect images/foo@snap
     rbd -k $KEYRING --id volumes flatten volumes/child
-    ! rbd -k $KEYRING --id images snap unprotect images/foo@snap
+    expect 16 rbd -k $KEYRING --id images snap unprotect images/foo@snap
     rbd -k $KEYRING --id volumes snap rm volumes/child@snap2
-    ! rbd -k $KEYRING --id images snap unprotect images/foo@snap
-    ! rbd -k $KEYRING --id volumes snap rm volumes/child@snap2
+    expect 16 rbd -k $KEYRING --id images snap unprotect images/foo@snap
+    expect 2 rbd -k $KEYRING --id volumes snap rm volumes/child@snap2
     rbd -k $KEYRING --id volumes snap unprotect volumes/child@snap1
-    ! rbd -k $KEYRING --id images snap unprotect images/foo@snap
+    expect 16 rbd -k $KEYRING --id images snap unprotect images/foo@snap
 
     # clean up
     rbd -k $KEYRING --id volumes snap rm volumes/child@snap1

src/common/config_opts.h

@@ -251,6 +251,7 @@ OPTION(mon_osd_min_down_reporters, OPT_INT, 1)   // number of OSDs who need to r
 OPTION(mon_osd_min_down_reports, OPT_INT, 3)     // number of times a down OSD must be reported for it to count
 OPTION(mon_osd_force_trim_to, OPT_INT, 0)   // force mon to trim maps to this point, regardless of min_last_epoch_clean (dangerous, use with care)
 OPTION(mon_mds_force_trim_to, OPT_INT, 0)   // force mon to trim mdsmaps to this point (dangerous, use with care)
+OPTION(crushtool, OPT_STR, "crushtool")
 
 // dump transactions
 OPTION(mon_debug_dump_transactions, OPT_BOOL, false)

src/crush/CrushTester.cc

@@ -1,4 +1,5 @@
 
+#include "include/stringify.h"
 #include "CrushTester.h"
 
 #include <algorithm>
@@ -355,14 +356,17 @@ void CrushTester::write_integer_indexed_scalar_data_string(vector<string> &dst,
   dst.push_back( data_buffer.str() );
 }
 
-int CrushTester::test_with_crushtool()
+int CrushTester::test_with_crushtool(const string& crushtool,
+                                     int timeout)
 {
+  string timeout_string = stringify(timeout);
   vector<const char *> cmd_args;
-  cmd_args.push_back("crushtool");
+  cmd_args.push_back("timeout");
+  cmd_args.push_back(timeout_string.c_str());
+  cmd_args.push_back(crushtool.c_str());
   cmd_args.push_back("-i");
   cmd_args.push_back("-");
   cmd_args.push_back("--test");
-  cmd_args.push_back("--output-csv");
   cmd_args.push_back(NULL);
 
   int pipefds[2];
@@ -413,9 +417,13 @@ int CrushTester::test_with_crushtool()
     // major success!
     return 0;
   }
+  if (r == 124) {
+    // the test takes longer than timeout and was interrupted
+    return -EINTR;
+  }
 
   if (r == ENOENT) {
-    err << "unable to find 'crushtool' to test the map";
+    err << "unable to find " << cmd_args << " to test the map";
     return -ENOENT;
   }
 

src/crush/CrushTester.h

@@ -334,7 +334,8 @@ class CrushTester {
   }
 
   int test();
-  int test_with_crushtool();
+  int test_with_crushtool(const string& crushtool,
+			  int timeout);
 };
 
 #endif

src/crush/grammar.h

@@ -113,11 +113,7 @@ struct crush_grammar : public grammar<crush_grammar>
 
       // buckets
       bucket_id = str_p("id") >> negint;
-      bucket_alg = str_p("alg") >> ( str_p("uniform") |
-				     str_p("list") |
-				     str_p("tree") |
-				     str_p("straw") |
-				     str_p("straw2"));
+      bucket_alg = str_p("alg") >> name;
       bucket_hash = str_p("hash") >> ( integer |
 				       str_p("rjenkins1") );
       bucket_item = str_p("item") >> name

src/include/util.h

@@ -30,13 +30,47 @@ struct ceph_data_stats
   uint64_t byte_avail;
   int avail_percent;
 
+  ceph_data_stats() :
+    byte_total(0),
+    byte_used(0),
+    byte_avail(0),
+    avail_percent(0)
+  { }
+
   void dump(Formatter *f) const {
     assert(f != NULL);
     f->dump_int("total", byte_total);
     f->dump_int("used", byte_used);
     f->dump_int("avail", byte_avail);
     f->dump_int("avail_percent", avail_percent);
   }
+
+  void encode(bufferlist &bl) const {
+    ENCODE_START(1, 1, bl);
+    ::encode(byte_total, bl);
+    ::encode(byte_used, bl);
+    ::encode(byte_avail, bl);
+    ::encode(avail_percent, bl);
+    ENCODE_FINISH(bl);
+  }
+
+  void decode(bufferlist::iterator &p) {
+    DECODE_START(1, p);
+    ::decode(byte_total, p);
+    ::decode(byte_used, p);
+    ::decode(byte_avail, p);
+    ::decode(avail_percent, p);
+    DECODE_FINISH(p);
+  }
+
+  static void generate_test_instances(list<ceph_data_stats*>& ls) {
+    ls.push_back(new ceph_data_stats);
+    ls.push_back(new ceph_data_stats);
+    ls.back()->byte_total = 1024*1024;
+    ls.back()->byte_used = 512*1024;
+    ls.back()->byte_avail = 512*1024;
+    ls.back()->avail_percent = 50;
+  }
 };
 typedef struct ceph_data_stats ceph_data_stats_t;
 

src/librbd/AioCompletion.cc

@@ -26,14 +26,7 @@ namespace librbd {
   void AioCompletion::finish_adding_requests(CephContext *cct)
   {
     ldout(cct, 20) << "AioCompletion::finish_adding_requests " << (void*)this << " pending " << pending_count << dendl;
-    lock.Lock();
-    assert(building);
-    building = false;
-    if (!pending_count) {
-      finalize(cct, rval);
-      complete();
-    }
-    lock.Unlock();
+    unblock(cct);
   }
 
   int AioCompletion::wait_for_complete() {
@@ -89,6 +82,7 @@ namespace librbd {
       break;
     }
 
+    // note: possible for image to be closed after op marked finished
     async_op.finish_op();
 
     if (complete_cb) {
@@ -113,7 +107,7 @@ namespace librbd {
     }
     assert(pending_count);
     int count = --pending_count;
-    if (!count && !building) {
+    if (!count && blockers == 0) {
       finalize(cct, rval);
       complete();
     }