From 11165b1de3b355cb9d4ec077a00bc97844f8fe7e Mon Sep 17 00:00:00 2001 From: Jason Dillaman Date: Thu, 10 Dec 2020 16:13:23 -0500 Subject: [PATCH 1/4] librbd/mirror: increase debug logging of snapshot state machines Try to keep debug level 20 for IO state machines so that setting the debug level to something lower should show the manipulation of the mirror snapshots. Signed-off-by: Jason Dillaman --- .../snapshot/CreateNonPrimaryRequest.cc | 22 +++++------ .../mirror/snapshot/CreatePrimaryRequest.cc | 20 +++++----- src/librbd/mirror/snapshot/DemoteRequest.cc | 6 +-- .../mirror/snapshot/GetImageStateRequest.cc | 6 +-- src/librbd/mirror/snapshot/PromoteRequest.cc | 38 +++++++++---------- .../snapshot/RemoveImageStateRequest.cc | 10 ++--- .../mirror/snapshot/SetImageStateRequest.cc | 20 +++++----- .../mirror/snapshot/UnlinkPeerRequest.cc | 26 ++++++------- .../mirror/snapshot/WriteImageStateRequest.cc | 6 +-- 9 files changed, 77 insertions(+), 77 deletions(-) diff --git a/src/librbd/mirror/snapshot/CreateNonPrimaryRequest.cc b/src/librbd/mirror/snapshot/CreateNonPrimaryRequest.cc index ee6b93ec2cbfe..eed0aa506a303 100644 --- a/src/librbd/mirror/snapshot/CreateNonPrimaryRequest.cc +++ b/src/librbd/mirror/snapshot/CreateNonPrimaryRequest.cc @@ -51,7 +51,7 @@ void CreateNonPrimaryRequest::refresh_image() { } CephContext *cct = m_image_ctx->cct; - ldout(cct, 20) << dendl; + ldout(cct, 15) << dendl; auto ctx = create_context_callback< CreateNonPrimaryRequest, @@ -62,7 +62,7 @@ void CreateNonPrimaryRequest::refresh_image() { template void CreateNonPrimaryRequest::handle_refresh_image(int r) { CephContext *cct = m_image_ctx->cct; - ldout(cct, 20) << "r=" << r << dendl; + ldout(cct, 15) << "r=" << r << dendl; if (r < 0) { lderr(cct) << "failed to refresh image: " << cpp_strerror(r) << dendl; @@ -76,7 +76,7 @@ void CreateNonPrimaryRequest::handle_refresh_image(int r) { template void CreateNonPrimaryRequest::get_mirror_image() { CephContext *cct = m_image_ctx->cct; - ldout(cct, 20) << dendl; + ldout(cct, 15) << dendl; librados::ObjectReadOperation op; cls_client::mirror_image_get_start(&op, m_image_ctx->id); @@ -92,7 +92,7 @@ void CreateNonPrimaryRequest::get_mirror_image() { template void CreateNonPrimaryRequest::handle_get_mirror_image(int r) { CephContext *cct = m_image_ctx->cct; - ldout(cct, 20) << "r=" << r << dendl; + ldout(cct, 15) << "r=" << r << dendl; cls::rbd::MirrorImage mirror_image; if (r == 0) { @@ -134,7 +134,7 @@ void CreateNonPrimaryRequest::get_mirror_peers() { } CephContext *cct = m_image_ctx->cct; - ldout(cct, 20) << dendl; + ldout(cct, 15) << dendl; librados::ObjectReadOperation op; cls_client::mirror_peer_list_start(&op); @@ -151,7 +151,7 @@ void CreateNonPrimaryRequest::get_mirror_peers() { template void CreateNonPrimaryRequest::handle_get_mirror_peers(int r) { CephContext *cct = m_image_ctx->cct; - ldout(cct, 20) << "r=" << r << dendl; + ldout(cct, 15) << "r=" << r << dendl; std::vector peers; if (r == 0) { @@ -189,7 +189,7 @@ void CreateNonPrimaryRequest::create_snapshot() { } ns.snap_seqs = m_snap_seqs; ns.complete = is_orphan(); - ldout(cct, 20) << "ns=" << ns << dendl; + ldout(cct, 15) << "ns=" << ns << dendl; auto ctx = create_context_callback< CreateNonPrimaryRequest, @@ -200,7 +200,7 @@ void CreateNonPrimaryRequest::create_snapshot() { template void CreateNonPrimaryRequest::handle_create_snapshot(int r) { CephContext *cct = m_image_ctx->cct; - ldout(cct, 20) << "r=" << r << dendl; + ldout(cct, 15) << "r=" << r << dendl; if (r < 0) { lderr(cct) << "failed to create mirror snapshot: " << cpp_strerror(r) @@ -231,7 +231,7 @@ void CreateNonPrimaryRequest::write_image_state() { } CephContext *cct = m_image_ctx->cct; - ldout(cct, 20) << dendl; + ldout(cct, 15) << dendl; auto ctx = create_context_callback< CreateNonPrimaryRequest, @@ -245,7 +245,7 @@ void CreateNonPrimaryRequest::write_image_state() { template void CreateNonPrimaryRequest::handle_write_image_state(int r) { CephContext *cct = m_image_ctx->cct; - ldout(cct, 20) << "r=" << r << dendl; + ldout(cct, 15) << "r=" << r << dendl; if (r < 0) { lderr(cct) << "failed to write image state: " << cpp_strerror(r) @@ -260,7 +260,7 @@ void CreateNonPrimaryRequest::handle_write_image_state(int r) { template void CreateNonPrimaryRequest::finish(int r) { CephContext *cct = m_image_ctx->cct; - ldout(cct, 20) << "r=" << r << dendl; + ldout(cct, 15) << "r=" << r << dendl; m_on_finish->complete(r); delete this; diff --git a/src/librbd/mirror/snapshot/CreatePrimaryRequest.cc b/src/librbd/mirror/snapshot/CreatePrimaryRequest.cc index 36961d35e848e..70962cc6efbc2 100644 --- a/src/librbd/mirror/snapshot/CreatePrimaryRequest.cc +++ b/src/librbd/mirror/snapshot/CreatePrimaryRequest.cc @@ -59,7 +59,7 @@ void CreatePrimaryRequest::send() { template void CreatePrimaryRequest::get_mirror_peers() { CephContext *cct = m_image_ctx->cct; - ldout(cct, 20) << dendl; + ldout(cct, 15) << dendl; librados::ObjectReadOperation op; cls_client::mirror_peer_list_start(&op); @@ -76,7 +76,7 @@ void CreatePrimaryRequest::get_mirror_peers() { template void CreatePrimaryRequest::handle_get_mirror_peers(int r) { CephContext *cct = m_image_ctx->cct; - ldout(cct, 20) << "r=" << r << dendl; + ldout(cct, 15) << "r=" << r << dendl; std::vector peers; if (r == 0) { @@ -117,7 +117,7 @@ void CreatePrimaryRequest::create_snapshot() { m_mirror_peer_uuids, "", m_clean_since_snap_id}; CephContext *cct = m_image_ctx->cct; - ldout(cct, 20) << "name=" << m_snap_name << ", " + ldout(cct, 15) << "name=" << m_snap_name << ", " << "ns=" << ns << dendl; auto ctx = create_context_callback< CreatePrimaryRequest, @@ -129,7 +129,7 @@ void CreatePrimaryRequest::create_snapshot() { template void CreatePrimaryRequest::handle_create_snapshot(int r) { CephContext *cct = m_image_ctx->cct; - ldout(cct, 20) << "r=" << r << dendl; + ldout(cct, 15) << "r=" << r << dendl; if (r < 0) { lderr(cct) << "failed to create mirror snapshot: " << cpp_strerror(r) @@ -151,7 +151,7 @@ void CreatePrimaryRequest::refresh_image() { } CephContext *cct = m_image_ctx->cct; - ldout(cct, 20) << dendl; + ldout(cct, 15) << dendl; auto ctx = create_context_callback< CreatePrimaryRequest, @@ -162,7 +162,7 @@ void CreatePrimaryRequest::refresh_image() { template void CreatePrimaryRequest::handle_refresh_image(int r) { CephContext *cct = m_image_ctx->cct; - ldout(cct, 20) << "r=" << r << dendl; + ldout(cct, 15) << "r=" << r << dendl; if (r < 0) { lderr(cct) << "failed to refresh image: " << cpp_strerror(r) << dendl; @@ -174,7 +174,7 @@ void CreatePrimaryRequest::handle_refresh_image(int r) { std::shared_lock image_locker{m_image_ctx->image_lock}; *m_snap_id = m_image_ctx->get_snap_id( cls::rbd::MirrorSnapshotNamespace{}, m_snap_name); - ldout(cct, 20) << "snap_id=" << *m_snap_id << dendl; + ldout(cct, 15) << "snap_id=" << *m_snap_id << dendl; } unlink_peer(); @@ -225,7 +225,7 @@ void CreatePrimaryRequest::unlink_peer() { } CephContext *cct = m_image_ctx->cct; - ldout(cct, 20) << "peer=" << peer_uuid << ", snap_id=" << snap_id << dendl; + ldout(cct, 15) << "peer=" << peer_uuid << ", snap_id=" << snap_id << dendl; auto ctx = create_context_callback< CreatePrimaryRequest, @@ -237,7 +237,7 @@ void CreatePrimaryRequest::unlink_peer() { template void CreatePrimaryRequest::handle_unlink_peer(int r) { CephContext *cct = m_image_ctx->cct; - ldout(cct, 20) << "r=" << r << dendl; + ldout(cct, 15) << "r=" << r << dendl; if (r < 0) { lderr(cct) << "failed to unlink peer: " << cpp_strerror(r) << dendl; @@ -251,7 +251,7 @@ void CreatePrimaryRequest::handle_unlink_peer(int r) { template void CreatePrimaryRequest::finish(int r) { CephContext *cct = m_image_ctx->cct; - ldout(cct, 20) << "r=" << r << dendl; + ldout(cct, 15) << "r=" << r << dendl; m_on_finish->complete(r); delete this; diff --git a/src/librbd/mirror/snapshot/DemoteRequest.cc b/src/librbd/mirror/snapshot/DemoteRequest.cc index f9c14ea8a2b36..ccaa33c83492b 100644 --- a/src/librbd/mirror/snapshot/DemoteRequest.cc +++ b/src/librbd/mirror/snapshot/DemoteRequest.cc @@ -66,7 +66,7 @@ void DemoteRequest::handle_enable_non_primary_feature(int r) { template void DemoteRequest::create_snapshot() { CephContext *cct = m_image_ctx->cct; - ldout(cct, 20) << dendl; + ldout(cct, 15) << dendl; auto ctx = create_context_callback< DemoteRequest, &DemoteRequest::handle_create_snapshot>(this); @@ -82,7 +82,7 @@ void DemoteRequest::create_snapshot() { template void DemoteRequest::handle_create_snapshot(int r) { CephContext *cct = m_image_ctx->cct; - ldout(cct, 20) << "r=" << r << dendl; + ldout(cct, 15) << "r=" << r << dendl; if (r < 0) { lderr(cct) << "failed to create mirror snapshot: " << cpp_strerror(r) @@ -97,7 +97,7 @@ void DemoteRequest::handle_create_snapshot(int r) { template void DemoteRequest::finish(int r) { CephContext *cct = m_image_ctx->cct; - ldout(cct, 20) << "r=" << r << dendl; + ldout(cct, 15) << "r=" << r << dendl; m_on_finish->complete(r); delete this; diff --git a/src/librbd/mirror/snapshot/GetImageStateRequest.cc b/src/librbd/mirror/snapshot/GetImageStateRequest.cc index e7db57bfc598d..4692f88cb80ee 100644 --- a/src/librbd/mirror/snapshot/GetImageStateRequest.cc +++ b/src/librbd/mirror/snapshot/GetImageStateRequest.cc @@ -33,7 +33,7 @@ void GetImageStateRequest::read_object() { auto oid = util::image_state_object_name(m_image_ctx, m_snap_id, m_object_index); - ldout(cct, 20) << oid << dendl; + ldout(cct, 15) << oid << dendl; librados::ObjectReadOperation op; m_bl.clear(); @@ -50,7 +50,7 @@ void GetImageStateRequest::read_object() { template void GetImageStateRequest::handle_read_object(int r) { CephContext *cct = m_image_ctx->cct; - ldout(cct, 20) << "r=" << r << dendl; + ldout(cct, 15) << "r=" << r << dendl; if (r < 0) { lderr(cct) << "failed to read image state object: " << cpp_strerror(r) @@ -91,7 +91,7 @@ void GetImageStateRequest::handle_read_object(int r) { template void GetImageStateRequest::finish(int r) { CephContext *cct = m_image_ctx->cct; - ldout(cct, 20) << "r=" << r << dendl; + ldout(cct, 15) << "r=" << r << dendl; if (r == 0) { try { diff --git a/src/librbd/mirror/snapshot/PromoteRequest.cc b/src/librbd/mirror/snapshot/PromoteRequest.cc index f80db3ada7785..9718c299e0378 100644 --- a/src/librbd/mirror/snapshot/PromoteRequest.cc +++ b/src/librbd/mirror/snapshot/PromoteRequest.cc @@ -45,7 +45,7 @@ void PromoteRequest::send() { return; } - ldout(cct, 20) << "requires_orphan=" << requires_orphan << ", " + ldout(cct, 15) << "requires_orphan=" << requires_orphan << ", " << "rollback_snap_id=" << m_rollback_snap_id << dendl; create_orphan_snapshot(); } @@ -53,7 +53,7 @@ void PromoteRequest::send() { template void PromoteRequest::create_orphan_snapshot() { CephContext *cct = m_image_ctx->cct; - ldout(cct, 20) << dendl; + ldout(cct, 15) << dendl; auto ctx = create_context_callback< PromoteRequest, @@ -67,7 +67,7 @@ void PromoteRequest::create_orphan_snapshot() { template void PromoteRequest::handle_create_orphan_snapshot(int r) { CephContext *cct = m_image_ctx->cct; - ldout(cct, 20) << "r=" << r << dendl; + ldout(cct, 15) << "r=" << r << dendl; if (r < 0) { lderr(cct) << "failed to create orphan snapshot: " << cpp_strerror(r) @@ -82,7 +82,7 @@ void PromoteRequest::handle_create_orphan_snapshot(int r) { template void PromoteRequest::list_watchers() { CephContext *cct = m_image_ctx->cct; - ldout(cct, 20) << dendl; + ldout(cct, 15) << dendl; auto ctx = create_context_callback< PromoteRequest, @@ -99,7 +99,7 @@ void PromoteRequest::list_watchers() { template void PromoteRequest::handle_list_watchers(int r) { CephContext *cct = m_image_ctx->cct; - ldout(cct, 20) << "r=" << r << dendl; + ldout(cct, 15) << "r=" << r << dendl; if (r < 0) { lderr(cct) << "failed to list watchers: " << cpp_strerror(r) @@ -119,7 +119,7 @@ void PromoteRequest::handle_list_watchers(int r) { template void PromoteRequest::wait_update_notify() { CephContext *cct = m_image_ctx->cct; - ldout(cct, 20) << dendl; + ldout(cct, 15) << dendl; ImageCtx::get_timer_instance(cct, &m_timer, &m_timer_lock); @@ -142,7 +142,7 @@ void PromoteRequest::wait_update_notify() { template void PromoteRequest::handle_update_notify() { CephContext *cct = m_image_ctx->cct; - ldout(cct, 20) << dendl; + ldout(cct, 15) << dendl; std::lock_guard timer_lock{*m_timer_lock}; m_scheduler_ticks = 0; @@ -153,7 +153,7 @@ void PromoteRequest::scheduler_unregister_update_watcher() { ceph_assert(ceph_mutex_is_locked(*m_timer_lock)); CephContext *cct = m_image_ctx->cct; - ldout(cct, 20) << "scheduler_ticks=" << m_scheduler_ticks << dendl; + ldout(cct, 15) << "scheduler_ticks=" << m_scheduler_ticks << dendl; if (m_scheduler_ticks > 0) { m_scheduler_ticks--; @@ -171,7 +171,7 @@ void PromoteRequest::scheduler_unregister_update_watcher() { template void PromoteRequest::unregister_update_watcher() { CephContext *cct = m_image_ctx->cct; - ldout(cct, 20) << dendl; + ldout(cct, 15) << dendl; auto ctx = create_context_callback< PromoteRequest, @@ -183,7 +183,7 @@ void PromoteRequest::unregister_update_watcher() { template void PromoteRequest::handle_unregister_update_watcher(int r) { CephContext *cct = m_image_ctx->cct; - ldout(cct, 20) << "r=" << r << dendl; + ldout(cct, 15) << "r=" << r << dendl; if (r < 0) { lderr(cct) << "failed to unregister update watcher: " << cpp_strerror(r) @@ -202,7 +202,7 @@ void PromoteRequest::acquire_exclusive_lock() { if (m_image_ctx->exclusive_lock != nullptr && !m_image_ctx->exclusive_lock->is_lock_owner()) { CephContext *cct = m_image_ctx->cct; - ldout(cct, 20) << dendl; + ldout(cct, 15) << dendl; m_lock_acquired = true; m_image_ctx->exclusive_lock->block_requests(0); @@ -222,7 +222,7 @@ void PromoteRequest::acquire_exclusive_lock() { template void PromoteRequest::handle_acquire_exclusive_lock(int r) { CephContext *cct = m_image_ctx->cct; - ldout(cct, 20) << "r=" << r << dendl; + ldout(cct, 15) << "r=" << r << dendl; if (r < 0) { lderr(cct) << "failed to acquire exclusive lock: " << cpp_strerror(r) @@ -252,7 +252,7 @@ void PromoteRequest::rollback() { } CephContext *cct = m_image_ctx->cct; - ldout(cct, 20) << dendl; + ldout(cct, 15) << dendl; std::shared_lock owner_locker{m_image_ctx->owner_lock}; std::shared_lock image_locker{m_image_ctx->image_lock}; @@ -275,7 +275,7 @@ void PromoteRequest::rollback() { template void PromoteRequest::handle_rollback(int r) { CephContext *cct = m_image_ctx->cct; - ldout(cct, 20) << "r=" << r << dendl; + ldout(cct, 15) << "r=" << r << dendl; if (r < 0) { lderr(cct) << "failed to rollback: " << cpp_strerror(r) << dendl; @@ -289,7 +289,7 @@ void PromoteRequest::handle_rollback(int r) { template void PromoteRequest::create_promote_snapshot() { CephContext *cct = m_image_ctx->cct; - ldout(cct, 20) << dendl; + ldout(cct, 15) << dendl; auto ctx = create_context_callback< PromoteRequest, @@ -306,7 +306,7 @@ void PromoteRequest::create_promote_snapshot() { template void PromoteRequest::handle_create_promote_snapshot(int r) { CephContext *cct = m_image_ctx->cct; - ldout(cct, 20) << "r=" << r << dendl; + ldout(cct, 15) << "r=" << r << dendl; if (r < 0) { lderr(cct) << "failed to create promote snapshot: " << cpp_strerror(r) @@ -358,7 +358,7 @@ void PromoteRequest::release_exclusive_lock() { std::unique_lock locker{m_image_ctx->owner_lock}; if (m_image_ctx->exclusive_lock != nullptr) { CephContext *cct = m_image_ctx->cct; - ldout(cct, 20) << dendl; + ldout(cct, 15) << dendl; m_image_ctx->exclusive_lock->unblock_requests(); @@ -377,7 +377,7 @@ void PromoteRequest::release_exclusive_lock() { template void PromoteRequest::handle_release_exclusive_lock(int r) { CephContext *cct = m_image_ctx->cct; - ldout(cct, 20) << "r=" << r << dendl; + ldout(cct, 15) << "r=" << r << dendl; if (r < 0) { lderr(cct) << "failed to release exclusive lock: " << cpp_strerror(r) @@ -392,7 +392,7 @@ void PromoteRequest::handle_release_exclusive_lock(int r) { template void PromoteRequest::finish(int r) { CephContext *cct = m_image_ctx->cct; - ldout(cct, 20) << "r=" << r << dendl; + ldout(cct, 15) << "r=" << r << dendl; m_on_finish->complete(r); delete this; diff --git a/src/librbd/mirror/snapshot/RemoveImageStateRequest.cc b/src/librbd/mirror/snapshot/RemoveImageStateRequest.cc index 05920c26f51bd..204e0489adf82 100644 --- a/src/librbd/mirror/snapshot/RemoveImageStateRequest.cc +++ b/src/librbd/mirror/snapshot/RemoveImageStateRequest.cc @@ -32,7 +32,7 @@ void RemoveImageStateRequest::get_object_count() { CephContext *cct = m_image_ctx->cct; auto oid = util::image_state_object_name(m_image_ctx, m_snap_id, 0); - ldout(cct, 20) << oid << dendl; + ldout(cct, 15) << oid << dendl; librados::ObjectReadOperation op; op.read(0, 0, &m_bl, nullptr); @@ -48,7 +48,7 @@ void RemoveImageStateRequest::get_object_count() { template void RemoveImageStateRequest::handle_get_object_count(int r) { CephContext *cct = m_image_ctx->cct; - ldout(cct, 20) << "r=" << r << dendl; + ldout(cct, 15) << "r=" << r << dendl; if (r < 0) { lderr(cct) << "failed to read image state object: " << cpp_strerror(r) @@ -82,7 +82,7 @@ void RemoveImageStateRequest::remove_object() { auto oid = util::image_state_object_name(m_image_ctx, m_snap_id, m_object_count); - ldout(cct, 20) << oid << dendl; + ldout(cct, 15) << oid << dendl; librados::ObjectWriteOperation op; op.remove(); @@ -98,7 +98,7 @@ void RemoveImageStateRequest::remove_object() { template void RemoveImageStateRequest::handle_remove_object(int r) { CephContext *cct = m_image_ctx->cct; - ldout(cct, 20) << "r=" << r << dendl; + ldout(cct, 15) << "r=" << r << dendl; if (r < 0 && r != -ENOENT) { lderr(cct) << "failed to remove image state object: " << cpp_strerror(r) @@ -118,7 +118,7 @@ void RemoveImageStateRequest::handle_remove_object(int r) { template void RemoveImageStateRequest::finish(int r) { CephContext *cct = m_image_ctx->cct; - ldout(cct, 20) << "r=" << r << dendl; + ldout(cct, 15) << "r=" << r << dendl; m_on_finish->complete(r); delete this; diff --git a/src/librbd/mirror/snapshot/SetImageStateRequest.cc b/src/librbd/mirror/snapshot/SetImageStateRequest.cc index d0caccdf58aee..f68bff8b92c99 100644 --- a/src/librbd/mirror/snapshot/SetImageStateRequest.cc +++ b/src/librbd/mirror/snapshot/SetImageStateRequest.cc @@ -33,7 +33,7 @@ void SetImageStateRequest::send() { template void SetImageStateRequest::get_snap_limit() { CephContext *cct = m_image_ctx->cct; - ldout(cct, 20) << dendl; + ldout(cct, 15) << dendl; librados::ObjectReadOperation op; cls_client::snapshot_get_limit_start(&op); @@ -51,7 +51,7 @@ void SetImageStateRequest::get_snap_limit() { template void SetImageStateRequest::handle_get_snap_limit(int r) { CephContext *cct = m_image_ctx->cct; - ldout(cct, 20) << "r=" << r << dendl; + ldout(cct, 15) << "r=" << r << dendl; if (r == 0) { auto it = m_bl.cbegin(); @@ -65,7 +65,7 @@ void SetImageStateRequest::handle_get_snap_limit(int r) { return; } - ldout(cct, 20) << "snap_limit=" << m_image_state.snap_limit << dendl; + ldout(cct, 15) << "snap_limit=" << m_image_state.snap_limit << dendl; get_metadata(); } @@ -73,7 +73,7 @@ void SetImageStateRequest::handle_get_snap_limit(int r) { template void SetImageStateRequest::get_metadata() { CephContext *cct = m_image_ctx->cct; - ldout(cct, 20) << dendl; + ldout(cct, 15) << dendl; auto ctx = create_context_callback< SetImageStateRequest, @@ -87,7 +87,7 @@ void SetImageStateRequest::get_metadata() { template void SetImageStateRequest::handle_get_metadata(int r) { CephContext *cct = m_image_ctx->cct; - ldout(cct, 20) << "r=" << r << dendl; + ldout(cct, 15) << "r=" << r << dendl; if (r < 0) { lderr(cct) << "failed to retrieve metadata: " << cpp_strerror(r) @@ -122,7 +122,7 @@ void SetImageStateRequest::handle_get_metadata(int r) { template void SetImageStateRequest::write_image_state() { CephContext *cct = m_image_ctx->cct; - ldout(cct, 20) << dendl; + ldout(cct, 15) << dendl; auto ctx = create_context_callback< SetImageStateRequest, @@ -136,7 +136,7 @@ void SetImageStateRequest::write_image_state() { template void SetImageStateRequest::handle_write_image_state(int r) { CephContext *cct = m_image_ctx->cct; - ldout(cct, 20) << "r=" << r << dendl; + ldout(cct, 15) << "r=" << r << dendl; if (r < 0) { lderr(cct) << "failed to write image state: " << cpp_strerror(r) @@ -151,7 +151,7 @@ void SetImageStateRequest::handle_write_image_state(int r) { template void SetImageStateRequest::update_primary_snapshot() { CephContext *cct = m_image_ctx->cct; - ldout(cct, 20) << dendl; + ldout(cct, 15) << dendl; librados::ObjectWriteOperation op; librbd::cls_client::mirror_image_snapshot_set_copy_progress( @@ -169,7 +169,7 @@ void SetImageStateRequest::update_primary_snapshot() { template void SetImageStateRequest::handle_update_primary_snapshot(int r) { CephContext *cct = m_image_ctx->cct; - ldout(cct, 20) << "r=" << r << dendl; + ldout(cct, 15) << "r=" << r << dendl; if (r < 0) { lderr(cct) << "failed to update primary snapshot: " << cpp_strerror(r) @@ -184,7 +184,7 @@ void SetImageStateRequest::handle_update_primary_snapshot(int r) { template void SetImageStateRequest::finish(int r) { CephContext *cct = m_image_ctx->cct; - ldout(cct, 20) << "r=" << r << dendl; + ldout(cct, 15) << "r=" << r << dendl; m_on_finish->complete(r); delete this; diff --git a/src/librbd/mirror/snapshot/UnlinkPeerRequest.cc b/src/librbd/mirror/snapshot/UnlinkPeerRequest.cc index d82d42d5bad44..2c89d251392cd 100644 --- a/src/librbd/mirror/snapshot/UnlinkPeerRequest.cc +++ b/src/librbd/mirror/snapshot/UnlinkPeerRequest.cc @@ -35,7 +35,7 @@ void UnlinkPeerRequest::send() { template void UnlinkPeerRequest::refresh_image() { CephContext *cct = m_image_ctx->cct; - ldout(cct, 20) << dendl; + ldout(cct, 15) << dendl; auto ctx = create_context_callback< UnlinkPeerRequest, &UnlinkPeerRequest::handle_refresh_image>(this); @@ -45,7 +45,7 @@ void UnlinkPeerRequest::refresh_image() { template void UnlinkPeerRequest::handle_refresh_image(int r) { CephContext *cct = m_image_ctx->cct; - ldout(cct, 20) << "r=" << r << dendl; + ldout(cct, 15) << "r=" << r << dendl; if (r < 0) { lderr(cct) << "failed to refresh image: " << cpp_strerror(r) << dendl; @@ -72,14 +72,14 @@ void UnlinkPeerRequest::unlink_peer() { &snap_it->second.snap_namespace); } else if (boost::get( &snap_it->second.snap_namespace) != nullptr) { - ldout(cct, 20) << "located newer mirror snapshot" << dendl; + ldout(cct, 15) << "located newer mirror snapshot" << dendl; m_newer_mirror_snapshots = true; break; } } if (r == -ENOENT) { - ldout(cct, 20) << "missing snapshot: snap_id=" << m_snap_id << dendl; + ldout(cct, 15) << "missing snapshot: snap_id=" << m_snap_id << dendl; m_image_ctx->image_lock.unlock_shared(); finish(r); return; @@ -102,7 +102,7 @@ void UnlinkPeerRequest::unlink_peer() { } m_image_ctx->image_lock.unlock_shared(); - ldout(cct, 20) << "snap_id=" << m_snap_id << ", " + ldout(cct, 15) << "snap_id=" << m_snap_id << ", " << "mirror_peer_uuid=" << m_mirror_peer_uuid << dendl; librados::ObjectWriteOperation op; librbd::cls_client::mirror_image_snapshot_unlink_peer(&op, m_snap_id, @@ -117,7 +117,7 @@ void UnlinkPeerRequest::unlink_peer() { template void UnlinkPeerRequest::handle_unlink_peer(int r) { CephContext *cct = m_image_ctx->cct; - ldout(cct, 20) << "r=" << r << dendl; + ldout(cct, 15) << "r=" << r << dendl; if (r == -ERESTART || r == -ENOENT) { refresh_image(); @@ -136,7 +136,7 @@ void UnlinkPeerRequest::handle_unlink_peer(int r) { template void UnlinkPeerRequest::notify_update() { CephContext *cct = m_image_ctx->cct; - ldout(cct, 20) << dendl; + ldout(cct, 15) << dendl; auto ctx = create_context_callback< UnlinkPeerRequest, &UnlinkPeerRequest::handle_notify_update>(this); @@ -146,7 +146,7 @@ void UnlinkPeerRequest::notify_update() { template void UnlinkPeerRequest::handle_notify_update(int r) { CephContext *cct = m_image_ctx->cct; - ldout(cct, 20) << "r=" << r << dendl; + ldout(cct, 15) << "r=" << r << dendl; if (r < 0 && r != -ENOENT) { lderr(cct) << "failed to notify update: " << cpp_strerror(r) << dendl; @@ -160,7 +160,7 @@ void UnlinkPeerRequest::handle_notify_update(int r) { template void UnlinkPeerRequest::remove_snapshot() { CephContext *cct = m_image_ctx->cct; - ldout(cct, 20) << dendl; + ldout(cct, 15) << dendl; cls::rbd::SnapshotNamespace snap_namespace; std::string snap_name; @@ -178,7 +178,7 @@ void UnlinkPeerRequest::remove_snapshot() { } if (r == -ENOENT) { - ldout(cct, 20) << "failed to locate snapshot " << m_snap_id << dendl; + ldout(cct, 15) << "failed to locate snapshot " << m_snap_id << dendl; finish(0); return; } @@ -188,7 +188,7 @@ void UnlinkPeerRequest::remove_snapshot() { info.mirror_peer_uuids.erase(m_mirror_peer_uuid); if (!info.mirror_peer_uuids.empty() || !m_newer_mirror_snapshots) { - ldout(cct, 20) << "skipping removal of snapshot: " + ldout(cct, 15) << "skipping removal of snapshot: " << "snap_id=" << m_snap_id << ": " << "mirror_peer_uuid=" << m_mirror_peer_uuid << ", " << "mirror_peer_uuids=" << info.mirror_peer_uuids << dendl; @@ -204,7 +204,7 @@ void UnlinkPeerRequest::remove_snapshot() { template void UnlinkPeerRequest::handle_remove_snapshot(int r) { CephContext *cct = m_image_ctx->cct; - ldout(cct, 20) << "r=" << r << dendl; + ldout(cct, 15) << "r=" << r << dendl; if (r < 0 && r != -ENOENT) { lderr(cct) << "failed to remove snapshot: " << cpp_strerror(r) << dendl; @@ -218,7 +218,7 @@ void UnlinkPeerRequest::handle_remove_snapshot(int r) { template void UnlinkPeerRequest::finish(int r) { CephContext *cct = m_image_ctx->cct; - ldout(cct, 20) << "r=" << r << dendl; + ldout(cct, 15) << "r=" << r << dendl; auto on_finish = m_on_finish; delete this; diff --git a/src/librbd/mirror/snapshot/WriteImageStateRequest.cc b/src/librbd/mirror/snapshot/WriteImageStateRequest.cc index c179b9f021d3f..c79dd7e2ce6b8 100644 --- a/src/librbd/mirror/snapshot/WriteImageStateRequest.cc +++ b/src/librbd/mirror/snapshot/WriteImageStateRequest.cc @@ -66,7 +66,7 @@ void WriteImageStateRequest::write_object() { auto oid = util::image_state_object_name(m_image_ctx, m_snap_id, m_object_count); - ldout(cct, 20) << oid << dendl; + ldout(cct, 15) << oid << dendl; size_t off = m_object_count * m_object_size; size_t len = std::min(m_bl.length() - off, m_object_size); @@ -87,7 +87,7 @@ void WriteImageStateRequest::write_object() { template void WriteImageStateRequest::handle_write_object(int r) { CephContext *cct = m_image_ctx->cct; - ldout(cct, 20) << "r=" << r << dendl; + ldout(cct, 15) << "r=" << r << dendl; if (r < 0) { lderr(cct) << "failed to write object: " << cpp_strerror(r) @@ -107,7 +107,7 @@ void WriteImageStateRequest::handle_write_object(int r) { template void WriteImageStateRequest::finish(int r) { CephContext *cct = m_image_ctx->cct; - ldout(cct, 20) << "r=" << r << dendl; + ldout(cct, 15) << "r=" << r << dendl; m_on_finish->complete(r); delete this; From 529d3c9a073b6ab11f469fd54c72de2d3650cf18 Mon Sep 17 00:00:00 2001 From: Jason Dillaman Date: Thu, 10 Dec 2020 15:05:04 -0500 Subject: [PATCH 2/4] librbd/mirror: ignore non-fatal errors when unlinking the snapshot Notifications can fail if a client crashes or if we race with the removal of the image. Either case should be ignored. Signed-off-by: Jason Dillaman --- src/librbd/mirror/snapshot/UnlinkPeerRequest.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/librbd/mirror/snapshot/UnlinkPeerRequest.cc b/src/librbd/mirror/snapshot/UnlinkPeerRequest.cc index 2c89d251392cd..6e18842493606 100644 --- a/src/librbd/mirror/snapshot/UnlinkPeerRequest.cc +++ b/src/librbd/mirror/snapshot/UnlinkPeerRequest.cc @@ -148,7 +148,10 @@ void UnlinkPeerRequest::handle_notify_update(int r) { CephContext *cct = m_image_ctx->cct; ldout(cct, 15) << "r=" << r << dendl; - if (r < 0 && r != -ENOENT) { + if (r == -ENOENT || r == -ETIMEDOUT) { + // non-fatel errors + lderr(cct) << "failed to notify update: " << cpp_strerror(r) << dendl; + } else if (r < 0) { lderr(cct) << "failed to notify update: " << cpp_strerror(r) << dendl; finish(r); return; From a888bff8d00e3e496ec80e4273e01a47b67da5dc Mon Sep 17 00:00:00 2001 From: Jason Dillaman Date: Thu, 10 Dec 2020 17:32:16 -0500 Subject: [PATCH 3/4] librbd/mirror: tweak which snapshot is unlinked when at capacity The rbd-mirror daemon will attempt to sync from the last synced snapshot to the next mirror snapshot. When the limit is at 3, this currently can result in a situation where an in-use sync snapshot is deleted. Instead of unlinking the second oldest snapshot, always unlink the third oldest. Fixes: https://tracker.ceph.com/issues/48553 Signed-off-by: Jason Dillaman --- src/librbd/mirror/snapshot/CreatePrimaryRequest.cc | 13 +++++++------ .../snapshot/test_mock_CreatePrimaryRequest.cc | 2 +- src/test/librbd/test_mirroring.cc | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/librbd/mirror/snapshot/CreatePrimaryRequest.cc b/src/librbd/mirror/snapshot/CreatePrimaryRequest.cc index 70962cc6efbc2..ce14df952e981 100644 --- a/src/librbd/mirror/snapshot/CreatePrimaryRequest.cc +++ b/src/librbd/mirror/snapshot/CreatePrimaryRequest.cc @@ -192,7 +192,7 @@ void CreatePrimaryRequest::unlink_peer() { for (auto &peer : m_mirror_peer_uuids) { std::shared_lock image_locker{m_image_ctx->image_lock}; size_t count = 0; - uint64_t prev_snap_id = 0; + uint64_t unlink_snap_id = 0; for (auto &snap_it : m_image_ctx->snap_info) { auto info = boost::get( &snap_it.second.snap_namespace); @@ -202,15 +202,16 @@ void CreatePrimaryRequest::unlink_peer() { if (info->state != cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY) { // reset counters -- we count primary snapshots after the last promotion count = 0; - prev_snap_id = 0; + unlink_snap_id = 0; continue; } count++; - if (count == max_snapshots - 1) { - prev_snap_id = snap_it.first; - } else if (count > max_snapshots) { + if (count == 3) { + unlink_snap_id = snap_it.first; + } + if (count > max_snapshots) { peer_uuid = peer; - snap_id = prev_snap_id; + snap_id = unlink_snap_id; break; } } diff --git a/src/test/librbd/mirror/snapshot/test_mock_CreatePrimaryRequest.cc b/src/test/librbd/mirror/snapshot/test_mock_CreatePrimaryRequest.cc index f38e58d8e8738..2627e33a802ce 100644 --- a/src/test/librbd/mirror/snapshot/test_mock_CreatePrimaryRequest.cc +++ b/src/test/librbd/mirror/snapshot/test_mock_CreatePrimaryRequest.cc @@ -313,7 +313,7 @@ TEST_F(TestMockMirrorSnapshotCreatePrimaryRequest, SuccessUnlinkPeer) { expect_create_snapshot(mock_image_ctx, 0); MockUnlinkPeerRequest mock_unlink_peer_request; auto it = mock_image_ctx.snap_info.rbegin(); - auto snap_id = (++it)->first; + auto snap_id = it->first; expect_unlink_peer(mock_image_ctx, mock_unlink_peer_request, snap_id, "uuid", 0); C_SaferCond ctx; diff --git a/src/test/librbd/test_mirroring.cc b/src/test/librbd/test_mirroring.cc index 75d18208cc702..daab119fe3a49 100644 --- a/src/test/librbd/test_mirroring.cc +++ b/src/test/librbd/test_mirroring.cc @@ -1153,7 +1153,7 @@ TEST_F(TestMirroring, Snapshot) ASSERT_EQ(0, image.snap_list(snaps1)); ASSERT_EQ(3U, snaps1.size()); ASSERT_EQ(snaps1[0].id, snaps[0].id); - ASSERT_EQ(snaps1[1].id, snaps[2].id); + ASSERT_EQ(snaps1[1].id, snaps[1].id); ASSERT_EQ(snaps1[2].id, snap_id); librbd::snap_namespace_type_t snap_ns_type; From fb69efc6c19e6ee2bd8947129fb7f35442acb907 Mon Sep 17 00:00:00 2001 From: Jason Dillaman Date: Thu, 10 Dec 2020 19:31:45 -0500 Subject: [PATCH 4/4] rbd-mirror: validate that remote start snapshot still exists Perform a basic sanity check to verify that the remote start snapshot still exists. This was previosly being deleted as part of the unlink process due to a race condition between the remote side completing a sync between snapshots 1 and 2 and snapshot 2 being unlinked due to reaching max snapshots. Signed-off-by: Jason Dillaman --- .../snapshot/test_mock_Replayer.cc | 85 ++++++++++++++++++- .../image_replayer/snapshot/Replayer.cc | 71 +++++++++------- 2 files changed, 125 insertions(+), 31 deletions(-) diff --git a/src/test/rbd_mirror/image_replayer/snapshot/test_mock_Replayer.cc b/src/test/rbd_mirror/image_replayer/snapshot/test_mock_Replayer.cc index 50a155d241826..ad89a29f384cd 100644 --- a/src/test/rbd_mirror/image_replayer/snapshot/test_mock_Replayer.cc +++ b/src/test/rbd_mirror/image_replayer/snapshot/test_mock_Replayer.cc @@ -786,6 +786,10 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, SyncSnapshot) { expect_is_refresh_required(mock_remote_image_ctx, true); expect_refresh( mock_remote_image_ctx, { + {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, + "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}, {2U, librbd::SnapInfo{"snap2", cls::rbd::UserSnapshotNamespace{}, 0, {}, 0, 0, {}}}, {3U, librbd::SnapInfo{"snap3", cls::rbd::MirrorSnapshotNamespace{ @@ -839,6 +843,10 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, SyncSnapshot) { expect_is_refresh_required(mock_remote_image_ctx, true); expect_refresh( mock_remote_image_ctx, { + {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, + "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}, {2U, librbd::SnapInfo{"snap2", cls::rbd::UserSnapshotNamespace{}, 0, {}, 0, 0, {}}}, {3U, librbd::SnapInfo{"snap3", cls::rbd::MirrorSnapshotNamespace{ @@ -862,7 +870,14 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, SyncSnapshot) { 4, true, 0, {}}, 0, {}, 0, 0, {}}}, }, 0); - expect_is_refresh_required(mock_remote_image_ctx, false); + expect_is_refresh_required(mock_remote_image_ctx, true); + expect_refresh( + mock_remote_image_ctx, { + {4U, librbd::SnapInfo{"snap4", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, + "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}} + }, 0); // fire init C_SaferCond init_ctx; @@ -2010,6 +2025,74 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, SplitBrain) { mock_remote_image_ctx)); } +TEST_F(TestMockImageReplayerSnapshotReplayer, RemoteSnapshotMissingSplitBrain) { + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; + librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + + MockReplayerListener mock_replayer_listener; + expect_notification(mock_threads, mock_replayer_listener); + + InSequence seq; + + MockInstanceWatcher mock_instance_watcher; + MockImageMeta mock_image_meta; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_image_ctx, + mock_image_meta); + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; + m_pool_meta_cache.set_remote_pool_meta( + m_remote_io_ctx.get_id(), + {"remote mirror uuid", "remote mirror peer uuid"}); + + librbd::UpdateWatchCtx* update_watch_ctx = nullptr; + ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx, + mock_replayer_listener, + mock_image_meta, + &update_watch_ctx)); + + // inject a missing remote start snap (deleted) + mock_local_image_ctx.snap_info = { + {11U, librbd::SnapInfo{"snap3", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, + "remote mirror uuid", 1, true, 0, + {{1, CEPH_NOSNAP}}}, + 0, {}, 0, 0, {}}}}; + mock_remote_image_ctx.snap_info = { + {2U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, + "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}, + {3U, librbd::SnapInfo{"snap3", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, + "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}}; + + // split-brain due to missing snapshot 1 + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, false); + expect_is_refresh_required(mock_remote_image_ctx, false); + + // wake-up replayer + update_watch_ctx->handle_notify(); + + // wait for sync to complete and expect replay complete + ASSERT_EQ(0, wait_for_notification(1)); + ASSERT_FALSE(mock_replayer.is_replaying()); + ASSERT_EQ(-EEXIST, mock_replayer.get_error_code()); + ASSERT_EQ(std::string{"split-brain"}, mock_replayer.get_error_description()); + + ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx)); +} + TEST_F(TestMockImageReplayerSnapshotReplayer, RemoteFailover) { librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; diff --git a/src/tools/rbd_mirror/image_replayer/snapshot/Replayer.cc b/src/tools/rbd_mirror/image_replayer/snapshot/Replayer.cc index 1b9d08f0b48d0..dec37ad8b3a91 100644 --- a/src/tools/rbd_mirror/image_replayer/snapshot/Replayer.cc +++ b/src/tools/rbd_mirror/image_replayer/snapshot/Replayer.cc @@ -613,43 +613,54 @@ void Replayer::scan_remote_mirror_snapshots( m_remote_snap_id_end = remote_snap_id; m_remote_mirror_snap_ns = *mirror_ns; } - image_locker.unlock(); - - unlink_snap_ids.erase(m_remote_snap_id_start); - unlink_snap_ids.erase(m_remote_snap_id_end); - if (!unlink_snap_ids.empty()) { - locker->unlock(); - // retry the unlinking process for a remote snapshot that we do not - // need anymore - auto remote_snap_id = *unlink_snap_ids.begin(); - dout(10) << "unlinking from remote snapshot " << remote_snap_id << dendl; - unlink_peer(remote_snap_id); - return; + if (m_remote_snap_id_start != 0 && + remote_image_ctx->snap_info.count(m_remote_snap_id_start) == 0) { + // the remote start snapshot was deleted out from under us + derr << "failed to locate remote start snapshot: " + << "snap_id=" << m_remote_snap_id_start << dendl; + split_brain = true; } - if (m_remote_snap_id_end != CEPH_NOSNAP) { - dout(10) << "found remote mirror snapshot: " - << "remote_snap_id_start=" << m_remote_snap_id_start << ", " - << "remote_snap_id_end=" << m_remote_snap_id_end << ", " - << "remote_snap_ns=" << m_remote_mirror_snap_ns << dendl; - if (m_remote_mirror_snap_ns.complete) { + image_locker.unlock(); + + if (!split_brain) { + unlink_snap_ids.erase(m_remote_snap_id_start); + unlink_snap_ids.erase(m_remote_snap_id_end); + if (!unlink_snap_ids.empty()) { locker->unlock(); - if (m_local_snap_id_end != CEPH_NOSNAP && - !m_local_mirror_snap_ns.complete) { - // attempt to resume image-sync - dout(10) << "local image contains in-progress mirror snapshot" - << dendl; - get_local_image_state(); + // retry the unlinking process for a remote snapshot that we do not + // need anymore + auto remote_snap_id = *unlink_snap_ids.begin(); + dout(10) << "unlinking from remote snapshot " << remote_snap_id << dendl; + unlink_peer(remote_snap_id); + return; + } + + if (m_remote_snap_id_end != CEPH_NOSNAP) { + dout(10) << "found remote mirror snapshot: " + << "remote_snap_id_start=" << m_remote_snap_id_start << ", " + << "remote_snap_id_end=" << m_remote_snap_id_end << ", " + << "remote_snap_ns=" << m_remote_mirror_snap_ns << dendl; + if (m_remote_mirror_snap_ns.complete) { + locker->unlock(); + + if (m_local_snap_id_end != CEPH_NOSNAP && + !m_local_mirror_snap_ns.complete) { + // attempt to resume image-sync + dout(10) << "local image contains in-progress mirror snapshot" + << dendl; + get_local_image_state(); + } else { + copy_snapshots(); + } + return; } else { - copy_snapshots(); + // might have raced with the creation of a remote mirror snapshot + // so we will need to refresh and rescan once it completes + dout(15) << "remote mirror snapshot not complete" << dendl; } - return; - } else { - // might have raced with the creation of a remote mirror snapshot - // so we will need to refresh and rescan once it completes - dout(15) << "remote mirror snapshot not complete" << dendl; } }