From 5a22db8968a69bec835d1ed9a96ab3381719e0c0 Mon Sep 17 00:00:00 2001 From: Lars Ellenberg Date: Fri, 17 Dec 2010 21:14:23 +0100 Subject: [PATCH] drbd: serialize sending of resync uuid with pending w_send_oos To improve the latency of IO requests during bitmap exchange, we recently allowed writes while waiting for the bitmap, sending "set out-of-sync" information packets for any newly dirtied bits. We have to make sure that the new resync-uuid does not overtake these "set oos" packets. Once the resync-uuid is received, the sync target starts the resync process, and expects the bitmap to only be cleared, not re-set. If we use this protocol extension, we queue the generation and sending of the resync-uuid on the worker, which naturally serializes with all previously queued packets. Signed-off-by: Philipp Reisner Signed-off-by: Lars Ellenberg --- drivers/block/drbd/drbd_int.h | 2 +- drivers/block/drbd/drbd_main.c | 22 +++++++++++++++++++--- drivers/block/drbd/drbd_worker.c | 27 ++++++++++++++------------- 3 files changed, 34 insertions(+), 17 deletions(-) diff --git a/drivers/block/drbd/drbd_int.h b/drivers/block/drbd/drbd_int.h index 5a2d0ec72b34..ec06e744be42 100644 --- a/drivers/block/drbd/drbd_int.h +++ b/drivers/block/drbd/drbd_int.h @@ -1204,7 +1204,7 @@ extern int drbd_send(struct drbd_conf *mdev, struct socket *sock, extern int drbd_send_protocol(struct drbd_conf *mdev); extern int drbd_send_uuids(struct drbd_conf *mdev); extern int drbd_send_uuids_skip_initial_sync(struct drbd_conf *mdev); -extern int drbd_send_sync_uuid(struct drbd_conf *mdev, u64 val); +extern int drbd_gen_and_send_sync_uuid(struct drbd_conf *mdev); extern int drbd_send_sizes(struct drbd_conf *mdev, int trigger_reply, enum dds_flags flags); extern int _drbd_send_state(struct drbd_conf *mdev); extern int drbd_send_state(struct drbd_conf *mdev); diff --git a/drivers/block/drbd/drbd_main.c b/drivers/block/drbd/drbd_main.c index 4da6f11cc82e..2190064d59bd 100644 --- a/drivers/block/drbd/drbd_main.c +++ b/drivers/block/drbd/drbd_main.c @@ -1387,6 +1387,17 @@ static void after_state_ch(struct drbd_conf *mdev, union drbd_state os, spin_unlock_irq(&mdev->req_lock); } + /* Became sync source. With protocol >= 96, we still need to send out + * the sync uuid now. Need to do that before any drbd_send_state, or + * the other side may go "paused sync" before receiving the sync uuids, + * which is unexpected. */ + if ((os.conn != C_SYNC_SOURCE && os.conn != C_PAUSED_SYNC_S) && + (ns.conn == C_SYNC_SOURCE || ns.conn == C_PAUSED_SYNC_S) && + mdev->agreed_pro_version >= 96 && get_ldev(mdev)) { + drbd_gen_and_send_sync_uuid(mdev); + put_ldev(mdev); + } + /* Do not change the order of the if above and the two below... */ if (os.pdsk == D_DISKLESS && ns.pdsk > D_DISKLESS) { /* attach on the peer */ drbd_send_uuids(mdev); @@ -1980,12 +1991,17 @@ int drbd_send_uuids_skip_initial_sync(struct drbd_conf *mdev) return _drbd_send_uuids(mdev, 8); } - -int drbd_send_sync_uuid(struct drbd_conf *mdev, u64 val) +int drbd_gen_and_send_sync_uuid(struct drbd_conf *mdev) { struct p_rs_uuid p; + u64 uuid; + + D_ASSERT(mdev->state.disk == D_UP_TO_DATE); - p.uuid = cpu_to_be64(val); + get_random_bytes(&uuid, sizeof(u64)); + drbd_uuid_set(mdev, UI_BITMAP, uuid); + drbd_md_sync(mdev); + p.uuid = cpu_to_be64(uuid); return drbd_send_cmd(mdev, USE_DATA_SOCKET, P_SYNC_UUID, (struct p_header80 *)&p, sizeof(p)); diff --git a/drivers/block/drbd/drbd_worker.c b/drivers/block/drbd/drbd_worker.c index 2374454cdf17..3df37e65c118 100644 --- a/drivers/block/drbd/drbd_worker.c +++ b/drivers/block/drbd/drbd_worker.c @@ -1516,18 +1516,6 @@ void drbd_start_resync(struct drbd_conf *mdev, enum drbd_conns side) return; } - if (side == C_SYNC_TARGET) { - mdev->bm_resync_fo = 0; - } else /* side == C_SYNC_SOURCE */ { - u64 uuid; - - get_random_bytes(&uuid, sizeof(u64)); - drbd_uuid_set(mdev, UI_BITMAP, uuid); - drbd_send_sync_uuid(mdev, uuid); - - D_ASSERT(mdev->state.disk == D_UP_TO_DATE); - } - write_lock_irq(&global_state_lock); ns = mdev->state; @@ -1565,7 +1553,19 @@ void drbd_start_resync(struct drbd_conf *mdev, enum drbd_conns side) _drbd_pause_after(mdev); } write_unlock_irq(&global_state_lock); - put_ldev(mdev); + + if (side == C_SYNC_TARGET) + mdev->bm_resync_fo = 0; + + /* Since protocol 96, we must serialize drbd_gen_and_send_sync_uuid + * with w_send_oos, or the sync target will get confused as to + * how much bits to resync. We cannot do that always, because for an + * empty resync and protocol < 95, we need to do it here, as we call + * drbd_resync_finished from here in that case. + * We drbd_gen_and_send_sync_uuid here for protocol < 96, + * and from after_state_ch otherwise. */ + if (side == C_SYNC_SOURCE && mdev->agreed_pro_version < 96) + drbd_gen_and_send_sync_uuid(mdev); if (r == SS_SUCCESS) { dev_info(DEV, "Began resync as %s (will sync %lu KB [%lu bits set]).\n", @@ -1601,6 +1601,7 @@ void drbd_start_resync(struct drbd_conf *mdev, enum drbd_conns side) drbd_md_sync(mdev); } + put_ldev(mdev); drbd_state_unlock(mdev); } -- 2.34.1