15210 smbtorture lease and oplock failures

Reviewed by: Joyce McIntosh <jmcintosh@racktopsystems.com>
Reviewed by: Andy Stormont <andyjstormont@gmail.com>
Reviewed by: Matt Barden <mbarden@racktopsystems.com>
Approved by: Dan McDonald <danmcd@mnx.io>

5 files changed, 199 insertions, 96 deletions

diff --git a/usr/src/uts/common/fs/smbsrv/smb2_lease.c b/usr/src/uts/common/fs/smbsrv/smb2_lease.c
index 81bf93d8fe..c440fa16c8 100644
--- a/usr/src/uts/common/fs/smbsrv/smb2_lease.c
+++ b/usr/src/uts/common/fs/smbsrv/smb2_lease.c
@@ -301,7 +301,6 @@ smb2_lease_break_ack(smb_request_t *sr)
 	smb_node_t  *node;
 	smb_ofile_t *ofile;
 	uint32_t LeaseState;
-	uint32_t BreakTo;
 	uint32_t status;
 	int rc = 0;
 
@@ -363,7 +362,7 @@ smb2_lease_break_ack(smb_request_t *sr)
 	/* Success, so have sr->fid_ofile */
 	ofile = sr->fid_ofile;
 
-	if (lease->ls_breaking == 0) {
+	if (lease->ls_breaking == B_FALSE) {
 		/*
 		 * This ACK is either unsolicited or too late,
 		 * eg. we timed out the ACK and did it locally.
@@ -376,9 +375,7 @@ smb2_lease_break_ack(smb_request_t *sr)
 	 * If the new LeaseState has any bits in excess of
 	 * the lease state we sent in the break, error...
 	 */
-	BreakTo = (lease->ls_breaking >> BREAK_SHIFT) &
-	    OPLOCK_LEVEL_CACHE_MASK;
-	if ((LeaseState & ~BreakTo) != 0) {
+	if ((LeaseState & ~(lease->ls_breakto)) != 0) {
 		status = NT_STATUS_REQUEST_NOT_ACCEPTED;
 		goto errout;
 	}
@@ -388,16 +385,18 @@ smb2_lease_break_ack(smb_request_t *sr)
 	 *
 	 * Clear breaking flags before we ack,
 	 * because ack might set those.
+	 * Signal both CVs, out of paranoia.
 	 */
-	ofile->f_oplock.og_breaking = 0;
-	lease->ls_breaking = 0;
+	ofile->f_oplock.og_breaking = B_FALSE;
+	cv_broadcast(&ofile->f_oplock.og_ack_cv);
+	lease->ls_breaking = B_FALSE;
 	cv_broadcast(&lease->ls_ack_cv);
 
 	LeaseState |= OPLOCK_LEVEL_GRANULAR;
 	status = smb_oplock_ack_break(sr, ofile, &LeaseState);
 
 	ofile->f_oplock.og_state = LeaseState;
-	lease->ls_state = LeaseState & CACHE_RWH;
+	lease->ls_state = LeaseState;
 	/* ls_epoch does not change here */
 
 	if (ofile->dh_persist)
@@ -676,7 +675,7 @@ smb2_lease_send_break(smb_request_t *sr)
 		 * We're expecting an ACK.  Wait in this thread
 		 * so we can log clients that don't respond.
 		 */
-		status = smb_oplock_wait_ack(sr);
+		status = smb_oplock_wait_ack(sr, NewLevel);
 		if (status == 0)
 			return;
 
@@ -718,14 +717,14 @@ smb2_lease_send_break(smb_request_t *sr)
 	/*
 	 * Now continue like the non-lease code
 	 */
-	ofile->f_oplock.og_breaking = 0;
-	lease->ls_breaking = 0;
+	ofile->f_oplock.og_breaking = B_FALSE;
+	lease->ls_breaking = B_FALSE;
 	cv_broadcast(&lease->ls_ack_cv);
 
 	status = smb_oplock_ack_break(sr, ofile, &NewLevel);
 
 	ofile->f_oplock.og_state = NewLevel;
-	lease->ls_state = NewLevel & CACHE_RWH;
+	lease->ls_state = NewLevel;
 	/* ls_epoch does not change here */
 
 	if (ofile->dh_persist)
@@ -748,7 +747,7 @@ unlock_out:
  * Convert SMB2 lease request info in to internal form,
  * call common oplock code, convert result to SMB2.
  *
- * If necessary, "go async" here.
+ * If necessary, "go async" here (at the end).
  */
 void
 smb2_lease_acquire(smb_request_t *sr)
@@ -807,15 +806,18 @@ smb2_lease_acquire(smb_request_t *sr)
 	mutex_enter(&node->n_oplock.ol_mutex);
 
 	/*
-	 * Disallow downgrade
+	 * MS-SMB2 3.3.5.9.8 and 3.3.5.9.11 Lease (V2) create contexts
 	 *
-	 * Note that open with a lease is not allowed to turn off
-	 * any cache rights.  If the client tries to "downgrade",
-	 * any bits, just return the existing lease cache bits.
+	 * If the caching state requested in LeaseState of the (create ctx)
+	 * is not a superset of Lease.LeaseState or if Lease.Breaking is TRUE,
+	 * the server MUST NOT promote Lease.LeaseState. If the lease state
+	 * requested is a superset of Lease.LeaseState and Lease.Breaking is
+	 * FALSE, the server MUST request promotion of the lease state from
+	 * the underlying object store to the new caching state.
 	 */
 	have = lease->ls_state & CACHE_RWH;
 	want = op->op_oplock_state & CACHE_RWH;
-	if ((have & ~want) != 0) {
+	if ((have & ~want) != 0 || lease->ls_breaking) {
 		op->op_oplock_state = have |
 		    OPLOCK_LEVEL_GRANULAR;
 		goto done;
@@ -840,9 +842,7 @@ smb2_lease_acquire(smb_request_t *sr)
 	 * Try exclusive (request is RW or RWH)
 	 */
 	if ((op->op_oplock_state & WRITE_CACHING) != 0) {
-		want = op->op_oplock_state & CACHE_RWH;
-		if (have == want)
-			goto done;
+		/* Alread checked (want & ~have) */
 
 		status = smb_oplock_request_LH(sr, ofile,
 		    &op->op_oplock_state);
@@ -881,7 +881,7 @@ smb2_lease_acquire(smb_request_t *sr)
 	 */
 	if ((op->op_oplock_state & HANDLE_CACHING) != 0) {
 		want = op->op_oplock_state & CACHE_RWH;
-		if (have == want)
+		if ((want & ~have) == 0)
 			goto done;
 
 		status = smb_oplock_request_LH(sr, ofile,
@@ -907,7 +907,7 @@ smb2_lease_acquire(smb_request_t *sr)
 	 */
 	if ((op->op_oplock_state & READ_CACHING) != 0) {
 		want = op->op_oplock_state & CACHE_RWH;
-		if (have == want)
+		if ((want & ~have) == 0)
 			goto done;
 
 		status = smb_oplock_request_LH(sr, ofile,
@@ -945,11 +945,13 @@ done:
 	 * this has to be using granular oplocks.
 	 */
 	if (NewGrant) {
-		ofile->f_oplock.og_state = op->op_oplock_state;
-		ofile->f_oplock.og_breaking = 0;
+		ofile->f_oplock.og_state   = op->op_oplock_state;
+		ofile->f_oplock.og_breakto = op->op_oplock_state;
+		ofile->f_oplock.og_breaking = B_FALSE;
 
 		lease->ls_oplock_ofile = ofile;
-		lease->ls_state = op->op_oplock_state;
+		lease->ls_state   = ofile->f_oplock.og_state;
+		lease->ls_breakto = ofile->f_oplock.og_breakto;
 		lease->ls_breaking = B_FALSE;
 		lease->ls_epoch++;
 
@@ -988,6 +990,11 @@ done:
  * This ofile has a lease and is about to close.
  * Called by smb_ofile_close when there's a lease.
  *
+ * Note that a client may close an ofile in response to an
+ * oplock break or lease break intead of doing an Ack break,
+ * so this must wake anything that might be waiting on an ack
+ * when the last close of a lease happens.
+ *
  * With leases, just one ofile on a lease owns the oplock.
  * If an ofile with a lease is closed and it's the one that
  * owns the oplock, try to move the oplock to another ofile
@@ -1009,6 +1016,12 @@ smb2_lease_ofile_close(smb_ofile_t *ofile)
 	ASSERT(RW_READ_HELD(&node->n_ofile_list.ll_lock));
 	ASSERT(MUTEX_HELD(&node->n_oplock.ol_mutex));
 
+#ifdef	DEBUG
+	FOREACH_NODE_OFILE(node, o) {
+		DTRACE_PROBE1(each_ofile, smb_ofile_t *, o);
+	}
+#endif
+
 	/*
 	 * If this ofile was not the oplock owner for this lease,
 	 * we can leave things as they are.
@@ -1077,7 +1090,8 @@ smb2_lease_ofile_close(smb_ofile_t *ofile)
 	 * Wakeup ACK waiters too.
 	 */
 	lease->ls_state = 0;
-	lease->ls_breaking = 0;
+	lease->ls_breakto = 0;
+	lease->ls_breaking = B_FALSE;
 	cv_broadcast(&lease->ls_ack_cv);
 
 	lease->ls_oplock_ofile = NULL;
diff --git a/usr/src/uts/common/fs/smbsrv/smb2_oplock.c b/usr/src/uts/common/fs/smbsrv/smb2_oplock.c
index 613b56dac9..0d807870bc 100644
--- a/usr/src/uts/common/fs/smbsrv/smb2_oplock.c
+++ b/usr/src/uts/common/fs/smbsrv/smb2_oplock.c
@@ -133,7 +133,7 @@ smb2_oplock_break_ack(smb_request_t *sr)
 	smb_llist_enter(&node->n_ofile_list, RW_READER);
 	mutex_enter(&node->n_oplock.ol_mutex);
 
-	if (og->og_breaking == 0) {
+	if (og->og_breaking == B_FALSE) {
 		/*
 		 * This is an unsolicited Ack. (There is no
 		 * outstanding oplock break in progress now.)
@@ -154,7 +154,7 @@ smb2_oplock_break_ack(smb_request_t *sr)
 	 * Clear breaking flags before we ack,
 	 * because ack might set those.
 	 */
-	ofile->f_oplock.og_breaking = 0;
+	ofile->f_oplock.og_breaking = B_FALSE;
 	cv_broadcast(&ofile->f_oplock.og_ack_cv);
 
 	status = smb_oplock_ack_break(sr, ofile, &NewLevel);
@@ -355,7 +355,7 @@ smb2_oplock_send_break(smb_request_t *sr)
 		 * We're expecting an ACK.  Wait in this thread
 		 * so we can log clients that don't respond.
 		 */
-		status = smb_oplock_wait_ack(sr);
+		status = smb_oplock_wait_ack(sr, NewLevel);
 		if (status == 0)
 			return;
 
@@ -377,7 +377,7 @@ smb2_oplock_send_break(smb_request_t *sr)
 	smb_llist_enter(&node->n_ofile_list, RW_READER);
 	mutex_enter(&node->n_oplock.ol_mutex);
 
-	ofile->f_oplock.og_breaking = 0;
+	ofile->f_oplock.og_breaking = B_FALSE;
 	cv_broadcast(&ofile->f_oplock.og_ack_cv);
 
 	status = smb_oplock_ack_break(sr, ofile, &NewLevel);
@@ -487,8 +487,9 @@ smb2_oplock_acquire(smb_request_t *sr)
 	case NT_STATUS_SUCCESS:
 	case NT_STATUS_OPLOCK_BREAK_IN_PROGRESS:
 		ofile->f_oplock.og_dialect = SMB_VERS_2_002;
-		ofile->f_oplock.og_state = op->op_oplock_state;
-		ofile->f_oplock.og_breaking = 0;
+		ofile->f_oplock.og_state   = op->op_oplock_state;
+		ofile->f_oplock.og_breakto = op->op_oplock_state;
+		ofile->f_oplock.og_breaking = B_FALSE;
 		if (ofile->dh_persist) {
 			smb2_dh_update_oplock(sr, ofile);
 		}
diff --git a/usr/src/uts/common/fs/smbsrv/smb_cmn_oplock.c b/usr/src/uts/common/fs/smbsrv/smb_cmn_oplock.c
index 441dc59a6e..840ab49367 100644
--- a/usr/src/uts/common/fs/smbsrv/smb_cmn_oplock.c
+++ b/usr/src/uts/common/fs/smbsrv/smb_cmn_oplock.c
@@ -563,6 +563,12 @@ smb_oplock_req_excl(
 	ASSERT(RW_READ_HELD(&node->n_ofile_list.ll_lock));
 	ASSERT(MUTEX_HELD(&node->n_oplock.ol_mutex));
 
+#ifdef	DEBUG
+	FOREACH_NODE_OFILE(node, o) {
+		DTRACE_PROBE1(each_ofile, smb_ofile_t *, o);
+	}
+#endif
+
 	/*
 	 * Don't allow grants on closing ofiles.
 	 */
@@ -1063,6 +1069,12 @@ smb_oplock_req_shared(
 	ASSERT(RW_READ_HELD(&node->n_ofile_list.ll_lock));
 	ASSERT(MUTEX_HELD(&node->n_oplock.ol_mutex));
 
+#ifdef	DEBUG
+	FOREACH_NODE_OFILE(node, o) {
+		DTRACE_PROBE1(each_ofile, smb_ofile_t *, o);
+	}
+#endif
+
 	/*
 	 * Don't allow grants on closing ofiles.
 	 */
@@ -1601,21 +1613,10 @@ smb_oplock_ack_break(
 			 *	OplockCompletionStatus = STATUS_SUCCESS.
 			 * (Because BreakingOplockOpen is equal to the
 			 * passed-in Open, the operation ends at this point.)
-			 *
-			 * It should be OK to return the reduced oplock
-			 * (*rop = LEVEL_NONE) here and avoid the need
-			 * to send another oplock break.  This is safe
-			 * because we already have an Ack of the break
-			 * to Level_II, and the additional break to none
-			 * would use AckRequired = FALSE.
-			 *
-			 * If we followed the spec here, we'd have:
-			 * smb_oplock_ind_break(ofile,
-			 *    LEVEL_NONE, B_FALSE,
-			 *    NT_STATUS_SUCCESS);
-			 * (Or smb_oplock_ind_break_in_ack...)
 			 */
-			*rop = LEVEL_NONE;	/* Reduced from L2 */
+			smb_oplock_ind_break_in_ack(
+			    sr, ofile,
+			    LEVEL_NONE, B_FALSE);
 		}
 		status = NT_STATUS_SUCCESS;
 		goto out;
@@ -1691,9 +1692,19 @@ smb_oplock_ack_break(
 		BreakToLevel = READ_CACHING;
 		break;
 	case BREAK_TO_NO_CACHING:
-	default:
 		BreakToLevel = LEVEL_NONE;
 		break;
+	default:
+		ASSERT(0);
+		/* FALLTHROUGH */
+	case 0:
+		/*
+		 * This can happen when we have multiple RH opens,
+		 * and one of them breaks (RH to R).  Happens in
+		 * the smbtorture smb2.lease.v2rename test.
+		 */
+		BreakToLevel = CACHE_R;
+		break;
 	}
 
 	/* Switch Open.Stream.Oplock.State */
@@ -1704,6 +1715,21 @@ smb_oplock_ack_break(
 	case (READ_CACHING|HANDLE_CACHING|BREAK_TO_READ_CACHING):
 	case (READ_CACHING|HANDLE_CACHING|BREAK_TO_NO_CACHING):
 		/*
+		 * XXX: Missing from [MS-FSA]
+		 *
+		 * If we previously sent a break to none and the
+		 * client Ack level is R instead of none, we
+		 * need to send another break. We can then
+		 * proceed as if we got level = none.
+		 */
+		if (level == CACHE_R && BreakToLevel == LEVEL_NONE) {
+			smb_oplock_ind_break_in_ack(
+			    sr, ofile,
+			    LEVEL_NONE, B_FALSE);
+			level = LEVEL_NONE;
+		}
+
+		/*
 		 * For each RHOpContext ThisContext in
 		 * Open.Stream.Oplock.RHBreakQueue:
 		 *	If ThisContext.Open equals Open:
@@ -2001,7 +2027,11 @@ smb_oplock_ack_break(
 		 */
 
 		/*
-		 * Breaking R to none?  This is like:
+		 * Breaking R to none.
+		 *
+		 * We sent break exclusive (RWH or RW) to none and
+		 * the client Ack reduces to R instead of to none.
+		 * Need to send another break. This is like:
 		 * "If BreakCacheLevel contains READ_CACHING..."
 		 * from smb_oplock_break_cmn.
 		 */
@@ -2015,16 +2045,24 @@ smb_oplock_ack_break(
 		}
 
 		/*
-		 * Breaking RH to R or RH to none?  This is like:
+		 * Breaking RH to R or RH to none.
+		 *
+		 * We sent break from (RWH or RW) to (R or none),
+		 * and the client Ack reduces to RH instead of none.
+		 * Need to send another break. This is like:
 		 * "If BreakCacheLevel equals HANDLE_CACHING..."
 		 * from smb_oplock_break_cmn.
+		 *
+		 * Note: Windows always does break to CACHE_R here,
+		 * letting another Ack and ind_break round trip
+		 * take us the rest of the way from R to none.
 		 */
 		if (level == CACHE_RH &&
 		    (BreakToLevel == CACHE_R ||
 		    BreakToLevel == LEVEL_NONE)) {
 			smb_oplock_ind_break_in_ack(
 			    sr, ofile,
-			    BreakToLevel, B_TRUE);
+			    CACHE_R, B_TRUE);
 
 			ofile->f_oplock.BreakingToRead =
 			    (BreakToLevel & READ_CACHING) ? 1: 0;
@@ -2331,6 +2369,12 @@ smb_oplock_break_CLOSE(smb_node_t *node, smb_ofile_t *ofile)
 	ASSERT(RW_READ_HELD(&node->n_ofile_list.ll_lock));
 	ASSERT(MUTEX_HELD(&node->n_oplock.ol_mutex));
 
+#ifdef	DEBUG
+	FOREACH_NODE_OFILE(node, o) {
+		DTRACE_PROBE1(each_ofile, smb_ofile_t *, o);
+	}
+#endif
+
 	/*
 	 * If Oplock.IIOplocks is not empty:
 	 *   For each Open ThisOpen in Oplock.IIOplocks:
@@ -2701,6 +2745,12 @@ smb_oplock_break_cmn(smb_node_t *node,
 	smb_llist_enter(&node->n_ofile_list, RW_READER);
 	mutex_enter(&node->n_oplock.ol_mutex);
 
+#ifdef	DEBUG
+	FOREACH_NODE_OFILE(node, o) {
+		DTRACE_PROBE1(each_ofile, smb_ofile_t *, o);
+	}
+#endif
+
 	if (node->n_oplock.ol_state == 0 ||
 	    node->n_oplock.ol_state == NO_OPLOCK)
 		goto out;
diff --git a/usr/src/uts/common/fs/smbsrv/smb_oplock.c b/usr/src/uts/common/fs/smbsrv/smb_oplock.c
index a3fd103fa1..58447a98f3 100644
--- a/usr/src/uts/common/fs/smbsrv/smb_oplock.c
+++ b/usr/src/uts/common/fs/smbsrv/smb_oplock.c
@@ -55,7 +55,7 @@ smb1_oplock_ack_break(smb_request_t *sr, uchar_t oplock_level)
 	smb_llist_enter(&node->n_ofile_list, RW_READER);
 	mutex_enter(&node->n_oplock.ol_mutex);
 
-	ofile->f_oplock.og_breaking = 0;
+	ofile->f_oplock.og_breaking = B_FALSE;
 	cv_broadcast(&ofile->f_oplock.og_ack_cv);
 
 	(void) smb_oplock_ack_break(sr, ofile, &NewLevel);
@@ -189,7 +189,7 @@ smb1_oplock_send_break(smb_request_t *sr)
 	 * We're expecting an ACK.  Wait in this thread
 	 * so we can log clients that don't respond.
 	 */
-	status = smb_oplock_wait_ack(sr);
+	status = smb_oplock_wait_ack(sr, NewLevel);
 	if (status == 0)
 		return;
 
@@ -207,7 +207,7 @@ smb1_oplock_send_break(smb_request_t *sr)
 	smb_llist_enter(&node->n_ofile_list, RW_READER);
 	mutex_enter(&node->n_oplock.ol_mutex);
 
-	ofile->f_oplock.og_breaking = 0;
+	ofile->f_oplock.og_breaking = B_FALSE;
 	cv_broadcast(&ofile->f_oplock.og_ack_cv);
 
 	status = smb_oplock_ack_break(sr, ofile, &NewLevel);
@@ -313,8 +313,9 @@ smb1_oplock_acquire(smb_request_t *sr, boolean_t level2ok)
 	case NT_STATUS_OPLOCK_BREAK_IN_PROGRESS:
 		ofile->f_oplock.og_dialect = (level2ok) ?
 		    NT_LM_0_12 : LANMAN2_1;
-		ofile->f_oplock.og_state = op->op_oplock_state;
-		ofile->f_oplock.og_breaking = 0;
+		ofile->f_oplock.og_state   = op->op_oplock_state;
+		ofile->f_oplock.og_breakto = op->op_oplock_state;
+		ofile->f_oplock.og_breaking = B_FALSE;
 		break;
 	case NT_STATUS_OPLOCK_NOT_GRANTED:
 		op->op_oplock_level = SMB_OPLOCK_NONE;
diff --git a/usr/src/uts/common/fs/smbsrv/smb_srv_oplock.c b/usr/src/uts/common/fs/smbsrv/smb_srv_oplock.c
index df917c3d0c..fbb2b51c24 100644
--- a/usr/src/uts/common/fs/smbsrv/smb_srv_oplock.c
+++ b/usr/src/uts/common/fs/smbsrv/smb_srv_oplock.c
@@ -227,9 +227,18 @@ smb_oplock_ind_break_in_ack(smb_request_t *ack_sr, smb_ofile_t *ofile,
 		sr->uid_user = ofile->f_user;
 		smb_user_hold_internal(sr->uid_user);
 	}
+	if (ofile->f_lease != NULL)
+		NewLevel |= OPLOCK_LEVEL_GRANULAR;
+
 	sr->arg.olbrk.NewLevel = NewLevel;
 	sr->arg.olbrk.AckRequired = AckRequired;
 
+	/*
+	 * Could do this in _hdl_update but this way it's
+	 * visible in the dtrace fbt entry probe.
+	 */
+	sr->arg.olbrk.OldLevel = ofile->f_oplock.og_breakto;
+
 	smb_oplock_hdl_update(sr);
 
 	if (use_postwork) {
@@ -350,14 +359,23 @@ smb_oplock_ind_break(smb_ofile_t *ofile, uint32_t NewLevel,
 		sr->uid_user = ofile->f_user;
 		smb_user_hold_internal(sr->uid_user);
 	}
+	if (ofile->f_lease != NULL)
+		NewLevel |= OPLOCK_LEVEL_GRANULAR;
+
 	sr->arg.olbrk.NewLevel = NewLevel;
 	sr->arg.olbrk.AckRequired = AckRequired;
 	sr->smb2_status = CompletionStatus;
 
+	/*
+	 * Could do this in _hdl_update but this way it's
+	 * visible in the dtrace fbt entry probe.
+	 */
+	sr->arg.olbrk.OldLevel = ofile->f_oplock.og_breakto;
+
 	smb_oplock_hdl_update(sr);
 
-	(void) taskq_dispatch(
-	    sv->sv_worker_pool,
+	/* Will call smb_oplock_send_break */
+	(void) taskq_dispatch(sv->sv_worker_pool,
 	    smb_oplock_async_break, sr, TQ_SLEEP);
 }
 
@@ -462,7 +480,8 @@ smb_oplock_hdl_moved(smb_ofile_t *ofile)
 		ls->ls_oplock_ofile = (smb_ofile_t *)&invalid_ofile;
 
 	ofile->f_oplock.og_state = 0;
-	ofile->f_oplock.og_breaking = 0;
+	ofile->f_oplock.og_breakto = 0;
+	ofile->f_oplock.og_breaking = B_FALSE;
 }
 
 /*
@@ -488,7 +507,8 @@ smb_oplock_hdl_closed(smb_ofile_t *ofile)
 		}
 	}
 	ofile->f_oplock.og_state = 0;
-	ofile->f_oplock.og_breaking = 0;
+	ofile->f_oplock.og_breakto = 0;
+	ofile->f_oplock.og_breaking = B_FALSE;
 }
 
 /*
@@ -499,8 +519,16 @@ smb_oplock_hdl_closed(smb_ofile_t *ofile)
  * so we can make any state changes that should happen immediately.
  *
  * Here, keep track of what we will send to the client.
- * More importantly, if a break ack is expected, set
- * the og_breaking flags to note that fact.
+ * Saves old state in arg.olbck.OldLevel
+ *
+ * Note that because we may be in the midst of processing an
+ * smb_oplock_ack_break call here, the _breaking flag will be
+ * temporarily false, and is set true again if this ack causes
+ * another break.  This makes it tricky to know when to update
+ * the epoch, which is not supposed to increment when there's
+ * already an unacknowledged break out to the client.
+ * We can recognize that by comparing ls_state vs ls_breakto.
+ * If no unacknowledged break, ls_state == ls_breakto.
  */
 static void
 smb_oplock_hdl_update(smb_request_t *sr)
@@ -516,36 +544,28 @@ smb_oplock_hdl_update(smb_request_t *sr)
 	ASSERT(MUTEX_HELD(&node->n_oplock.ol_mutex));
 #endif
 
+	/* Caller sets arg.olbrk.OldLevel */
+	ofile->f_oplock.og_breakto = NewLevel;
+	ofile->f_oplock.og_breaking = B_TRUE;
 	if (lease != NULL) {
-		sr->arg.olbrk.OldLevel = lease->ls_state;
-		lease->ls_epoch++;
-		NewLevel |= OPLOCK_LEVEL_GRANULAR;
-	}
+		// If no unacknowledged break, update epoch.
+		if (lease->ls_breakto == lease->ls_state)
+			lease->ls_epoch++;
 
-	if (AckReq) {
-		uint32_t BreakTo;
+		lease->ls_breakto = NewLevel;
+		lease->ls_breaking = B_TRUE;
+	}
 
-		if (lease != NULL) {
-			BreakTo = (NewLevel & CACHE_RWH) << BREAK_SHIFT;
-			if (BreakTo == 0)
-				BreakTo = BREAK_TO_NO_CACHING;
-			lease->ls_breaking = BreakTo;
-		} else {
-			if ((NewLevel & LEVEL_TWO_OPLOCK) != 0)
-				BreakTo = BREAK_TO_TWO;
-			else
-				BreakTo = BREAK_TO_NONE;
-		}
-		ofile->f_oplock.og_breaking = BreakTo;
-		/* Will update ls/og_state in ack. */
-	} else {
+	if (!AckReq) {
 		/*
 		 * Not expecting an Ack from the client.
 		 * Update state immediately.
 		 */
 		ofile->f_oplock.og_state = NewLevel;
+		ofile->f_oplock.og_breaking = B_FALSE;
 		if (lease != NULL) {
-			lease->ls_state = NewLevel & CACHE_RWH;
+			lease->ls_state = NewLevel;
+			lease->ls_breaking = B_FALSE;
 		}
 		if (ofile->dh_persist) {
 			smb2_dh_update_oplock(sr, ofile);
@@ -577,7 +597,8 @@ smb_oplock_close(smb_ofile_t *ofile)
 		smb_oplock_break_CLOSE(node, ofile);
 
 		ofile->f_oplock.og_state = 0;
-		ofile->f_oplock.og_breaking = 0;
+		ofile->f_oplock.og_breakto = 0;
+		ofile->f_oplock.og_breaking = B_FALSE;
 		cv_broadcast(&ofile->f_oplock.og_ack_cv);
 	}
 
@@ -605,34 +626,49 @@ smb_oplock_wait_ack_cancel(smb_request_t *sr)
  * Wait for an oplock break ACK to arrive.  This is called after
  * we've sent an oplock break or lease break to the client where
  * an "Ack break" is expected back.  If we get an Ack, that will
- * wake us up via smb2_oplock_break_ack or smb2_lease_break_ack,
- * which signal the CV on which we wait here.
+ * wake us up via smb2_oplock_break_ack or smb2_lease_break_ack.
+ *
+ * Wait until state reduced to NewLevel (or less).
+ * Note that in multi-break cases, we might wait here for just
+ * one ack when another has become pending, in which case the
+ * og_breakto might be a subset of NewLevel.  Wait until the
+ * state field is no longer a superset of NewLevel.
  */
 uint32_t
-smb_oplock_wait_ack(smb_request_t *sr)
+smb_oplock_wait_ack(smb_request_t *sr, uint32_t NewLevel)
 {
 	smb_ofile_t	*ofile = sr->fid_ofile;
 	smb_lease_t	*lease = ofile->f_lease;
 	smb_node_t	*node = ofile->f_node;
 	smb_oplock_t	*ol = &node->n_oplock;
-	uint32_t	*brp;
-	kcondvar_t	*cvp;
+	uint32_t	*state_p;
+	kcondvar_t	*cv_p;
 	clock_t		time, rv;
 	uint32_t	status = 0;
 	smb_req_state_t  srstate;
+	uint32_t	wait_mask;
 
 	time = ddi_get_lbolt() +
 	    MSEC_TO_TICK(smb_oplock_timeout_ack);
 
+	/*
+	 * Wait on either lease state or oplock state
+	 */
 	if (lease != NULL) {
-		brp = &lease->ls_breaking;
-		cvp = &lease->ls_ack_cv;
+		state_p = &lease->ls_state;
+		cv_p = &lease->ls_ack_cv;
 	} else {
-		brp = &ofile->f_oplock.og_breaking;
-		cvp = &ofile->f_oplock.og_ack_cv;
+		state_p = &ofile->f_oplock.og_state;
+		cv_p = &ofile->f_oplock.og_ack_cv;
 	}
 
 	/*
+	 * These are all the bits that we wait to be cleared.
+	 */
+	wait_mask = ~NewLevel & (CACHE_RWH |
+	    LEVEL_TWO | LEVEL_ONE | LEVEL_BATCH);
+
+	/*
 	 * Setup cancellation callback
 	 */
 	mutex_enter(&sr->sr_mutex);
@@ -642,15 +678,16 @@ smb_oplock_wait_ack(smb_request_t *sr)
 	}
 	sr->sr_state = SMB_REQ_STATE_WAITING_OLBRK;
 	sr->cancel_method = smb_oplock_wait_ack_cancel;
-	sr->cancel_arg2 = cvp;
+	sr->cancel_arg2 = cv_p;
 	mutex_exit(&sr->sr_mutex);
 
 	/*
 	 * Enter the wait loop
 	 */
 	mutex_enter(&ol->ol_mutex);
-	while (*brp != 0) {
-		rv = cv_timedwait(cvp, &ol->ol_mutex, time);
+
+	while ((*state_p & wait_mask) != 0) {
+		rv = cv_timedwait(cv_p, &ol->ol_mutex, time);
 		if (rv < 0) {
 			/* cv_timewait timeout */
 			status = NT_STATUS_CANNOT_BREAK_OPLOCK;