OS-6912 bhyve virtio-blk should sync when needed

Reviewed by: Mike Gerdts <mike.gerdts@joyent.com>
Reviewed by: Patrick Mooney <patrick.mooney@joyent.com>
Approved by: Patrick Mooney <patrick.mooney@joyent.com>

4 files changed, 50 insertions, 2 deletions

diff --git a/usr/src/cmd/bhyve/block_if.c b/usr/src/cmd/bhyve/block_if.c
index 86978bbda2..c8ec96690c 100644
--- a/usr/src/cmd/bhyve/block_if.c
+++ b/usr/src/cmd/bhyve/block_if.c
@@ -71,6 +71,9 @@ __FBSDID("$FreeBSD$");
 enum blockop {
 	BOP_READ,
 	BOP_WRITE,
+#ifndef __FreeBSD__
+	BOP_WRITE_SYNC,
+#endif
 	BOP_FLUSH,
 	BOP_DELETE
 };
@@ -143,6 +146,9 @@ blockif_enqueue(struct blockif_ctxt *bc, struct blockif_req *breq,
 	switch (op) {
 	case BOP_READ:
 	case BOP_WRITE:
+#ifndef __FreeBSD__
+	case BOP_WRITE_SYNC:
+#endif
 	case BOP_DELETE:
 		off = breq->br_offset;
 		for (i = 0; i < breq->br_iovcnt; i++)
@@ -215,6 +221,8 @@ blockif_proc(struct blockif_ctxt *bc, struct blockif_elem *be, uint8_t *buf)
 	struct blockif_req *br;
 #ifdef	__FreeBSD__
 	off_t arg[2];
+#else
+	boolean_t sync = B_FALSE;
 #endif
 	ssize_t clen, len, off, boff, voff;
 	int i, err;
@@ -260,6 +268,11 @@ blockif_proc(struct blockif_ctxt *bc, struct blockif_elem *be, uint8_t *buf)
 			br->br_resid -= len;
 		}
 		break;
+#ifndef __FreeBSD__
+	case BOP_WRITE_SYNC:
+		sync = B_TRUE;
+		/* FALLTHROUGH */
+#endif
 	case BOP_WRITE:
 		if (bc->bc_rdonly) {
 			err = EROFS;
@@ -301,13 +314,16 @@ blockif_proc(struct blockif_ctxt *bc, struct blockif_elem *be, uint8_t *buf)
 		}
 		break;
 	case BOP_FLUSH:
-		if (bc->bc_ischr) {
 #ifdef	__FreeBSD__
+		if (bc->bc_ischr) {
 			if (ioctl(bc->bc_fd, DIOCGFLUSH))
 				err = errno;
-#endif
 		} else if (fsync(bc->bc_fd))
 			err = errno;
+#else
+		if (fsync(bc->bc_fd))
+			err = errno;
+#endif
 		break;
 	case BOP_DELETE:
 		if (!bc->bc_candelete)
@@ -332,6 +348,11 @@ blockif_proc(struct blockif_ctxt *bc, struct blockif_elem *be, uint8_t *buf)
 		break;
 	}
 
+#ifndef __FreeBSD__
+	if (sync && err == 0 && fsync(bc->bc_fd) < 0)
+		err = errno;
+#endif
+
 	be->be_status = BST_DONE;
 
 	(*br->br_callback)(br, err);
@@ -641,11 +662,19 @@ blockif_read(struct blockif_ctxt *bc, struct blockif_req *breq)
 }
 
 int
+#ifdef __FreeBSD__
 blockif_write(struct blockif_ctxt *bc, struct blockif_req *breq)
+#else
+blockif_write(struct blockif_ctxt *bc, struct blockif_req *breq, boolean_t sync)
+#endif
 {
 
 	assert(bc->bc_magic == BLOCKIF_SIG);
+#ifdef __FreeBSD__
 	return (blockif_request(bc, breq, BOP_WRITE));
+#else
+	return (blockif_request(bc, breq, sync ? BOP_WRITE_SYNC : BOP_WRITE));
+#endif
 }
 
 int
diff --git a/usr/src/cmd/bhyve/block_if.h b/usr/src/cmd/bhyve/block_if.h
index be6bc9a8cd..930cd62e83 100644
--- a/usr/src/cmd/bhyve/block_if.h
+++ b/usr/src/cmd/bhyve/block_if.h
@@ -67,7 +67,12 @@ int	blockif_queuesz(struct blockif_ctxt *bc);
 int	blockif_is_ro(struct blockif_ctxt *bc);
 int	blockif_candelete(struct blockif_ctxt *bc);
 int	blockif_read(struct blockif_ctxt *bc, struct blockif_req *breq);
+#ifdef __FreeBSD__
 int	blockif_write(struct blockif_ctxt *bc, struct blockif_req *breq);
+#else
+int	blockif_write(struct blockif_ctxt *bc, struct blockif_req *breq,
+    boolean_t sync);
+#endif
 int	blockif_flush(struct blockif_ctxt *bc, struct blockif_req *breq);
 int	blockif_delete(struct blockif_ctxt *bc, struct blockif_req *breq);
 int	blockif_cancel(struct blockif_ctxt *bc, struct blockif_req *breq);
diff --git a/usr/src/cmd/bhyve/pci_ahci.c b/usr/src/cmd/bhyve/pci_ahci.c
index 909d5c19af..ad0da7c669 100644
--- a/usr/src/cmd/bhyve/pci_ahci.c
+++ b/usr/src/cmd/bhyve/pci_ahci.c
@@ -738,7 +738,16 @@ ahci_handle_rw(struct ahci_port *p, int slot, uint8_t *cfis, uint32_t done)
 	if (readop)
 		err = blockif_read(p->bctx, breq);
 	else
+#ifdef __FreeBSD__
 		err = blockif_write(p->bctx, breq);
+#else
+		/*
+		 * XXX: We currently don't handle disabling of the write cache.
+		 * Once this is fixed we need to revisit this code to do sync
+		 * writes when the cache is disabled.
+		 */
+		err = blockif_write(p->bctx, breq, B_FALSE);
+#endif
 	assert(err == 0);
 }
 
diff --git a/usr/src/cmd/bhyve/pci_virtio_block.c b/usr/src/cmd/bhyve/pci_virtio_block.c
index 0c72c1f503..63f59e6468 100644
--- a/usr/src/cmd/bhyve/pci_virtio_block.c
+++ b/usr/src/cmd/bhyve/pci_virtio_block.c
@@ -279,7 +279,12 @@ pci_vtblk_proc(struct pci_vtblk_softc *sc, struct vqueue_info *vq)
 		err = blockif_read(sc->bc, &io->io_req);
 		break;
 	case VBH_OP_WRITE:
+#ifdef __FreeBSD__
 		err = blockif_write(sc->bc, &io->io_req);
+#else
+		err = blockif_write(sc->bc, &io->io_req,
+		    (sc->vbsc_vs.vs_negotiated_caps & VTBLK_F_FLUSH) == 0);
+#endif
 		break;
 	case VBH_OP_FLUSH:
 	case VBH_OP_FLUSH_OUT: