path: root/usr/src/uts/common/xen/io/blk_common.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */


#include <sys/errno.h>
#include <sys/types.h>
#include <sys/conf.h>
#include <sys/kmem.h>
#include <sys/ddi.h>
#include <sys/stat.h>
#include <sys/sunddi.h>
#include <sys/file.h>
#include <sys/open.h>
#include <sys/modctl.h>
#include <sys/ddi_impldefs.h>
#include <sys/sysmacros.h>
#include <sys/ddidevmap.h>
#include <sys/xendev.h>
#include <public/io/protocols.h>
#include <xen/io/blkif_impl.h>

#include "blk_common.h"


/* blk interface status */
enum blk_if_state {
	/*
	 * initial state
	 */
	BLK_IF_UNKNOWN = 0,
	/*
	 * frontend xenbus state changed to XenbusStateConnected,
	 * we finally connect
	 */
	BLK_IF_CONNECTED,
	/*
	 * frontend xenbus state changed to XenbusStateClosed,
	 * interface disconnected
	 */
	BLK_IF_DISCONNECTED
};

/* backend device status */
enum blk_be_state {
	/* initial state */
	BLK_BE_UNKNOWN = 0,
	/* backend device is ready (hotplug script finishes successfully) */
	BLK_BE_READY
};

/* frontend status */
enum blk_fe_state {
	/* initial state */
	BLK_FE_UNKNOWN = 0,
	/*
	 * frontend's xenbus state has changed to
	 * XenbusStateInitialised, is ready for connecting
	 */
	BLK_FE_READY
};

typedef struct blk_ring_state_s {
	kmutex_t		rs_mutex;
	boolean_t		rs_sleeping_on_ring;
	boolean_t		rs_ring_up;
	kcondvar_t		rs_cv;
} blk_ring_state_t;

/* Disk Statistics */
static char *blk_stats[] = {
	"rd_reqs",
	"wr_reqs",
	"br_reqs",
	"fl_reqs",
	"oo_reqs"
};

typedef struct blk_stats_s {
	uint64_t bs_req_reads;
	uint64_t bs_req_writes;
	uint64_t bs_req_barriers;
	uint64_t bs_req_flushes;
} blk_stats_t;

struct blk_ring_s {
	kmutex_t		ri_mutex;
	dev_info_t		*ri_dip;

	kstat_t			*ri_kstats;
	blk_stats_t		ri_stats;

	blk_intr_t		ri_intr;
	caddr_t			ri_intr_arg;
	blk_ring_cb_t		ri_ringup;
	caddr_t			ri_ringup_arg;
	blk_ring_cb_t		ri_ringdown;
	caddr_t			ri_ringdown_arg;

	/* blk interface, backend, and frontend status */
	enum blk_if_state	ri_if_status;
	enum blk_be_state	ri_be_status;
	enum blk_fe_state	ri_fe_status;

	domid_t			ri_fe;

	enum blkif_protocol	ri_protocol;
	size_t			ri_nentry;
	size_t			ri_entrysize;

	xendev_ring_t		*ri_ring;
	blk_ring_state_t	ri_state;
};


static void blk_oe_state_change(dev_info_t *dip, ddi_eventcookie_t id,
    void *arg, void *impl_data);
static void blk_hp_state_change(dev_info_t *dip, ddi_eventcookie_t id,
    void *arg, void *impl_data);
static int blk_check_state_transition(blk_ring_t ring, XenbusState oestate);
static int blk_start_connect(blk_ring_t ring);
static void blk_start_disconnect(blk_ring_t ring);
static void blk_ring_close(blk_ring_t ring);
static int blk_bindto_frontend(blk_ring_t ring);
static void blk_unbindfrom_frontend(blk_ring_t ring);
static uint_t blk_intr(caddr_t arg);

static int blk_kstat_init(blk_ring_t ring);
static void blk_kstat_fini(blk_ring_t ring);
static int blk_kstat_update(kstat_t *ksp, int flag);

static void blk_ring_request_32(blkif_request_t *dst,
    blkif_x86_32_request_t *src);
static void blk_ring_request_64(blkif_request_t *dst,
    blkif_x86_64_request_t *src);

static void blk_ring_response_32(blkif_x86_32_response_t *dst,
    blkif_response_t *src);
static void blk_ring_response_64(blkif_x86_64_response_t *dst,
    blkif_response_t *src);


/*
 * blk_ring_init()
 */
int
blk_ring_init(blk_ringinit_args_t *args, blk_ring_t *ringp)
{
	blk_ring_t ring;
	int e;


	ring = kmem_zalloc(sizeof (struct blk_ring_s), KM_SLEEP);
	mutex_init(&ring->ri_mutex, NULL, MUTEX_DRIVER, NULL);
	ring->ri_dip = args->ar_dip;
	ring->ri_intr = args->ar_intr;
	ring->ri_intr_arg = args->ar_intr_arg;
	ring->ri_ringup = args->ar_ringup;
	ring->ri_ringup_arg = args->ar_ringup_arg;
	ring->ri_ringdown = args->ar_ringdown;
	ring->ri_ringdown_arg = args->ar_ringdown_arg;

	ring->ri_if_status = BLK_IF_UNKNOWN;
	ring->ri_be_status = BLK_BE_UNKNOWN;
	ring->ri_fe_status = BLK_FE_UNKNOWN;
	ring->ri_state.rs_sleeping_on_ring = B_FALSE;
	ring->ri_state.rs_ring_up = B_FALSE;

	mutex_init(&ring->ri_state.rs_mutex, NULL, MUTEX_DRIVER, NULL);
	cv_init(&ring->ri_state.rs_cv, NULL, CV_DRIVER, NULL);

	e = blk_kstat_init(ring);
	if (e != DDI_SUCCESS) {
		goto ringinitfail_kstat;
	}

	/* Watch frontend and hotplug state change */
	if (xvdi_add_event_handler(ring->ri_dip, XS_OE_STATE,
	    blk_oe_state_change, ring) != DDI_SUCCESS) {
		goto ringinitfail_oestate;
	}
	if (xvdi_add_event_handler(ring->ri_dip, XS_HP_STATE,
	    blk_hp_state_change, ring) != DDI_SUCCESS) {
		goto ringinitfail_hpstate;
	}

	/*
	 * Kick-off hotplug script
	 */
	if (xvdi_post_event(ring->ri_dip, XEN_HP_ADD) != DDI_SUCCESS) {
		cmn_err(CE_WARN, "blk@%s: failed to start hotplug script",
		    ddi_get_name_addr(ring->ri_dip));
		goto ringinitfail_postevent;
	}

	/*
	 * start waiting for hotplug event and otherend state event
	 * mainly for debugging, frontend will not take any op seeing this
	 */
	(void) xvdi_switch_state(ring->ri_dip, XBT_NULL, XenbusStateInitWait);

	*ringp = ring;
	return (DDI_SUCCESS);

ringinitfail_postevent:
	xvdi_remove_event_handler(ring->ri_dip, XS_HP_STATE);
ringinitfail_hpstate:
	xvdi_remove_event_handler(ring->ri_dip, XS_OE_STATE);
ringinitfail_oestate:
	blk_kstat_fini(ring);
ringinitfail_kstat:
	cv_destroy(&ring->ri_state.rs_cv);
	mutex_destroy(&ring->ri_state.rs_mutex);
	mutex_destroy(&ring->ri_mutex);
	kmem_free(ring, sizeof (struct blk_ring_s));
	return (DDI_FAILURE);
}


/*
 * blk_ring_fini()
 */
void
blk_ring_fini(blk_ring_t *ringp)
{
	blk_ring_t ring;


	ring = *ringp;

	mutex_enter(&ring->ri_mutex);
	if (ring->ri_if_status != BLK_IF_DISCONNECTED) {
		blk_ring_close(ring);
	}
	mutex_exit(&ring->ri_mutex);

	xvdi_remove_event_handler(ring->ri_dip, NULL);
	blk_kstat_fini(ring);
	cv_destroy(&ring->ri_state.rs_cv);
	mutex_destroy(&ring->ri_state.rs_mutex);
	mutex_destroy(&ring->ri_mutex);
	kmem_free(ring, sizeof (struct blk_ring_s));

	*ringp = NULL;
}


/*
 * blk_kstat_init()
 */
static int
blk_kstat_init(blk_ring_t ring)
{
	int nstat = sizeof (blk_stats) / sizeof (blk_stats[0]);
	char **cp = blk_stats;
	kstat_named_t *knp;

	ring->ri_kstats = kstat_create(ddi_get_name(ring->ri_dip),
	    ddi_get_instance(ring->ri_dip), "req_statistics", "block",
	    KSTAT_TYPE_NAMED, nstat, 0);
	if (ring->ri_kstats == NULL) {
		return (DDI_FAILURE);
	}

	ring->ri_kstats->ks_private = ring;
	ring->ri_kstats->ks_update = blk_kstat_update;

	knp = ring->ri_kstats->ks_data;
	while (nstat > 0) {
		kstat_named_init(knp, *cp, KSTAT_DATA_UINT64);
		knp++;
		cp++;
		nstat--;
	}

	kstat_install(ring->ri_kstats);

	return (DDI_SUCCESS);
}


/*
 * blk_kstat_fini()
 */
static void
blk_kstat_fini(blk_ring_t ring)
{
	kstat_delete(ring->ri_kstats);
}


/*
 * blk_kstat_update()
 */
static int
blk_kstat_update(kstat_t *ksp, int flag)
{
	kstat_named_t *knp;
	blk_stats_t *stats;
	blk_ring_t ring;


	if (flag != KSTAT_READ) {
		return (EACCES);
	}

	ring = ksp->ks_private;
	stats = &ring->ri_stats;
	knp = ksp->ks_data;

	/*
	 * Assignment order should match that of the names in
	 * blk_stats.
	 */
	(knp++)->value.ui64 = stats->bs_req_reads;
	(knp++)->value.ui64 = stats->bs_req_writes;
	(knp++)->value.ui64 = stats->bs_req_barriers;
	(knp++)->value.ui64 = stats->bs_req_flushes;
	(knp++)->value.ui64 = 0; /* oo_req */

	return (0);
}


/*
 * blk_oe_state_change()
 */
/*ARGSUSED*/
static void
blk_oe_state_change(dev_info_t *dip, ddi_eventcookie_t id, void *arg,
    void *impl_data)
{
	XenbusState new_state;
	blk_ring_t ring;


	ring = (blk_ring_t)arg;
	new_state = *(XenbusState *)impl_data;

	mutex_enter(&ring->ri_mutex);

	if (blk_check_state_transition(ring, new_state) == DDI_FAILURE) {
		mutex_exit(&ring->ri_mutex);
		return;
	}

	switch (new_state) {
	case XenbusStateInitialised:
		ASSERT(ring->ri_if_status == BLK_IF_UNKNOWN);

		/* frontend is ready for connecting */
		ring->ri_fe_status = BLK_FE_READY;

		if (ring->ri_be_status == BLK_BE_READY) {
			mutex_exit(&ring->ri_mutex);
			if (blk_start_connect(ring) != DDI_SUCCESS)
				(void) blk_start_disconnect(ring);
			mutex_enter(&ring->ri_mutex);
		}
		break;
	case XenbusStateClosing:
		(void) xvdi_switch_state(dip, XBT_NULL, XenbusStateClosing);
		break;
	case XenbusStateClosed:
		/* clean up */
		(void) xvdi_post_event(ring->ri_dip, XEN_HP_REMOVE);
		if (ring->ri_ringdown != NULL) {
			(*(ring->ri_ringdown))(ring->ri_ringdown_arg);
		}
		blk_ring_close(ring);

		/* reset state in case of reconnect */
		ring->ri_if_status = BLK_IF_UNKNOWN;
		ring->ri_be_status = BLK_BE_UNKNOWN;
		ring->ri_fe_status = BLK_FE_UNKNOWN;
		ring->ri_state.rs_sleeping_on_ring = B_FALSE;
		ring->ri_state.rs_ring_up = B_FALSE;

		break;
	default:
		ASSERT(0);
	}

	mutex_exit(&ring->ri_mutex);
}


/*
 * blk_hp_state_change()
 */
/*ARGSUSED*/
static void
blk_hp_state_change(dev_info_t *dip, ddi_eventcookie_t id, void *arg,
    void *impl_data)
{
	xendev_hotplug_state_t hpstate;
	blk_ring_t ring;


	ring = (blk_ring_t)arg;
	hpstate = *(xendev_hotplug_state_t *)impl_data;

	mutex_enter(&ring->ri_mutex);
	if (hpstate == Connected) {
		/* Hotplug script has completed successfully */
		if (ring->ri_be_status == BLK_BE_UNKNOWN) {
			ring->ri_be_status = BLK_BE_READY;
			if (ring->ri_fe_status == BLK_FE_READY) {
				mutex_exit(&ring->ri_mutex);
				/* try to connect to frontend */
				if (blk_start_connect(ring) != DDI_SUCCESS)
					(void) blk_start_disconnect(ring);
				mutex_enter(&ring->ri_mutex);
			}
		}
	}
	mutex_exit(&ring->ri_mutex);
}


/*
 * blk_check_state_transition()
 *    check the XenbusState change to see if the change is a valid transition
 *    or not. The new state is written by frontend domain, or by running
 *    xenstore-write to change it manually in dom0.
 */
static int
blk_check_state_transition(blk_ring_t ring, XenbusState oestate)
{
	switch (ring->ri_if_status) {
	case BLK_IF_UNKNOWN:
		if (ring->ri_fe_status == BLK_FE_UNKNOWN) {
			if ((oestate == XenbusStateUnknown)		||
			    (oestate == XenbusStateConnected))
				goto statechkfail_bug;
			else if ((oestate == XenbusStateInitialising)	||
			    (oestate == XenbusStateInitWait))
				goto statechkfail_nop;
		} else {
			if ((oestate == XenbusStateUnknown)		||
			    (oestate == XenbusStateInitialising)	||
			    (oestate == XenbusStateInitWait)		||
			    (oestate == XenbusStateConnected))
				goto statechkfail_bug;
			else if (oestate == XenbusStateInitialised)
				goto statechkfail_nop;
		}
		break;

	case BLK_IF_CONNECTED:
		if ((oestate == XenbusStateUnknown)		||
		    (oestate == XenbusStateInitialising)	||
		    (oestate == XenbusStateInitWait)		||
		    (oestate == XenbusStateInitialised))
			goto statechkfail_bug;
		else if (oestate == XenbusStateConnected)
			goto statechkfail_nop;
		break;

	case BLK_IF_DISCONNECTED:
	default:
		goto statechkfail_bug;
	}

	return (DDI_SUCCESS);

statechkfail_bug:
	cmn_err(CE_NOTE, "blk@%s: unexpected otherend "
	    "state change to %d!, when status is %d",
	    ddi_get_name_addr(ring->ri_dip), oestate,
	    ring->ri_if_status);

statechkfail_nop:
	return (DDI_FAILURE);
}


/*
 * blk_start_connect()
 *    Kick-off connect process
 *    If ri_fe_status == BLK_FE_READY and ri_be_status == BLK_BE_READY
 *    the ri_if_status will be changed to BLK_IF_CONNECTED on success,
 *    otherwise, ri_if_status will not be changed
 */
static int
blk_start_connect(blk_ring_t ring)
{
	xenbus_transaction_t xbt;
	dev_info_t *dip;
	char *barrier;
	char *xsnode;
	uint_t len;
	int e;


	dip = ring->ri_dip;

	/*
	 * Start connect to frontend only when backend device are ready
	 * and frontend has moved to XenbusStateInitialised, which means
	 * ready to connect
	 */
	ASSERT(ring->ri_fe_status == BLK_FE_READY);
	ASSERT(ring->ri_be_status == BLK_BE_READY);

	xsnode = xvdi_get_xsname(dip);
	if (xsnode == NULL) {
		goto startconnectfail_get_xsname;
	}

	ring->ri_fe = xvdi_get_oeid(dip);
	if (ring->ri_fe == (domid_t)-1) {
		goto startconnectfail_get_oeid;
	}

	e =  xvdi_switch_state(dip, XBT_NULL, XenbusStateInitialised);
	if (e > 0) {
		goto startconnectfail_switch_init;
	}

	e = blk_bindto_frontend(ring);
	if (e != DDI_SUCCESS) {
		goto startconnectfail_bindto_frontend;
	}
	ring->ri_if_status = BLK_IF_CONNECTED;

	e = ddi_add_intr(dip, 0, NULL, NULL, blk_intr, (caddr_t)ring);
	if (e != DDI_SUCCESS) {
		goto startconnectfail_add_intr;
	}

trans_retry:
	e = xenbus_transaction_start(&xbt);
	if (e != 0) {
		xvdi_fatal_error(dip, e, "transaction start");
		goto startconnectfail_transaction_start;
	}

	/* xentop requires the instance in xenstore */
	e = xenbus_printf(xbt, xsnode, "instance", "%d",
	    ddi_get_instance(ring->ri_dip));
	if (e != 0) {
		cmn_err(CE_WARN, "xdb@%s: failed to write 'instance'",
		    ddi_get_name_addr(dip));
		xvdi_fatal_error(dip, e, "writing 'instance'");
		(void) xenbus_transaction_end(xbt, 1);
		goto startconnectfail_xenbus_printf;
	}

	/* If feature-barrier isn't present in xenstore, add it */
	e = xenbus_read(xbt, xsnode, "feature-barrier", (void **)&barrier,
	    &len);
	if (e != 0) {
		e = xenbus_printf(xbt, xsnode, "feature-barrier", "%d", 1);
		if (e != 0) {
			cmn_err(CE_WARN, "xdb@%s: failed to write "
			    "'feature-barrier'", ddi_get_name_addr(dip));
			xvdi_fatal_error(dip, e, "writing 'feature-barrier'");
			(void) xenbus_transaction_end(xbt, 1);
			goto startconnectfail_xenbus_printf;
		}
	} else {
		kmem_free(barrier, len);
	}

	e = xvdi_switch_state(dip, xbt, XenbusStateConnected);
	if (e > 0) {
		xvdi_fatal_error(dip, e, "writing 'state'");
		(void) xenbus_transaction_end(xbt, 1);
		goto startconnectfail_switch_connected;
	}

	e = xenbus_transaction_end(xbt, 0);
	if (e != 0) {
		if (e == EAGAIN) {
			/* transaction is ended, don't need to abort it */
			goto trans_retry;
		}
		xvdi_fatal_error(dip, e, "completing transaction");
		goto startconnectfail_transaction_end;
	}

	mutex_enter(&ring->ri_state.rs_mutex);
	ring->ri_state.rs_ring_up = B_TRUE;
	if (ring->ri_state.rs_sleeping_on_ring) {
		ring->ri_state.rs_sleeping_on_ring = B_FALSE;
		cv_signal(&ring->ri_state.rs_cv);
	}
	mutex_exit(&ring->ri_state.rs_mutex);

	if (ring->ri_ringup != NULL) {
		(*(ring->ri_ringup))(ring->ri_ringup_arg);
	}

	return (DDI_SUCCESS);


startconnectfail_transaction_end:
startconnectfail_switch_connected:
startconnectfail_xenbus_printf:
startconnectfail_transaction_start:
	ddi_remove_intr(dip, 0, NULL);
startconnectfail_add_intr:
	blk_unbindfrom_frontend(ring);
	ring->ri_fe = (domid_t)-1;
startconnectfail_bindto_frontend:
	(void) xvdi_switch_state(dip, XBT_NULL, XenbusStateClosed);
startconnectfail_switch_init:
startconnectfail_get_oeid:
startconnectfail_get_xsname:
	return (DDI_FAILURE);
}


/*
 * blk_start_disconnect()
 *    Kick-off disconnect process. ri_if_status will not be changed
 */
static void
blk_start_disconnect(blk_ring_t ring)
{
	/* Kick-off disconnect process */
	(void) xvdi_switch_state(ring->ri_dip, XBT_NULL, XenbusStateClosing);
}


/*
 * blk_ring_close()
 *    Disconnect from frontend and close backend device
 *    ifstatus will be changed to BLK_DISCONNECTED
 *    Xenbus state will be changed to XenbusStateClosed
 */
static void
blk_ring_close(blk_ring_t ring)
{
	dev_info_t *dip;


	/* mutex protect ri_if_status only here */
	ASSERT(MUTEX_HELD(&ring->ri_mutex));

	dip = ring->ri_dip;

	if (ring->ri_if_status != BLK_IF_CONNECTED) {
		return;
	}

	ring->ri_if_status = BLK_IF_DISCONNECTED;
	mutex_exit(&ring->ri_mutex);

	/* stop accepting I/O request from frontend */
	ddi_remove_intr(dip, 0, NULL);

	blk_unbindfrom_frontend(ring);
	ring->ri_fe = (domid_t)-1;
	(void) xvdi_switch_state(dip, XBT_NULL, XenbusStateClosed);
	mutex_enter(&ring->ri_mutex);
}


/*
 * blk_bindto_frontend()
 */
static int
blk_bindto_frontend(blk_ring_t ring)
{
	evtchn_port_t evtchn;
	char protocol[64];
	grant_ref_t gref;
	dev_info_t *dip;
	char *oename;
	int e;


	dip = ring->ri_dip;
	protocol[0] = 0x0;

	/*
	 * Gather info from frontend
	 */
	oename = xvdi_get_oename(dip);
	if (oename == NULL) {
		return (DDI_FAILURE);
	}

	e = xenbus_gather(XBT_NULL, oename, "ring-ref", "%lu", &gref,
	    "event-channel", "%u", &evtchn, NULL);
	if (e != 0) {
		xvdi_fatal_error(dip, e,
		    "Getting ring-ref and evtchn from frontend");
		return (DDI_FAILURE);
	}

	e = xenbus_gather(XBT_NULL, oename, "protocol", "%63s",
	    protocol, NULL);
	if (e != 0) {
		(void) strcpy(protocol, "unspecified, assuming native");
	} else if (strcmp(protocol, XEN_IO_PROTO_ABI_NATIVE) == 0) {
		ring->ri_protocol = BLKIF_PROTOCOL_NATIVE;
		ring->ri_nentry = BLKIF_RING_SIZE;
		ring->ri_entrysize = sizeof (union blkif_sring_entry);
	} else if (strcmp(protocol, XEN_IO_PROTO_ABI_X86_32) == 0) {
		ring->ri_protocol = BLKIF_PROTOCOL_X86_32;
		ring->ri_nentry = BLKIF_X86_32_RING_SIZE;
		ring->ri_entrysize = sizeof (union blkif_x86_32_sring_entry);
	} else if (strcmp(protocol, XEN_IO_PROTO_ABI_X86_64) == 0) {
		ring->ri_protocol = BLKIF_PROTOCOL_X86_64;
		ring->ri_nentry = BLKIF_X86_64_RING_SIZE;
		ring->ri_entrysize = sizeof (union blkif_x86_64_sring_entry);
	} else {
		xvdi_fatal_error(dip, e, "unknown fe protocol");
		return (DDI_FAILURE);
	}

	/*
	 * map and init ring
	 */
	e = xvdi_map_ring(dip, ring->ri_nentry, ring->ri_entrysize, gref,
	    &ring->ri_ring);
	if (e != DDI_SUCCESS) {
		return (DDI_FAILURE);
	}

	/*
	 * bind event channel
	 */
	e = xvdi_bind_evtchn(dip, evtchn);
	if (e != DDI_SUCCESS) {
		xvdi_unmap_ring(ring->ri_ring);
		return (DDI_FAILURE);
	}


	return (DDI_SUCCESS);
}


/*
 * blk_unbindfrom_frontend()
 */
static void
blk_unbindfrom_frontend(blk_ring_t ring)
{
	xvdi_free_evtchn(ring->ri_dip);
	xvdi_unmap_ring(ring->ri_ring);
}


/*
 * blk_intr()
 */
static uint_t
blk_intr(caddr_t arg)
{
	blk_ring_t ring;

	ring = (blk_ring_t)arg;
	if (ring->ri_if_status != BLK_IF_CONNECTED) {
		return (DDI_INTR_CLAIMED);
	}

	(void) (*ring->ri_intr)(ring->ri_intr_arg);
	return (DDI_INTR_CLAIMED);
}


/*
 * blk_ring_request_get()
 */
boolean_t
blk_ring_request_get(blk_ring_t ring, blkif_request_t *req)
{
	blkif_request_t *src;
	blk_stats_t *stats;


	mutex_enter(&ring->ri_mutex);

	if (ring->ri_if_status != BLK_IF_CONNECTED) {
		mutex_exit(&ring->ri_mutex);
		return (B_FALSE);
	}

	src = xvdi_ring_get_request(ring->ri_ring);
	if (src == NULL) {
		mutex_exit(&ring->ri_mutex);
		return (B_FALSE);
	}

	switch (ring->ri_protocol) {
	case BLKIF_PROTOCOL_NATIVE:
		bcopy(src, req, sizeof (*req));
		break;
	case BLKIF_PROTOCOL_X86_32:
		blk_ring_request_32(req, (blkif_x86_32_request_t *)src);
		break;
	case BLKIF_PROTOCOL_X86_64:
		blk_ring_request_64(req, (blkif_x86_64_request_t *)src);
		break;
	default:
		cmn_err(CE_WARN, "blkif@%s: unrecognised protocol: %d",
		    ddi_get_name_addr(ring->ri_dip),
		    ring->ri_protocol);
	}
	mutex_exit(&ring->ri_mutex);

	stats = &ring->ri_stats;
	switch (req->operation) {
	case BLKIF_OP_READ:
		stats->bs_req_reads++;
		break;
	case BLKIF_OP_WRITE:
		stats->bs_req_writes++;
		break;
	case BLKIF_OP_WRITE_BARRIER:
		stats->bs_req_barriers++;
		break;
	case BLKIF_OP_FLUSH_DISKCACHE:
		stats->bs_req_flushes++;
		break;
	}

	return (B_TRUE);
}


/*
 * blk_ring_request_requeue()
 *    if a request is requeued, caller will have to poll for request
 *    later.
 */
void
blk_ring_request_requeue(blk_ring_t ring)
{
	mutex_enter(&ring->ri_mutex);

	if (ring->ri_if_status != BLK_IF_CONNECTED) {
		mutex_exit(&ring->ri_mutex);
		return;
	}

	ring->ri_ring->xr_sring.br.req_cons--;

	mutex_exit(&ring->ri_mutex);
}


/*
 * blk_ring_response_put()
 */
void
blk_ring_response_put(blk_ring_t ring, blkif_response_t *src)
{
	blkif_response_t *rsp;
	int e;


	mutex_enter(&ring->ri_mutex);

	if (ring->ri_if_status != BLK_IF_CONNECTED) {
		mutex_exit(&ring->ri_mutex);
		return;
	}

	rsp = xvdi_ring_get_response(ring->ri_ring);
	ASSERT(rsp);

	switch (ring->ri_protocol) {
	case BLKIF_PROTOCOL_NATIVE:
		bcopy(src, rsp, sizeof (*rsp));
		break;
	case BLKIF_PROTOCOL_X86_32:
		blk_ring_response_32((blkif_x86_32_response_t *)rsp, src);
		break;
	case BLKIF_PROTOCOL_X86_64:
		blk_ring_response_64((blkif_x86_64_response_t *)rsp, src);
		break;
	default:
		cmn_err(CE_WARN, "blk@%s: unrecognised protocol: %d",
		    ddi_get_name_addr(ring->ri_dip),
		    ring->ri_protocol);
	}

	e = xvdi_ring_push_response(ring->ri_ring);
	if (e != 0) {
		xvdi_notify_oe(ring->ri_dip);
	}

	mutex_exit(&ring->ri_mutex);
}


/*
 * blk_ring_request_32()
 */
static void
blk_ring_request_32(blkif_request_t *dst, blkif_x86_32_request_t *src)
{
	int i, n = BLKIF_MAX_SEGMENTS_PER_REQUEST;
	dst->operation = src->operation;
	dst->nr_segments = src->nr_segments;
	dst->handle = src->handle;
	dst->id = src->id;
	dst->sector_number = src->sector_number;
	if (n > src->nr_segments)
		n = src->nr_segments;
	for (i = 0; i < n; i++)
		dst->seg[i] = src->seg[i];
}


/*
 * blk_ring_request_64()
 */
static void
blk_ring_request_64(blkif_request_t *dst, blkif_x86_64_request_t *src)
{
	int i, n = BLKIF_MAX_SEGMENTS_PER_REQUEST;
	dst->operation = src->operation;
	dst->nr_segments = src->nr_segments;
	dst->handle = src->handle;
	dst->id = src->id;
	dst->sector_number = src->sector_number;
	if (n > src->nr_segments)
		n = src->nr_segments;
	for (i = 0; i < n; i++)
		dst->seg[i] = src->seg[i];
}


/*
 * blk_ring_response_32()
 */
static void
blk_ring_response_32(blkif_x86_32_response_t *dst, blkif_response_t *src)
{
	dst->id = src->id;
	dst->operation = src->operation;
	dst->status = src->status;
}


/*
 * blk_ring_response_64()
 */
static void
blk_ring_response_64(blkif_x86_64_response_t *dst, blkif_response_t *src)
{
	dst->id = src->id;
	dst->operation = src->operation;
	dst->status = src->status;
}


/*
 * blk_ring_request_dump()
 */
void
blk_ring_request_dump(blkif_request_t *req)
{
	int i;

	/*
	 * Exploit the public interface definitions for BLKIF_OP_READ
	 * etc..
	 */
	char *op_name[] = { "read", "write", "barrier", "flush" };

	cmn_err(CE_NOTE, "   op=%s", op_name[req->operation]);
	cmn_err(CE_NOTE, "   num of segments=%d", req->nr_segments);
	cmn_err(CE_NOTE, "   handle=%d", req->handle);
	cmn_err(CE_NOTE, "   id=0x%llx", (unsigned long long)req->id);
	cmn_err(CE_NOTE, "   start sector=%llu",
	    (unsigned long long)req->sector_number);
	for (i = 0; i < req->nr_segments; i++) {
		cmn_err(CE_NOTE, "   gref=%d, first sec=%d,"
		    "last sec=%d", req->seg[i].gref, req->seg[i].first_sect,
		    req->seg[i].last_sect);
	}
}


/*
 * blk_ring_response_dump()
 */
void
blk_ring_response_dump(blkif_response_t *resp)
{
	/*
	 * Exploit the public interface definitions for BLKIF_OP_READ
	 * etc..
	 */
	char *op_name[] = { "read", "write", "barrier", "flush" };

	cmn_err(CE_NOTE, "   op=%d:%s", resp->operation,
	    op_name[resp->operation]);
	cmn_err(CE_NOTE, "   op=%d", resp->operation);
	cmn_err(CE_NOTE, "   status=%d", resp->status);
}