PSARC 2008/480 DTrace CPC Provider

6486156 DTrace cpc provider

17 files changed, 1404 insertions, 83 deletions

diff --git a/usr/src/uts/common/Makefile.files b/usr/src/uts/common/Makefile.files
index ecefe16b9f..9da74d6444 100644
--- a/usr/src/uts/common/Makefile.files
+++ b/usr/src/uts/common/Makefile.files
@@ -406,6 +406,8 @@ LOCKSTAT_OBJS += lockstat.o
 
 FASTTRAP_OBJS += fasttrap.o fasttrap_isa.o
 
+DCPC_OBJS += dcpc.o
+
 #
 #	Driver (pseudo-driver) Modules
 #
diff --git a/usr/src/uts/common/brand/lx/dtrace/lx_systrace.c b/usr/src/uts/common/brand/lx/dtrace/lx_systrace.c
index ab9828a589..bfeb78330a 100644
--- a/usr/src/uts/common/brand/lx/dtrace/lx_systrace.c
+++ b/usr/src/uts/common/brand/lx/dtrace/lx_systrace.c
@@ -20,7 +20,7 @@
  */
 
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -121,7 +121,7 @@ lx_systrace_provide(void *arg, const dtrace_probedesc_t *desc)
 }
 
 /*ARGSUSED*/
-static void
+static int
 lx_systrace_enable(void *arg, dtrace_id_t id, void *parg)
 {
 	int sysnum = LX_SYSTRACE_SYSNUM((uintptr_t)parg);
@@ -138,6 +138,7 @@ lx_systrace_enable(void *arg, dtrace_id_t id, void *parg)
 	} else {
 		lx_systrace_sysent[sysnum].lss_return = id;
 	}
+	return (0);
 }
 
 /*ARGSUSED*/
diff --git a/usr/src/uts/common/dtrace/dcpc.c b/usr/src/uts/common/dtrace/dcpc.c
new file mode 100644
index 0000000000..05f304fdd7
--- /dev/null
+++ b/usr/src/uts/common/dtrace/dcpc.c
@@ -0,0 +1,1096 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#include <sys/errno.h>
+#include <sys/cpuvar.h>
+#include <sys/stat.h>
+#include <sys/modctl.h>
+#include <sys/cmn_err.h>
+#include <sys/ddi.h>
+#include <sys/sunddi.h>
+#include <sys/ksynch.h>
+#include <sys/conf.h>
+#include <sys/kmem.h>
+#include <sys/kcpc.h>
+#include <sys/cpc_pcbe.h>
+#include <sys/cpc_impl.h>
+#include <sys/dtrace_impl.h>
+
+/*
+ * DTrace CPU Performance Counter Provider
+ * ---------------------------------------
+ *
+ * The DTrace cpc provider allows DTrace consumers to access the CPU
+ * performance counter overflow mechanism of a CPU. The configuration
+ * presented in a probe specification is programmed into the performance
+ * counter hardware of all available CPUs on a system. Programming the
+ * hardware causes a counter on each CPU to begin counting events of the
+ * given type. When the specified number of events have occurred, an overflow
+ * interrupt will be generated and the probe is fired.
+ *
+ * The required configuration for the performance counter is encoded into
+ * the probe specification and this includes the performance counter event
+ * name, processor mode, overflow rate and an optional unit mask.
+ *
+ * Most processors provide several counters (PICs) which can count all or a
+ * subset of the events available for a given CPU. However, when overflow
+ * profiling is being used, not all CPUs can detect which counter generated the
+ * overflow interrupt. In this case we cannot reliably determine which counter
+ * overflowed and we therefore only allow such CPUs to configure one event at
+ * a time. Processors that can determine the counter which overflowed are
+ * allowed to program as many events at one time as possible (in theory up to
+ * the number of instrumentation counters supported by that platform).
+ * Therefore, multiple consumers can enable multiple probes at the same time
+ * on such platforms. Platforms which cannot determine the source of an
+ * overflow interrupt are only allowed to program a single event at one time.
+ *
+ * The performance counter hardware is made available to consumers on a
+ * first-come, first-served basis. Only a finite amount of hardware resource
+ * is available and, while we make every attempt to accomodate requests from
+ * consumers, we must deny requests when hardware resources have been exhausted.
+ * A consumer will fail to enable probes when resources are currently in use.
+ *
+ * The cpc provider contends for shared hardware resources along with other
+ * consumers of the kernel CPU performance counter subsystem (e.g. cpustat(1M)).
+ * Only one such consumer can use the performance counters at any one time and
+ * counters are made available on a first-come, first-served basis. As with
+ * cpustat, the cpc provider has priority over per-LWP libcpc usage (e.g.
+ * cputrack(1)). Invoking the cpc provider will cause all existing per-LWP
+ * counter contexts to be invalidated.
+ */
+
+typedef struct dcpc_probe {
+	char		dcpc_event_name[CPC_MAX_EVENT_LEN];
+	int		dcpc_flag;	/* flags (USER/SYS) */
+	uint32_t	dcpc_ovfval;	/* overflow value */
+	int64_t		dcpc_umask;	/* umask/emask for this event */
+	int		dcpc_picno;	/* pic this event is programmed in */
+	int		dcpc_enabled;	/* probe is actually enabled? */
+	int		dcpc_disabling;	/* probe is currently being disabled */
+	dtrace_id_t	dcpc_id;	/* probeid this request is enabling */
+	int		dcpc_actv_req_idx;	/* idx into dcpc_actv_reqs[] */
+} dcpc_probe_t;
+
+static dev_info_t			*dcpc_devi;
+static dtrace_provider_id_t		dcpc_pid;
+static dcpc_probe_t			**dcpc_actv_reqs;
+static uint32_t				dcpc_enablings = 0;
+static int				dcpc_ovf_mask = 0;
+static int				dcpc_mult_ovf_cap = 0;
+static int				dcpc_mask_type = 0;
+
+/*
+ * When the dcpc provider is loaded, dcpc_min_overflow is set to either
+ * DCPC_MIN_OVF_DEFAULT or the value that dcpc-min-overflow is set to in
+ * the dcpc.conf file. Decrease this value to set probes with smaller
+ * overflow values. Remember that very small values could render a system
+ * unusable with frequently occurring events.
+ */
+#define	DCPC_MIN_OVF_DEFAULT		5000
+static uint32_t				dcpc_min_overflow;
+
+static int dcpc_aframes = 0;	/* override for artificial frame setting */
+#if defined(__x86)
+#define	DCPC_ARTIFICIAL_FRAMES	8
+#elif defined(__sparc)
+#define	DCPC_ARTIFICIAL_FRAMES	2
+#endif
+
+/*
+ * Called from the platform overflow interrupt handler. 'bitmap' is a mask
+ * which contains the pic(s) that have overflowed.
+ */
+static void
+dcpc_fire(uint64_t bitmap)
+{
+	int i;
+
+	/*
+	 * No counter was marked as overflowing. Shout about it and get out.
+	 */
+	if ((bitmap & dcpc_ovf_mask) == 0) {
+		cmn_err(CE_NOTE, "dcpc_fire: no counter overflow found\n");
+		return;
+	}
+
+	/*
+	 * This is the common case of a processor that doesn't support
+	 * multiple overflow events. Such systems are only allowed a single
+	 * enabling and therefore we just look for the first entry in
+	 * the active request array.
+	 */
+	if (!dcpc_mult_ovf_cap) {
+		for (i = 0; i < cpc_ncounters; i++) {
+			if (dcpc_actv_reqs[i] != NULL) {
+				dtrace_probe(dcpc_actv_reqs[i]->dcpc_id,
+				    CPU->cpu_cpcprofile_pc,
+				    CPU->cpu_cpcprofile_upc, 0, 0, 0);
+				return;
+			}
+		}
+		return;
+	}
+
+	/*
+	 * This is a processor capable of handling multiple overflow events.
+	 * Iterate over the array of active requests and locate the counters
+	 * that overflowed (note: it is possible for more than one counter to
+	 * have overflowed at the same time).
+	 */
+	for (i = 0; i < cpc_ncounters; i++) {
+		if (dcpc_actv_reqs[i] != NULL &&
+		    (bitmap & (1ULL << dcpc_actv_reqs[i]->dcpc_picno))) {
+			dtrace_probe(dcpc_actv_reqs[i]->dcpc_id,
+			    CPU->cpu_cpcprofile_pc,
+			    CPU->cpu_cpcprofile_upc, 0, 0, 0);
+		}
+	}
+}
+
+static void
+dcpc_create_probe(dtrace_provider_id_t id, const char *probename,
+    char *eventname, int64_t umask, uint32_t ovfval, char flag)
+{
+	dcpc_probe_t *pp;
+	int nr_frames = DCPC_ARTIFICIAL_FRAMES + dtrace_mach_aframes();
+
+	if (dcpc_aframes)
+		nr_frames = dcpc_aframes;
+
+	if (dtrace_probe_lookup(id, NULL, NULL, probename) != 0)
+		return;
+
+	pp = kmem_zalloc(sizeof (dcpc_probe_t), KM_SLEEP);
+	(void) strncpy(pp->dcpc_event_name, eventname,
+	    sizeof (pp->dcpc_event_name) - 1);
+	pp->dcpc_event_name[sizeof (pp->dcpc_event_name) - 1] = '\0';
+	pp->dcpc_flag = flag | CPC_OVF_NOTIFY_EMT;
+	pp->dcpc_ovfval = ovfval;
+	pp->dcpc_umask = umask;
+	pp->dcpc_actv_req_idx = pp->dcpc_picno = pp->dcpc_disabling = -1;
+
+	pp->dcpc_id = dtrace_probe_create(id, NULL, NULL, probename,
+	    nr_frames, pp);
+}
+
+/*ARGSUSED*/
+static void
+dcpc_provide(void *arg, const dtrace_probedesc_t *desc)
+{
+	/*
+	 * The format of a probe is:
+	 *
+	 *	event_name-mode-{optional_umask}-overflow_rate
+	 * e.g.
+	 *	DC_refill_from_system-user-0x1e-50000, or,
+	 *	DC_refill_from_system-all-10000
+	 *
+	 */
+	char *str, *end, *p;
+	int i, flag = 0;
+	char event[CPC_MAX_EVENT_LEN];
+	long umask = -1, val = 0;
+	size_t evlen, len;
+
+	/*
+	 * The 'cpc' provider offers no probes by default.
+	 */
+	if (desc == NULL)
+		return;
+
+	len = strlen(desc->dtpd_name);
+	p = str = kmem_alloc(len + 1, KM_SLEEP);
+	(void) strcpy(str, desc->dtpd_name);
+
+	/*
+	 * We have a poor man's strtok() going on here. Replace any hyphens
+	 * in the the probe name with NULL characters in order to make it
+	 * easy to parse the string with regular string functions.
+	 */
+	for (i = 0; i < len; i++) {
+		if (str[i] == '-')
+			str[i] = '\0';
+	}
+
+	/*
+	 * The first part of the string must be either a platform event
+	 * name or a generic event name.
+	 */
+	evlen = strlen(p);
+	(void) strncpy(event, p, CPC_MAX_EVENT_LEN - 1);
+	event[CPC_MAX_EVENT_LEN - 1] = '\0';
+
+	/*
+	 * The next part of the name is the mode specification. Valid
+	 * settings are "user", "kernel" or "all".
+	 */
+	p += evlen + 1;
+
+	if (strcmp(p, "user") == 0)
+		flag |= CPC_COUNT_USER;
+	else if (strcmp(p, "kernel") == 0)
+		flag |= CPC_COUNT_SYSTEM;
+	else if (strcmp(p, "all") == 0)
+		flag |= CPC_COUNT_USER | CPC_COUNT_SYSTEM;
+	else
+		goto err;
+
+	/*
+	 * Next we either have a mask specification followed by an overflow
+	 * rate or just an overflow rate on its own.
+	 */
+	p += strlen(p) + 1;
+	if (p[0] == '0' && (p[1] == 'x' || p[1] == 'X')) {
+		/*
+		 * A unit mask can only be specified if:
+		 * 1) this performance counter back end supports masks.
+		 * 2) the specified event is platform specific.
+		 * 3) a valid hex number is converted.
+		 * 4) no extraneous characters follow the mask specification.
+		 */
+		if (dcpc_mask_type != 0 && strncmp(event, "PAPI", 4) != 0 &&
+		    ddi_strtol(p, &end, 16, &umask) == 0 &&
+		    end == p + strlen(p)) {
+			p += strlen(p) + 1;
+		} else {
+			goto err;
+		}
+	}
+
+	/*
+	 * This final part must be an overflow value which has to be greater
+	 * than the minimum permissible overflow rate.
+	 */
+	if ((ddi_strtol(p, &end, 10, &val) != 0) || end != p + strlen(p) ||
+	    val < dcpc_min_overflow)
+		goto err;
+
+	/*
+	 * Validate the event and create the probe.
+	 */
+	for (i = 0; i < cpc_ncounters; i++) {
+		if (strstr(kcpc_list_events(i), event) != NULL)
+			dcpc_create_probe(dcpc_pid, desc->dtpd_name, event,
+			    umask, (uint32_t)val, flag);
+	}
+
+err:
+	kmem_free(str, len + 1);
+}
+
+/*ARGSUSED*/
+static void
+dcpc_destroy(void *arg, dtrace_id_t id, void *parg)
+{
+	dcpc_probe_t *pp = parg;
+
+	ASSERT(pp->dcpc_enabled == 0);
+	kmem_free(pp, sizeof (dcpc_probe_t));
+}
+
+/*ARGSUSED*/
+static int
+dcpc_usermode(void *arg, dtrace_id_t id, void *parg)
+{
+	return (CPU->cpu_cpcprofile_pc == 0);
+}
+
+static void
+dcpc_populate_set(cpu_t *c, dcpc_probe_t *pp, kcpc_set_t *set, int reqno)
+{
+	kcpc_set_t *oset;
+	int i;
+
+	(void) strncpy(set->ks_req[reqno].kr_event, pp->dcpc_event_name,
+	    CPC_MAX_EVENT_LEN);
+	set->ks_req[reqno].kr_config = NULL;
+	set->ks_req[reqno].kr_index = reqno;
+	set->ks_req[reqno].kr_picnum = -1;
+	set->ks_req[reqno].kr_flags =  pp->dcpc_flag;
+
+	/*
+	 * If a unit mask has been specified then detect which attribute
+	 * the platform needs. For now, it's either "umask" or "emask".
+	 */
+	if (pp->dcpc_umask >= 0) {
+		set->ks_req[reqno].kr_attr =
+		    kmem_zalloc(sizeof (kcpc_attr_t), KM_SLEEP);
+		set->ks_req[reqno].kr_nattrs = 1;
+		if (dcpc_mask_type & DCPC_UMASK)
+			(void) strncpy(set->ks_req[reqno].kr_attr->ka_name,
+			    "umask", 5);
+		else
+			(void) strncpy(set->ks_req[reqno].kr_attr->ka_name,
+			    "emask", 5);
+		set->ks_req[reqno].kr_attr->ka_val = pp->dcpc_umask;
+	} else {
+		set->ks_req[reqno].kr_attr = NULL;
+		set->ks_req[reqno].kr_nattrs = 0;
+	}
+
+	/*
+	 * If this probe is enabled, obtain its current countdown value
+	 * and use that. The CPUs cpc context might not exist yet if we
+	 * are dealing with a CPU that is just coming online.
+	 */
+	if (pp->dcpc_enabled && (c->cpu_cpc_ctx != NULL)) {
+		oset = c->cpu_cpc_ctx->kc_set;
+
+		for (i = 0; i < oset->ks_nreqs; i++) {
+			if (strcmp(oset->ks_req[i].kr_event,
+			    set->ks_req[reqno].kr_event) == 0) {
+				set->ks_req[reqno].kr_preset =
+				    *(oset->ks_req[i].kr_data);
+			}
+		}
+	} else {
+		set->ks_req[reqno].kr_preset = UINT64_MAX - pp->dcpc_ovfval;
+	}
+
+	set->ks_nreqs++;
+}
+
+
+/*
+ * Create a fresh request set for the enablings represented in the
+ * 'dcpc_actv_reqs' array which contains the probes we want to be
+ * in the set. This can be called for several reasons:
+ *
+ * 1)	We are on a single or multi overflow platform and we have no
+ *	current events so we can just create the set and initialize it.
+ * 2)	We are on a multi-overflow platform and we already have one or
+ *	more existing events and we are adding a new enabling. Create a
+ *	new set and copy old requests in and then add the new request.
+ * 3)	We are on a multi-overflow platform and we have just removed an
+ *	enabling but we still have enablings whch are valid. Create a new
+ *	set and copy in still valid requests.
+ */
+static kcpc_set_t *
+dcpc_create_set(cpu_t *c)
+{
+	int i, reqno = 0;
+	int active_requests = 0;
+	kcpc_set_t *set;
+
+	/*
+	 * First get a count of the number of currently active requests.
+	 * Note that dcpc_actv_reqs[] should always reflect which requests
+	 * we want to be in the set that is to be created. It is the
+	 * responsibility of the caller of dcpc_create_set() to adjust that
+	 * array accordingly beforehand.
+	 */
+	for (i = 0; i < cpc_ncounters; i++) {
+		if (dcpc_actv_reqs[i] != NULL)
+			active_requests++;
+	}
+
+	set = kmem_zalloc(sizeof (kcpc_set_t), KM_SLEEP);
+
+	set->ks_req =
+	    kmem_zalloc(sizeof (kcpc_request_t) * active_requests, KM_SLEEP);
+
+	set->ks_data =
+	    kmem_zalloc(active_requests * sizeof (uint64_t), KM_SLEEP);
+
+	/*
+	 * Look for valid entries in the active requests array and populate
+	 * the request set for any entries found.
+	 */
+	for (i = 0; i < cpc_ncounters; i++) {
+		if (dcpc_actv_reqs[i] != NULL) {
+			dcpc_populate_set(c, dcpc_actv_reqs[i], set, reqno);
+			reqno++;
+		}
+	}
+
+	return (set);
+}
+
+static int
+dcpc_program_cpu_event(cpu_t *c)
+{
+	int i, j, subcode;
+	kcpc_ctx_t *ctx, *octx;
+	kcpc_set_t *set;
+
+	set = dcpc_create_set(c);
+
+	octx = NULL;
+	set->ks_ctx = ctx = kcpc_ctx_alloc();
+	ctx->kc_set = set;
+	ctx->kc_cpuid = c->cpu_id;
+
+	if (kcpc_assign_reqs(set, ctx) != 0)
+		goto err;
+
+	if (kcpc_configure_reqs(ctx, set, &subcode) != 0)
+		goto err;
+
+	for (i = 0; i < set->ks_nreqs; i++) {
+		for (j = 0; j < cpc_ncounters; j++) {
+			if (dcpc_actv_reqs[j] != NULL &&
+			    strcmp(set->ks_req[i].kr_event,
+			    dcpc_actv_reqs[j]->dcpc_event_name) == 0) {
+				dcpc_actv_reqs[j]->dcpc_picno =
+				    set->ks_req[i].kr_picnum;
+			}
+		}
+	}
+
+	/*
+	 * If we already have an active enabling then save the current cpc
+	 * context away.
+	 */
+	if (c->cpu_cpc_ctx != NULL)
+		octx = c->cpu_cpc_ctx;
+
+	c->cpu_cpc_ctx = ctx;
+	kcpc_remote_program(c);
+
+	if (octx != NULL) {
+		kcpc_set_t *oset = octx->kc_set;
+		kmem_free(oset->ks_data, oset->ks_nreqs * sizeof (uint64_t));
+		kcpc_free_set(oset);
+		kcpc_ctx_free(octx);
+	}
+
+	return (0);
+
+err:
+	/*
+	 * We failed to configure this request up so free things up and
+	 * get out.
+	 */
+	kmem_free(set->ks_data, set->ks_nreqs * sizeof (uint64_t));
+	kcpc_free_set(set);
+	kcpc_ctx_free(ctx);
+
+	return (-1);
+}
+
+static void
+dcpc_disable_cpu(cpu_t *c)
+{
+	kcpc_ctx_t *ctx;
+	kcpc_set_t *set;
+
+	/*
+	 * Leave this CPU alone if it's already offline.
+	 */
+	if (c->cpu_flags & CPU_OFFLINE)
+		return;
+
+	kcpc_remote_stop(c);
+
+	ctx = c->cpu_cpc_ctx;
+	set = ctx->kc_set;
+
+	kcpc_free_configs(set);
+
+	kmem_free(set->ks_data, set->ks_nreqs * sizeof (uint64_t));
+	kcpc_free_set(set);
+	kcpc_ctx_free(ctx);
+	c->cpu_cpc_ctx = NULL;
+}
+
+/*
+ * Stop overflow interrupts being actively processed so that per-CPU
+ * configuration state can be changed safely and correctly. Each CPU has a
+ * dcpc interrupt state byte which is transitioned from DCPC_INTR_FREE (the
+ * "free" state) to DCPC_INTR_CONFIG (the "configuration in process" state)
+ * before any configuration state is changed on any CPUs. The hardware overflow
+ * handler, kcpc_hw_overflow_intr(), will only process an interrupt when a
+ * configuration is not in process (i.e. the state is marked as free). During
+ * interrupt processing the state is set to DCPC_INTR_PROCESSING by the
+ * overflow handler.
+ */
+static void
+dcpc_block_interrupts(void)
+{
+	cpu_t *c;
+	uint8_t *state;
+
+	c = cpu_list;
+
+	do {
+		state = &cpu_core[c->cpu_id].cpuc_dcpc_intr_state;
+
+		while (atomic_cas_8(state, DCPC_INTR_FREE,
+		    DCPC_INTR_CONFIG) != DCPC_INTR_FREE)
+			continue;
+
+	} while ((c = c->cpu_next) != cpu_list);
+}
+
+/*
+ * Set all CPUs dcpc interrupt state to DCPC_INTR_FREE to indicate that
+ * overflow interrupts can be processed safely.
+ */
+static void
+dcpc_release_interrupts(void)
+{
+	cpu_t *c = cpu_list;
+
+	do {
+		cpu_core[c->cpu_id].cpuc_dcpc_intr_state = DCPC_INTR_FREE;
+		membar_producer();
+	} while ((c = c->cpu_next) != cpu_list);
+}
+
+/*
+ * dcpc_program_event() can be called owing to a new enabling or if a multi
+ * overflow platform has disabled a request but needs to  program the requests
+ * that are still valid.
+ *
+ * Every invocation of dcpc_program_event() will create a new kcpc_ctx_t
+ * and a new request set which contains the new enabling and any old enablings
+ * which are still valid (possible with multi-overflow platforms).
+ */
+static int
+dcpc_program_event(dcpc_probe_t *pp)
+{
+	cpu_t *c;
+	int ret = 0;
+
+	ASSERT(MUTEX_HELD(&cpu_lock));
+
+	kpreempt_disable();
+
+	dcpc_block_interrupts();
+
+	c = cpu_list;
+
+	do {
+		/*
+		 * Skip CPUs that are currently offline.
+		 */
+		if (c->cpu_flags & CPU_OFFLINE)
+			continue;
+
+		if (c->cpu_cpc_ctx != NULL)
+			kcpc_remote_stop(c);
+	} while ((c = c->cpu_next) != cpu_list);
+
+	dcpc_release_interrupts();
+
+	/*
+	 * If this enabling is being removed (in the case of a multi event
+	 * capable system with more than one active enabling), we can now
+	 * update the active request array to reflect the enablings that need
+	 * to be reprogrammed.
+	 */
+	if (pp->dcpc_disabling == 1)
+		dcpc_actv_reqs[pp->dcpc_actv_req_idx] = NULL;
+
+	do {
+		/*
+		 * Skip CPUs that are currently offline.
+		 */
+		if (c->cpu_flags & CPU_OFFLINE)
+			continue;
+
+		ret = dcpc_program_cpu_event(c);
+	} while ((c = c->cpu_next) != cpu_list && ret == 0);
+
+	/*
+	 * If dcpc_program_cpu_event() fails then it is because we couldn't
+	 * configure the requests in the set for the CPU and not because of
+	 * an error programming the hardware. If we have a failure here then
+	 * we assume no CPUs have been programmed in the above step as they
+	 * are all configured identically.
+	 */
+	if (ret != 0) {
+		pp->dcpc_enabled = 0;
+		kpreempt_enable();
+		return (-1);
+	}
+
+	if (pp->dcpc_disabling != 1)
+		pp->dcpc_enabled = 1;
+
+	kpreempt_enable();
+
+	return (0);
+}
+
+/*ARGSUSED*/
+static int
+dcpc_enable(void *arg, dtrace_id_t id, void *parg)
+{
+	dcpc_probe_t *pp = parg;
+	int i, found = 0;
+	cpu_t *c;
+
+	ASSERT(MUTEX_HELD(&cpu_lock));
+
+	/*
+	 * Bail out if the counters are being used by a libcpc consumer.
+	 */
+	rw_enter(&kcpc_cpuctx_lock, RW_READER);
+	if (kcpc_cpuctx > 0) {
+		rw_exit(&kcpc_cpuctx_lock);
+		return (-1);
+	}
+
+	dtrace_cpc_in_use++;
+	rw_exit(&kcpc_cpuctx_lock);
+
+	/*
+	 * Locate this enabling in the first free entry of the active
+	 * request array.
+	 */
+	for (i = 0; i < cpc_ncounters; i++) {
+		if (dcpc_actv_reqs[i] == NULL) {
+			dcpc_actv_reqs[i] = pp;
+			pp->dcpc_actv_req_idx = i;
+			found = 1;
+			break;
+		}
+	}
+
+	/*
+	 * If we couldn't find a slot for this probe then there is no
+	 * room at the inn.
+	 */
+	if (!found) {
+		dtrace_cpc_in_use--;
+		return (-1);
+	}
+
+	ASSERT(pp->dcpc_actv_req_idx >= 0);
+
+	/*
+	 * The following must hold true if we are to (attempt to) enable
+	 * this request:
+	 *
+	 * 1) No enablings currently exist. We allow all platforms to
+	 * proceed if this is true.
+	 *
+	 * OR
+	 *
+	 * 2) If the platform is multi overflow capable and there are
+	 * less valid enablings than there are counters. There is no
+	 * guarantee that a platform can accommodate as many events as
+	 * it has counters for but we will at least try to program
+	 * up to that many requests.
+	 *
+	 * The 'dcpc_enablings' variable is implictly protected by locking
+	 * provided by the DTrace framework and the cpu management framework.
+	 */
+	if (dcpc_enablings == 0 || (dcpc_mult_ovf_cap &&
+	    dcpc_enablings < cpc_ncounters)) {
+		/*
+		 * Before attempting to program the first enabling we need to
+		 * invalidate any lwp-based contexts.
+		 */
+		if (dcpc_enablings == 0)
+			kcpc_invalidate_all();
+
+		if (dcpc_program_event(pp) == 0) {
+			dcpc_enablings++;
+			return (0);
+		}
+	}
+
+	/*
+	 * If active enablings existed before we failed to enable this probe
+	 * on a multi event capable platform then we need to restart counters
+	 * as they will have been stopped in the attempted configuration. The
+	 * context should now just contain the request prior to this failed
+	 * enabling.
+	 */
+	if (dcpc_enablings > 0 && dcpc_mult_ovf_cap) {
+		c = cpu_list;
+
+		ASSERT(dcpc_mult_ovf_cap == 1);
+		do {
+			/*
+			 * Skip CPUs that are currently offline.
+			 */
+			if (c->cpu_flags & CPU_OFFLINE)
+				continue;
+
+			kcpc_remote_program(c);
+		} while ((c = c->cpu_next) != cpu_list);
+	}
+
+	dtrace_cpc_in_use--;
+	dcpc_actv_reqs[pp->dcpc_actv_req_idx] = NULL;
+	pp->dcpc_actv_req_idx = pp->dcpc_picno = -1;
+
+	return (-1);
+}
+
+/*
+ * If only one enabling is active then remove the context and free
+ * everything up. If there are multiple enablings active then remove this
+ * one, its associated meta-data and re-program the hardware.
+ */
+/*ARGSUSED*/
+static void
+dcpc_disable(void *arg, dtrace_id_t id, void *parg)
+{
+	cpu_t *c;
+	dcpc_probe_t *pp = parg;
+
+	ASSERT(MUTEX_HELD(&cpu_lock));
+
+	kpreempt_disable();
+
+	/*
+	 * This probe didn't actually make it as far as being fully enabled
+	 * so we needn't do anything with it.
+	 */
+	if (pp->dcpc_enabled == 0) {
+		/*
+		 * If we actually allocated this request a slot in the
+		 * request array but failed to enabled it then remove the
+		 * entry in the array.
+		 */
+		if (pp->dcpc_actv_req_idx >= 0) {
+			dcpc_actv_reqs[pp->dcpc_actv_req_idx] = NULL;
+			pp->dcpc_actv_req_idx = pp->dcpc_picno =
+			    pp->dcpc_disabling = -1;
+		}
+
+		kpreempt_enable();
+		return;
+	}
+
+	/*
+	 * If this is the only enabling then stop all the counters and
+	 * free up the meta-data.
+	 */
+	if (dcpc_enablings == 1) {
+		ASSERT(dtrace_cpc_in_use == 1);
+
+		dcpc_block_interrupts();
+
+		c = cpu_list;
+
+		do {
+			dcpc_disable_cpu(c);
+		} while ((c = c->cpu_next) != cpu_list);
+
+		dcpc_actv_reqs[pp->dcpc_actv_req_idx] = NULL;
+		dcpc_release_interrupts();
+	} else {
+		/*
+		 * This platform can support multiple overflow events and
+		 * the enabling being disabled is not the last one. Remove this
+		 * enabling and re-program the hardware with the new config.
+		 */
+		ASSERT(dcpc_mult_ovf_cap);
+		ASSERT(dcpc_enablings > 1);
+
+		pp->dcpc_disabling = 1;
+		(void) dcpc_program_event(pp);
+	}
+
+	kpreempt_enable();
+
+	dcpc_enablings--;
+	dtrace_cpc_in_use--;
+	pp->dcpc_enabled = 0;
+	pp->dcpc_actv_req_idx = pp->dcpc_picno = pp->dcpc_disabling = -1;
+}
+
+/*ARGSUSED*/
+static int
+dcpc_cpu_setup(cpu_setup_t what, processorid_t cpu, void *arg)
+{
+	cpu_t *c;
+	uint8_t *state;
+
+	ASSERT(MUTEX_HELD(&cpu_lock));
+
+	switch (what) {
+	case CPU_OFF:
+		/*
+		 * Offline CPUs are not allowed to take part so remove this
+		 * CPU if we are actively tracing.
+		 */
+		if (dtrace_cpc_in_use) {
+			c = cpu_get(cpu);
+			state = &cpu_core[c->cpu_id].cpuc_dcpc_intr_state;
+
+			/*
+			 * Indicate that a configuration is in process in
+			 * order to stop overflow interrupts being processed
+			 * on this CPU while we disable it.
+			 */
+			while (atomic_cas_8(state, DCPC_INTR_FREE,
+			    DCPC_INTR_CONFIG) != DCPC_INTR_FREE)
+				continue;
+
+			dcpc_disable_cpu(c);
+
+			/*
+			 * Reset this CPUs interrupt state as the configuration
+			 * has ended.
+			 */
+			cpu_core[c->cpu_id].cpuc_dcpc_intr_state =
+			    DCPC_INTR_FREE;
+			membar_producer();
+		}
+		break;
+
+	case CPU_ON:
+	case CPU_SETUP:
+		/*
+		 * This CPU is being initialized or brought online so program
+		 * it with the current request set if we are actively tracing.
+		 */
+		if (dtrace_cpc_in_use) {
+			c = cpu_get(cpu);
+
+			(void) dcpc_program_cpu_event(c);
+		}
+		break;
+
+	default:
+		break;
+	}
+
+	return (0);
+}
+
+static dtrace_pattr_t dcpc_attr = {
+{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
+{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
+{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
+{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_CPU },
+{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
+};
+
+static dtrace_pops_t dcpc_pops = {
+    dcpc_provide,
+    NULL,
+    dcpc_enable,
+    dcpc_disable,
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    dcpc_usermode,
+    dcpc_destroy
+};
+
+/*ARGSUSED*/
+static int
+dcpc_open(dev_t *devp, int flag, int otyp, cred_t *cred_p)
+{
+	return (0);
+}
+
+/*ARGSUSED*/
+static int
+dcpc_info(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result)
+{
+	int error;
+
+	switch (infocmd) {
+	case DDI_INFO_DEVT2DEVINFO:
+		*result = (void *)dcpc_devi;
+		error = DDI_SUCCESS;
+		break;
+	case DDI_INFO_DEVT2INSTANCE:
+		*result = (void *)0;
+		error = DDI_SUCCESS;
+		break;
+	default:
+		error = DDI_FAILURE;
+	}
+	return (error);
+}
+
+static int
+dcpc_detach(dev_info_t *devi, ddi_detach_cmd_t cmd)
+{
+	switch (cmd) {
+	case DDI_DETACH:
+		break;
+	case DDI_SUSPEND:
+		return (DDI_SUCCESS);
+	default:
+		return (DDI_FAILURE);
+	}
+
+	if (dtrace_unregister(dcpc_pid) != 0)
+		return (DDI_FAILURE);
+
+	ddi_remove_minor_node(devi, NULL);
+
+	mutex_enter(&cpu_lock);
+	unregister_cpu_setup_func(dcpc_cpu_setup, NULL);
+	mutex_exit(&cpu_lock);
+
+	kmem_free(dcpc_actv_reqs, cpc_ncounters * sizeof (dcpc_probe_t *));
+
+	kcpc_unregister_dcpc();
+
+	return (DDI_SUCCESS);
+}
+
+static int
+dcpc_attach(dev_info_t *devi, ddi_attach_cmd_t cmd)
+{
+	uint_t caps;
+	char *attrs;
+
+	switch (cmd) {
+	case DDI_ATTACH:
+		break;
+	case DDI_RESUME:
+		return (DDI_SUCCESS);
+	default:
+		return (DDI_FAILURE);
+	}
+
+	if (kcpc_pcbe_loaded() == -1)
+		return (DDI_FAILURE);
+
+	caps = kcpc_pcbe_capabilities();
+
+	if (!(caps & CPC_CAP_OVERFLOW_INTERRUPT)) {
+		cmn_err(CE_WARN, "dcpc: Counter Overflow not supported"\
+		    " on this processor\n");
+		return (DDI_FAILURE);
+	}
+
+	if (ddi_create_minor_node(devi, "dcpc", S_IFCHR, 0,
+	    DDI_PSEUDO, NULL) == DDI_FAILURE ||
+	    dtrace_register("cpc", &dcpc_attr, DTRACE_PRIV_KERNEL,
+	    NULL, &dcpc_pops, NULL, &dcpc_pid) != 0) {
+		ddi_remove_minor_node(devi, NULL);
+		return (DDI_FAILURE);
+	}
+
+	mutex_enter(&cpu_lock);
+	register_cpu_setup_func(dcpc_cpu_setup, NULL);
+	mutex_exit(&cpu_lock);
+
+	dcpc_ovf_mask = (1 << cpc_ncounters) - 1;
+	ASSERT(dcpc_ovf_mask != 0);
+
+	if (caps & CPC_CAP_OVERFLOW_PRECISE)
+		dcpc_mult_ovf_cap = 1;
+
+	/*
+	 * Determine which, if any, mask attribute the back-end can use.
+	 */
+	attrs = kcpc_list_attrs();
+	if (strstr(attrs, "umask") != NULL)
+		dcpc_mask_type |= DCPC_UMASK;
+	else if (strstr(attrs, "emask") != NULL)
+		dcpc_mask_type |= DCPC_EMASK;
+
+	/*
+	 * The dcpc_actv_reqs array is used to store the requests that
+	 * we currently have programmed. The order of requests in this
+	 * array is not necessarily the order that the event appears in
+	 * the kcpc_request_t array. Once entered into a slot in the array
+	 * the entry is not moved until it's removed.
+	 */
+	dcpc_actv_reqs =
+	    kmem_zalloc(cpc_ncounters * sizeof (dcpc_probe_t *), KM_SLEEP);
+
+	dcpc_min_overflow = ddi_prop_get_int(DDI_DEV_T_ANY, devi,
+	    DDI_PROP_DONTPASS, "dcpc-min-overflow", DCPC_MIN_OVF_DEFAULT);
+
+	kcpc_register_dcpc(dcpc_fire);
+
+	ddi_report_dev(devi);
+	dcpc_devi = devi;
+
+	return (DDI_SUCCESS);
+}
+
+static struct cb_ops dcpc_cb_ops = {
+	dcpc_open,		/* open */
+	nodev,			/* close */
+	nulldev,		/* strategy */
+	nulldev,		/* print */
+	nodev,			/* dump */
+	nodev,			/* read */
+	nodev,			/* write */
+	nodev,			/* ioctl */
+	nodev,			/* devmap */
+	nodev,			/* mmap */
+	nodev,			/* segmap */
+	nochpoll,		/* poll */
+	ddi_prop_op,		/* cb_prop_op */
+	0,			/* streamtab  */
+	D_NEW | D_MP		/* Driver compatibility flag */
+};
+
+static struct dev_ops dcpc_ops = {
+	DEVO_REV,		/* devo_rev, */
+	0,			/* refcnt  */
+	dcpc_info,		/* get_dev_info */
+	nulldev,		/* identify */
+	nulldev,		/* probe */
+	dcpc_attach,		/* attach */
+	dcpc_detach,		/* detach */
+	nodev,			/* reset */
+	&dcpc_cb_ops,		/* driver operations */
+	NULL,			/* bus operations */
+	nodev,			/* dev power */
+	ddi_quiesce_not_needed	/* quiesce */
+};
+
+/*
+ * Module linkage information for the kernel.
+ */
+static struct modldrv modldrv = {
+	&mod_driverops,		/* module type */
+	"DTrace CPC Module",	/* name of module */
+	&dcpc_ops,		/* driver ops */
+};
+
+static struct modlinkage modlinkage = {
+	MODREV_1,
+	(void *)&modldrv,
+	NULL
+};
+
+int
+_init(void)
+{
+	return (mod_install(&modlinkage));
+}
+
+int
+_info(struct modinfo *modinfop)
+{
+	return (mod_info(&modlinkage, modinfop));
+}
+
+int
+_fini(void)
+{
+	return (mod_remove(&modlinkage));
+}
diff --git a/usr/src/uts/common/dtrace/dcpc.conf b/usr/src/uts/common/dtrace/dcpc.conf
new file mode 100644
index 0000000000..3f18694062
--- /dev/null
+++ b/usr/src/uts/common/dtrace/dcpc.conf
@@ -0,0 +1,33 @@
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+#
+# Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
+# Use is subject to license terms.
+#
+name="dcpc" parent="pseudo" instance=0;
+
+#
+# dcpc-min-overflow is the lower limit on the rate of overflow that can be
+# set on a probe. The default value is 5000. Decrease this value to create
+# probes with lower overflow values. Setting this value too low on a rapidly
+# increasing event could render a system unusable.
+#
+#dcpc-min-overflow=5000;
diff --git a/usr/src/uts/common/dtrace/dtrace.c b/usr/src/uts/common/dtrace/dtrace.c
index 91e3230737..00e71888cd 100644
--- a/usr/src/uts/common/dtrace/dtrace.c
+++ b/usr/src/uts/common/dtrace/dtrace.c
@@ -20,7 +20,7 @@
  */
 
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -239,10 +239,16 @@ static void
 dtrace_nullop(void)
 {}
 
+static int
+dtrace_enable_nullop(void)
+{
+	return (0);
+}
+
 static dtrace_pops_t	dtrace_provider_ops = {
 	(void (*)(void *, const dtrace_probedesc_t *))dtrace_nullop,
 	(void (*)(void *, struct modctl *))dtrace_nullop,
-	(void (*)(void *, dtrace_id_t, void *))dtrace_nullop,
+	(int (*)(void *, dtrace_id_t, void *))dtrace_enable_nullop,
 	(void (*)(void *, dtrace_id_t, void *))dtrace_nullop,
 	(void (*)(void *, dtrace_id_t, void *))dtrace_nullop,
 	(void (*)(void *, dtrace_id_t, void *))dtrace_nullop,
@@ -426,6 +432,7 @@ dtrace_load##bits(uintptr_t addr)					\
 #define	DTRACE_DYNHASH_SINK	1
 #define	DTRACE_DYNHASH_VALID	2
 
+#define	DTRACE_MATCH_FAIL	-1
 #define	DTRACE_MATCH_NEXT	0
 #define	DTRACE_MATCH_DONE	1
 #define	DTRACE_ANCHORED(probe)	((probe)->dtpr_func[0] != '\0')
@@ -6654,7 +6661,7 @@ dtrace_match(const dtrace_probekey_t *pkp, uint32_t priv, uid_t uid,
 {
 	dtrace_probe_t template, *probe;
 	dtrace_hash_t *hash = NULL;
-	int len, best = INT_MAX, nmatched = 0;
+	int len, rc, best = INT_MAX, nmatched = 0;
 	dtrace_id_t i;
 
 	ASSERT(MUTEX_HELD(&dtrace_lock));
@@ -6666,7 +6673,8 @@ dtrace_match(const dtrace_probekey_t *pkp, uint32_t priv, uid_t uid,
 	if (pkp->dtpk_id != DTRACE_IDNONE) {
 		if ((probe = dtrace_probe_lookup_id(pkp->dtpk_id)) != NULL &&
 		    dtrace_match_probe(probe, pkp, priv, uid, zoneid) > 0) {
-			(void) (*matched)(probe, arg);
+			if ((*matched)(probe, arg) == DTRACE_MATCH_FAIL)
+				return (DTRACE_MATCH_FAIL);
 			nmatched++;
 		}
 		return (nmatched);
@@ -6713,8 +6721,12 @@ dtrace_match(const dtrace_probekey_t *pkp, uint32_t priv, uid_t uid,
 
 			nmatched++;
 
-			if ((*matched)(probe, arg) != DTRACE_MATCH_NEXT)
+			if ((rc = (*matched)(probe, arg)) !=
+			    DTRACE_MATCH_NEXT) {
+				if (rc == DTRACE_MATCH_FAIL)
+					return (DTRACE_MATCH_FAIL);
 				break;
+			}
 		}
 
 		return (nmatched);
@@ -6733,8 +6745,11 @@ dtrace_match(const dtrace_probekey_t *pkp, uint32_t priv, uid_t uid,
 
 		nmatched++;
 
-		if ((*matched)(probe, arg) != DTRACE_MATCH_NEXT)
+		if ((rc = (*matched)(probe, arg)) != DTRACE_MATCH_NEXT) {
+			if (rc == DTRACE_MATCH_FAIL)
+				return (DTRACE_MATCH_FAIL);
 			break;
+		}
 	}
 
 	return (nmatched);
@@ -6954,7 +6969,7 @@ dtrace_unregister(dtrace_provider_id_t id)
 	dtrace_probe_t *probe, *first = NULL;
 
 	if (old->dtpv_pops.dtps_enable ==
-	    (void (*)(void *, dtrace_id_t, void *))dtrace_nullop) {
+	    (int (*)(void *, dtrace_id_t, void *))dtrace_enable_nullop) {
 		/*
 		 * If DTrace itself is the provider, we're called with locks
 		 * already held.
@@ -7100,7 +7115,7 @@ dtrace_invalidate(dtrace_provider_id_t id)
 	dtrace_provider_t *pvp = (dtrace_provider_t *)id;
 
 	ASSERT(pvp->dtpv_pops.dtps_enable !=
-	    (void (*)(void *, dtrace_id_t, void *))dtrace_nullop);
+	    (int (*)(void *, dtrace_id_t, void *))dtrace_enable_nullop);
 
 	mutex_enter(&dtrace_provider_lock);
 	mutex_enter(&dtrace_lock);
@@ -7141,7 +7156,7 @@ dtrace_condense(dtrace_provider_id_t id)
 	 * Make sure this isn't the dtrace provider itself.
 	 */
 	ASSERT(prov->dtpv_pops.dtps_enable !=
-	    (void (*)(void *, dtrace_id_t, void *))dtrace_nullop);
+	    (int (*)(void *, dtrace_id_t, void *))dtrace_enable_nullop);
 
 	mutex_enter(&dtrace_provider_lock);
 	mutex_enter(&dtrace_lock);
@@ -9095,7 +9110,7 @@ dtrace_ecb_add(dtrace_state_t *state, dtrace_probe_t *probe)
 	return (ecb);
 }
 
-static void
+static int
 dtrace_ecb_enable(dtrace_ecb_t *ecb)
 {
 	dtrace_probe_t *probe = ecb->dte_probe;
@@ -9108,7 +9123,7 @@ dtrace_ecb_enable(dtrace_ecb_t *ecb)
 		/*
 		 * This is the NULL probe -- there's nothing to do.
 		 */
-		return;
+		return (0);
 	}
 
 	if (probe->dtpr_ecb == NULL) {
@@ -9122,8 +9137,8 @@ dtrace_ecb_enable(dtrace_ecb_t *ecb)
 		if (ecb->dte_predicate != NULL)
 			probe->dtpr_predcache = ecb->dte_predicate->dtp_cacheid;
 
-		prov->dtpv_pops.dtps_enable(prov->dtpv_arg,
-		    probe->dtpr_id, probe->dtpr_arg);
+		return (prov->dtpv_pops.dtps_enable(prov->dtpv_arg,
+		    probe->dtpr_id, probe->dtpr_arg));
 	} else {
 		/*
 		 * This probe is already active.  Swing the last pointer to
@@ -9136,6 +9151,7 @@ dtrace_ecb_enable(dtrace_ecb_t *ecb)
 		probe->dtpr_predcache = 0;
 
 		dtrace_sync();
+		return (0);
 	}
 }
 
@@ -9919,7 +9935,9 @@ dtrace_ecb_create_enable(dtrace_probe_t *probe, void *arg)
 	if ((ecb = dtrace_ecb_create(state, probe, enab)) == NULL)
 		return (DTRACE_MATCH_DONE);
 
-	dtrace_ecb_enable(ecb);
+	if (dtrace_ecb_enable(ecb) < 0)
+		return (DTRACE_MATCH_FAIL);
+
 	return (DTRACE_MATCH_NEXT);
 }
 
@@ -10714,7 +10732,7 @@ static int
 dtrace_enabling_match(dtrace_enabling_t *enab, int *nmatched)
 {
 	int i = 0;
-	int matched = 0;
+	int total_matched = 0, matched = 0;
 
 	ASSERT(MUTEX_HELD(&cpu_lock));
 	ASSERT(MUTEX_HELD(&dtrace_lock));
@@ -10725,7 +10743,14 @@ dtrace_enabling_match(dtrace_enabling_t *enab, int *nmatched)
 		enab->dten_current = ep;
 		enab->dten_error = 0;
 
-		matched += dtrace_probe_enable(&ep->dted_probe, enab);
+		/*
+		 * If a provider failed to enable a probe then get out and
+		 * let the consumer know we failed.
+		 */
+		if ((matched = dtrace_probe_enable(&ep->dted_probe, enab)) < 0)
+			return (EBUSY);
+
+		total_matched += matched;
 
 		if (enab->dten_error != 0) {
 			/*
@@ -10753,7 +10778,7 @@ dtrace_enabling_match(dtrace_enabling_t *enab, int *nmatched)
 
 	enab->dten_probegen = dtrace_probegen;
 	if (nmatched != NULL)
-		*nmatched = matched;
+		*nmatched = total_matched;
 
 	return (0);
 }
diff --git a/usr/src/uts/common/dtrace/fasttrap.c b/usr/src/uts/common/dtrace/fasttrap.c
index fee6d60a57..d25e85e6b9 100644
--- a/usr/src/uts/common/dtrace/fasttrap.c
+++ b/usr/src/uts/common/dtrace/fasttrap.c
@@ -20,7 +20,7 @@
  */
 
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -875,7 +875,7 @@ fasttrap_disable_callbacks(void)
 }
 
 /*ARGSUSED*/
-static void
+static int
 fasttrap_pid_enable(void *arg, dtrace_id_t id, void *parg)
 {
 	fasttrap_probe_t *probe = parg;
@@ -903,7 +903,7 @@ fasttrap_pid_enable(void *arg, dtrace_id_t id, void *parg)
 	 * provider can't go away while we're in this code path.
 	 */
 	if (probe->ftp_prov->ftp_retired)
-		return;
+		return (0);
 
 	/*
 	 * If we can't find the process, it may be that we're in the context of
@@ -912,7 +912,7 @@ fasttrap_pid_enable(void *arg, dtrace_id_t id, void *parg)
 	 */
 	if ((p = sprlock(probe->ftp_pid)) == NULL) {
 		if ((curproc->p_flag & SFORKING) == 0)
-			return;
+			return (0);
 
 		mutex_enter(&pidlock);
 		p = prfind(probe->ftp_pid);
@@ -974,7 +974,7 @@ fasttrap_pid_enable(void *arg, dtrace_id_t id, void *parg)
 			 * drop our reference on the trap table entry.
 			 */
 			fasttrap_disable_callbacks();
-			return;
+			return (0);
 		}
 	}
 
@@ -982,6 +982,7 @@ fasttrap_pid_enable(void *arg, dtrace_id_t id, void *parg)
 	sprunlock(p);
 
 	probe->ftp_enabled = 1;
+	return (0);
 }
 
 /*ARGSUSED*/
diff --git a/usr/src/uts/common/dtrace/lockstat.c b/usr/src/uts/common/dtrace/lockstat.c
index 55b3fcf8ff..69c8b72544 100644
--- a/usr/src/uts/common/dtrace/lockstat.c
+++ b/usr/src/uts/common/dtrace/lockstat.c
@@ -19,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -83,7 +83,7 @@ static kmutex_t		lockstat_test;	/* for testing purposes only */
 static dtrace_provider_id_t lockstat_id;
 
 /*ARGSUSED*/
-static void
+static int
 lockstat_enable(void *arg, dtrace_id_t id, void *parg)
 {
 	lockstat_probe_t *probe = parg;
@@ -102,6 +102,7 @@ lockstat_enable(void *arg, dtrace_id_t id, void *parg)
 	 */
 	mutex_enter(&lockstat_test);
 	mutex_exit(&lockstat_test);
+	return (0);
 }
 
 /*ARGSUSED*/
diff --git a/usr/src/uts/common/dtrace/profile.c b/usr/src/uts/common/dtrace/profile.c
index da8f58a378..c1a2d1f1c1 100644
--- a/usr/src/uts/common/dtrace/profile.c
+++ b/usr/src/uts/common/dtrace/profile.c
@@ -19,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -360,7 +360,7 @@ profile_offline(void *arg, cpu_t *cpu, void *oarg)
 }
 
 /*ARGSUSED*/
-static void
+static int
 profile_enable(void *arg, dtrace_id_t id, void *parg)
 {
 	profile_probe_t *prof = parg;
@@ -390,6 +390,7 @@ profile_enable(void *arg, dtrace_id_t id, void *parg)
 	} else {
 		prof->prof_cyclic = cyclic_add_omni(&omni);
 	}
+	return (0);
 }
 
 /*ARGSUSED*/
diff --git a/usr/src/uts/common/dtrace/systrace.c b/usr/src/uts/common/dtrace/systrace.c
index fe7bee1ac8..b864041c45 100644
--- a/usr/src/uts/common/dtrace/systrace.c
+++ b/usr/src/uts/common/dtrace/systrace.c
@@ -19,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -140,7 +140,7 @@ systrace_destroy(void *arg, dtrace_id_t id, void *parg)
 }
 
 /*ARGSUSED*/
-static void
+static int
 systrace_enable(void *arg, dtrace_id_t id, void *parg)
 {
 	int sysnum = SYSTRACE_SYSNUM((uintptr_t)parg);
@@ -161,7 +161,7 @@ systrace_enable(void *arg, dtrace_id_t id, void *parg)
 
 	if (enabled) {
 		ASSERT(sysent[sysnum].sy_callc == dtrace_systrace_syscall);
-		return;
+		return (0);
 	}
 
 	(void) casptr(&sysent[sysnum].sy_callc,
@@ -172,6 +172,7 @@ systrace_enable(void *arg, dtrace_id_t id, void *parg)
 	    (void *)systrace_sysent32[sysnum].stsy_underlying,
 	    (void *)dtrace_systrace_syscall32);
 #endif
+	return (0);
 }
 
 /*ARGSUSED*/
diff --git a/usr/src/uts/common/io/cpc.c b/usr/src/uts/common/io/cpc.c
index fbc0b77f92..6881380251 100644
--- a/usr/src/uts/common/io/cpc.c
+++ b/usr/src/uts/common/io/cpc.c
@@ -19,11 +19,10 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
-
 /*
  * CPU Performance Counter system calls and device driver.
  *
@@ -126,7 +125,7 @@ cpc(int cmd, id_t lwpid, void *udata1, void *udata2, void *udata3)
 		 */
 
 		rw_enter(&kcpc_cpuctx_lock, RW_READER);
-		if (kcpc_cpuctx) {
+		if (kcpc_cpuctx || dtrace_cpc_in_use) {
 			rw_exit(&kcpc_cpuctx_lock);
 			return (set_errno(EAGAIN));
 		}
@@ -486,6 +485,15 @@ kcpc_open(dev_t *dev, int flags, int otyp, cred_t *cr)
 	rw_enter(&kcpc_cpuctx_lock, RW_WRITER);
 	if (++kcpc_cpuctx == 1) {
 		ASSERT(kcpc_cpumap == NULL);
+
+		/*
+		 * Bail out if DTrace is already using the counters.
+		 */
+		if (dtrace_cpc_in_use) {
+			kcpc_cpuctx--;
+			rw_exit(&kcpc_cpuctx_lock);
+			return (EAGAIN);
+		}
 		kcpc_cpumap = kmem_zalloc(BT_SIZEOFMAP(max_cpuid + 1),
 		    KM_SLEEP);
 		/*
diff --git a/usr/src/uts/common/os/dtrace_subr.c b/usr/src/uts/common/os/dtrace_subr.c
index 6c0b7d3369..f2a9ac1b7d 100644
--- a/usr/src/uts/common/os/dtrace_subr.c
+++ b/usr/src/uts/common/os/dtrace_subr.c
@@ -20,12 +20,10 @@
  */
 
 /*
- * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
-#pragma ident	"%Z%%M%	%I%	%E% SMI"
-
 #include <sys/dtrace.h>
 #include <sys/cmn_err.h>
 #include <sys/tnf.h>
@@ -45,6 +43,7 @@ void (*dtrace_helpers_cleanup)(void);
 void (*dtrace_helpers_fork)(proc_t *, proc_t *);
 void (*dtrace_cpustart_init)(void);
 void (*dtrace_cpustart_fini)(void);
+void (*dtrace_cpc_fire)(uint64_t);
 
 void (*dtrace_debugger_init)(void);
 void (*dtrace_debugger_fini)(void);
@@ -52,6 +51,20 @@ void (*dtrace_debugger_fini)(void);
 dtrace_vtime_state_t dtrace_vtime_active = 0;
 dtrace_cacheid_t dtrace_predcache_id = DTRACE_CACHEIDNONE + 1;
 
+/*
+ * dtrace_cpc_in_use usage statement: this global variable is used by the cpc
+ * hardware overflow interrupt handler and the kernel cpc framework to check
+ * whether or not the DTrace cpc provider is currently in use. The variable is
+ * set before counters are enabled with the first enabling and cleared when
+ * the last enabling is disabled. Its value at any given time indicates the
+ * number of active dcpc based enablings. The global 'kcpc_cpuctx_lock' rwlock
+ * is held during initial setting to protect races between kcpc_open() and the
+ * first enabling. The locking provided by the DTrace subsystem, the kernel
+ * cpc framework and the cpu management framework protect consumers from race
+ * conditions on enabling and disabling probes.
+ */
+uint32_t dtrace_cpc_in_use = 0;
+
 typedef struct dtrace_hrestime {
 	lock_t		dthr_lock;		/* lock for this element */
 	timestruc_t	dthr_hrestime;		/* hrestime value */
diff --git a/usr/src/uts/common/os/kcpc.c b/usr/src/uts/common/os/kcpc.c
index 8c9a135243..e5cab151b8 100644
--- a/usr/src/uts/common/os/kcpc.c
+++ b/usr/src/uts/common/os/kcpc.c
@@ -20,7 +20,7 @@
  */
 
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -65,24 +65,16 @@ static uint32_t kcpc_intrctx_count;    /* # overflows in an interrupt handler */
 static uint32_t kcpc_nullctx_count;    /* # overflows in a thread with no ctx */
 
 /*
- * Is misbehaviour (overflow in a thread with no context) fatal?
+ * By setting 'kcpc_nullctx_panic' to 1, any overflow interrupts in a thread
+ * with no valid context will result in a panic.
  */
-#ifdef DEBUG
-static int kcpc_nullctx_panic = 1;
-#else
 static int kcpc_nullctx_panic = 0;
-#endif
 
 static void kcpc_lwp_create(kthread_t *t, kthread_t *ct);
 static void kcpc_restore(kcpc_ctx_t *ctx);
 static void kcpc_save(kcpc_ctx_t *ctx);
 static void kcpc_free(kcpc_ctx_t *ctx, int isexec);
-static int kcpc_configure_reqs(kcpc_ctx_t *ctx, kcpc_set_t *set, int *subcode);
-static void kcpc_free_configs(kcpc_set_t *set);
-static kcpc_ctx_t *kcpc_ctx_alloc(void);
 static void kcpc_ctx_clone(kcpc_ctx_t *ctx, kcpc_ctx_t *cctx);
-static void kcpc_ctx_free(kcpc_ctx_t *ctx);
-static int kcpc_assign_reqs(kcpc_set_t *set, kcpc_ctx_t *ctx);
 static int kcpc_tryassign(kcpc_set_t *set, int starting_req, int *scratch);
 static kcpc_set_t *kcpc_dup_set(kcpc_set_t *set);
 
@@ -93,6 +85,18 @@ kcpc_register_pcbe(pcbe_ops_t *ops)
 	cpc_ncounters = pcbe_ops->pcbe_ncounters();
 }
 
+void
+kcpc_register_dcpc(void (*func)(uint64_t))
+{
+	dtrace_cpc_fire = func;
+}
+
+void
+kcpc_unregister_dcpc(void)
+{
+	dtrace_cpc_fire = NULL;
+}
+
 int
 kcpc_bind_cpu(kcpc_set_t *set, processorid_t cpuid, int *subcode)
 {
@@ -272,7 +276,7 @@ kcpc_bind_thread(kcpc_set_t *set, kthread_t *t, int *subcode)
  * Walk through each request in the set and ask the PCBE to configure a
  * corresponding counter.
  */
-static int
+int
 kcpc_configure_reqs(kcpc_ctx_t *ctx, kcpc_set_t *set, int *subcode)
 {
 	int		i;
@@ -327,7 +331,7 @@ kcpc_configure_reqs(kcpc_ctx_t *ctx, kcpc_set_t *set, int *subcode)
 	return (0);
 }
 
-static void
+void
 kcpc_free_configs(kcpc_set_t *set)
 {
 	int i;
@@ -710,7 +714,7 @@ kcpc_next_config(void *token, void *current, uint64_t **data)
 }
 
 
-static kcpc_ctx_t *
+kcpc_ctx_t *
 kcpc_ctx_alloc(void)
 {
 	kcpc_ctx_t	*ctx;
@@ -790,7 +794,7 @@ kcpc_ctx_clone(kcpc_ctx_t *ctx, kcpc_ctx_t *cctx)
 }
 
 
-static void
+void
 kcpc_ctx_free(kcpc_ctx_t *ctx)
 {
 	kcpc_ctx_t	**loc;
@@ -881,9 +885,33 @@ kcpc_overflow_intr(caddr_t arg, uint64_t bitmap)
 		 * or something went terribly wrong i.e. we ended up
 		 * running a passivated interrupt thread, a kernel
 		 * thread or we interrupted idle, all of which are Very Bad.
+		 *
+		 * We also could end up here owing to an incredibly unlikely
+		 * race condition that exists on x86 based architectures when
+		 * the cpc provider is in use; overflow interrupts are directed
+		 * to the cpc provider if the 'dtrace_cpc_in_use' variable is
+		 * set when we enter the handler. This variable is unset after
+		 * overflow interrupts have been disabled on all CPUs and all
+		 * contexts have been torn down. To stop interrupts, the cpc
+		 * provider issues a xcall to the remote CPU before it tears
+		 * down that CPUs context. As high priority xcalls, on an x86
+		 * architecture, execute at a higher PIL than this handler, it
+		 * is possible (though extremely unlikely) that the xcall could
+		 * interrupt the overflow handler before the handler has
+		 * checked the 'dtrace_cpc_in_use' variable, stop the counters,
+		 * return to the cpc provider which could then rip down
+		 * contexts and unset 'dtrace_cpc_in_use' *before* the CPUs
+		 * overflow handler has had a chance to check the variable. In
+		 * that case, the handler would direct the overflow into this
+		 * code and no valid context will be found. The default behavior
+		 * when no valid context is found is now to shout a warning to
+		 * the console and bump the 'kcpc_nullctx_count' variable.
 		 */
 		if (kcpc_nullctx_panic)
 			panic("null cpc context, thread %p", (void *)t);
+
+		cmn_err(CE_WARN,
+		    "null cpc context found in overflow handler!\n");
 		atomic_add_32(&kcpc_nullctx_count, 1);
 	} else if ((ctx->kc_flags & KCPC_CTX_INVALID) == 0) {
 		/*
@@ -925,8 +953,9 @@ kcpc_overflow_intr(caddr_t arg, uint64_t bitmap)
 uint_t
 kcpc_hw_overflow_intr(caddr_t arg1, caddr_t arg2)
 {
-	kcpc_ctx_t	*ctx;
-	uint64_t	bitmap;
+	kcpc_ctx_t *ctx;
+	uint64_t bitmap;
+	uint8_t *state;
 
 	if (pcbe_ops == NULL ||
 	    (bitmap = pcbe_ops->pcbe_overflow_bitmap()) == 0)
@@ -937,8 +966,53 @@ kcpc_hw_overflow_intr(caddr_t arg1, caddr_t arg2)
 	 */
 	pcbe_ops->pcbe_allstop();
 
+	if (dtrace_cpc_in_use) {
+		state = &cpu_core[CPU->cpu_id].cpuc_dcpc_intr_state;
+
+		/*
+		 * Set the per-CPU state bit to indicate that we are currently
+		 * processing an interrupt if it is currently free. Drop the
+		 * interrupt if the state isn't free (i.e. a configuration
+		 * event is taking place).
+		 */
+		if (atomic_cas_8(state, DCPC_INTR_FREE,
+		    DCPC_INTR_PROCESSING) == DCPC_INTR_FREE) {
+			int i;
+			kcpc_request_t req;
+
+			ASSERT(dtrace_cpc_fire != NULL);
+
+			(*dtrace_cpc_fire)(bitmap);
+
+			ctx = curthread->t_cpu->cpu_cpc_ctx;
+
+			/* Reset any counters that have overflowed */
+			for (i = 0; i < ctx->kc_set->ks_nreqs; i++) {
+				req = ctx->kc_set->ks_req[i];
+
+				if (bitmap & (1 << req.kr_picnum)) {
+					pcbe_ops->pcbe_configure(req.kr_picnum,
+					    req.kr_event, req.kr_preset,
+					    req.kr_flags, req.kr_nattrs,
+					    req.kr_attr, &(req.kr_config),
+					    (void *)ctx);
+				}
+			}
+			pcbe_ops->pcbe_program(ctx);
+
+			/*
+			 * We've finished processing the interrupt so set
+			 * the state back to free.
+			 */
+			cpu_core[CPU->cpu_id].cpuc_dcpc_intr_state =
+			    DCPC_INTR_FREE;
+			membar_producer();
+		}
+		return (DDI_INTR_CLAIMED);
+	}
+
 	/*
-	 * Invoke the "generic" handler.
+	 * DTrace isn't involved so pass on accordingly.
 	 *
 	 * If the interrupt has occurred in the context of an lwp owning
 	 * the counters, then the handler posts an AST to the lwp to
@@ -1441,7 +1515,7 @@ kcpc_passivate(void)
  * Returns 0 if successful, -1 on failure.
  */
 /*ARGSUSED*/
-static int
+int
 kcpc_assign_reqs(kcpc_set_t *set, kcpc_ctx_t *ctx)
 {
 	int i;
@@ -1616,3 +1690,33 @@ kcpc_pcbe_tryload(const char *prefix, uint_t first, uint_t second, uint_t third)
 	return (modload_qualified("pcbe",
 	    "pcbe", prefix, ".", s, 3, NULL) < 0 ? -1 : 0);
 }
+
+char *
+kcpc_list_attrs(void)
+{
+	ASSERT(pcbe_ops != NULL);
+
+	return (pcbe_ops->pcbe_list_attrs());
+}
+
+char *
+kcpc_list_events(uint_t pic)
+{
+	ASSERT(pcbe_ops != NULL);
+
+	return (pcbe_ops->pcbe_list_events(pic));
+}
+
+uint_t
+kcpc_pcbe_capabilities(void)
+{
+	ASSERT(pcbe_ops != NULL);
+
+	return (pcbe_ops->pcbe_caps);
+}
+
+int
+kcpc_pcbe_loaded(void)
+{
+	return (pcbe_ops == NULL ? -1 : 0);
+}
diff --git a/usr/src/uts/common/sys/cpc_impl.h b/usr/src/uts/common/sys/cpc_impl.h
index bb9dc5e6e3..f9f6b4fdfc 100644
--- a/usr/src/uts/common/sys/cpc_impl.h
+++ b/usr/src/uts/common/sys/cpc_impl.h
@@ -19,15 +19,13 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
 #ifndef	_SYS_CPC_IMPL_H
 #define	_SYS_CPC_IMPL_H
 
-#pragma ident	"%Z%%M%	%I%	%E% SMI"
-
 #include <sys/types.h>
 #include <sys/time.h>
 #include <sys/ksynch.h>
@@ -181,6 +179,27 @@ typedef struct __cpc_args32 {
  */
 #define	KCPC_PIC_OVERFLOWED	0x1	/* pic overflowed & requested notify */
 
+/*
+ * The following flags are used by the DTrace CPU performance counter provider
+ * and the overflow handler. The 'DCPC_INTR_*' flags are used to synchronize
+ * performance counter configuration events performed by the cpc provider and
+ * interrupt processing carried out by the overflow handler.  The 'DCPC_?MASK'
+ * flags are used by the dcpc provider to indicate which type of mask attribute
+ * a platform supports.
+ */
+
+/* No configuration events or overflow interrupts are currently in process. */
+#define	DCPC_INTR_FREE 0
+
+/* An overflow interrupt is currently being processed. */
+#define	DCPC_INTR_PROCESSING 1
+
+/* The cpc subsystem is currently being configured by the dcpc provider. */
+#define	DCPC_INTR_CONFIG 2
+
+#define	DCPC_UMASK 1	/* The platform supports a "umask" attribute. */
+#define	DCPC_EMASK 2	/* The platform supports an "emask" attribute. */
+
 #ifdef __sparc
 extern uint64_t ultra_gettick(void);
 #define	KCPC_GET_TICK ultra_gettick
@@ -204,6 +223,14 @@ extern void kcpc_invalidate_all(void);
 extern void kcpc_passivate(void);
 extern void kcpc_remote_stop(struct cpu *cp);
 extern int kcpc_pcbe_tryload(const char *, uint_t, uint_t, uint_t);
+extern void kcpc_remote_program(struct cpu *cp);
+extern void kcpc_register_dcpc(void (*func)(uint64_t));
+extern void kcpc_unregister_dcpc(void);
+extern kcpc_ctx_t *kcpc_ctx_alloc(void);
+extern int kcpc_assign_reqs(struct _kcpc_set *, kcpc_ctx_t *);
+extern void kcpc_ctx_free(kcpc_ctx_t *);
+extern int kcpc_configure_reqs(kcpc_ctx_t *, struct _kcpc_set *, int *);
+extern void kcpc_free_configs(struct _kcpc_set *);
 
 #endif /* _KERNEL */
 
diff --git a/usr/src/uts/common/sys/cpuvar.h b/usr/src/uts/common/sys/cpuvar.h
index 69e324ffa6..2d056fa6ab 100644
--- a/usr/src/uts/common/sys/cpuvar.h
+++ b/usr/src/uts/common/sys/cpuvar.h
@@ -20,7 +20,7 @@
  */
 
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -211,6 +211,9 @@ typedef struct cpu {
 	uint64_t	cpu_curr_clock;		/* current clock freq in Hz */
 	char		*cpu_supp_freqs;	/* supported freqs in Hz */
 
+	uintptr_t	cpu_cpcprofile_pc;	/* kernel PC in cpc interrupt */
+	uintptr_t	cpu_cpcprofile_upc;	/* user PC in cpc interrupt */
+
 	/*
 	 * Interrupt load factor used by dispatcher & softcall
 	 */
@@ -240,12 +243,13 @@ typedef struct cpu {
  * is up to the platform to assure that this is performed properly.  Note that
  * the structure is sized to avoid false sharing.
  */
-#define	CPUC_SIZE		(sizeof (uint16_t) + sizeof (uintptr_t) + \
-				sizeof (kmutex_t))
+#define	CPUC_SIZE		(sizeof (uint16_t) + sizeof (uint8_t) + \
+				sizeof (uintptr_t) + sizeof (kmutex_t))
 #define	CPUC_PADSIZE		CPU_CACHE_COHERENCE_SIZE - CPUC_SIZE
 
 typedef struct cpu_core {
 	uint16_t	cpuc_dtrace_flags;	/* DTrace flags */
+	uint8_t		cpuc_dcpc_intr_state;	/* DCPC provider intr state */
 	uint8_t		cpuc_pad[CPUC_PADSIZE];	/* padding */
 	uintptr_t	cpuc_dtrace_illval;	/* DTrace illegal value */
 	kmutex_t	cpuc_pid_lock;		/* DTrace pid provider lock */
@@ -715,7 +719,8 @@ typedef enum {
 	CPU_ON,
 	CPU_OFF,
 	CPU_CPUPART_IN,
-	CPU_CPUPART_OUT
+	CPU_CPUPART_OUT,
+	CPU_SETUP
 } cpu_setup_t;
 
 typedef int cpu_setup_func_t(cpu_setup_t, int, void *);
diff --git a/usr/src/uts/common/sys/dtrace.h b/usr/src/uts/common/sys/dtrace.h
index b6e52ec1c4..046dd13dd4 100644
--- a/usr/src/uts/common/sys/dtrace.h
+++ b/usr/src/uts/common/sys/dtrace.h
@@ -20,15 +20,13 @@
  */
 
 /*
- * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
 #ifndef _SYS_DTRACE_H
 #define	_SYS_DTRACE_H
 
-#pragma ident	"%Z%%M%	%I%	%E% SMI"
-
 #ifdef	__cplusplus
 extern "C" {
 #endif
@@ -1382,7 +1380,7 @@ typedef struct dof_helper {
  *   dtps_provide_module(); see "Arguments and Notes" for dtrace_register(),
  *   below.
  *
- * 1.4  void dtps_enable(void *arg, dtrace_id_t id, void *parg)
+ * 1.4  int dtps_enable(void *arg, dtrace_id_t id, void *parg)
  *
  * 1.4.1  Overview
  *
@@ -1403,7 +1401,8 @@ typedef struct dof_helper {
  *
  * 1.4.3  Return value
  *
- *   None.
+ *   On success, dtps_enable() should return 0. On failure, -1 should be
+ *   returned.
  *
  * 1.4.4  Caller's context
  *
@@ -1957,7 +1956,7 @@ typedef struct dof_helper {
 typedef struct dtrace_pops {
 	void (*dtps_provide)(void *arg, const dtrace_probedesc_t *spec);
 	void (*dtps_provide_module)(void *arg, struct modctl *mp);
-	void (*dtps_enable)(void *arg, dtrace_id_t id, void *parg);
+	int (*dtps_enable)(void *arg, dtrace_id_t id, void *parg);
 	void (*dtps_disable)(void *arg, dtrace_id_t id, void *parg);
 	void (*dtps_suspend)(void *arg, dtrace_id_t id, void *parg);
 	void (*dtps_resume)(void *arg, dtrace_id_t id, void *parg);
diff --git a/usr/src/uts/common/sys/kcpc.h b/usr/src/uts/common/sys/kcpc.h
index 0763723bdf..f30e093f78 100644
--- a/usr/src/uts/common/sys/kcpc.h
+++ b/usr/src/uts/common/sys/kcpc.h
@@ -19,15 +19,13 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
 #ifndef _SYS_KCPC_H
 #define	_SYS_KCPC_H
 
-#pragma ident	"%Z%%M%	%I%	%E% SMI"
-
 #include <sys/cpc_impl.h>
 #include <sys/ksynch.h>
 
@@ -130,11 +128,24 @@ extern void kcpc_idle_restore(struct cpu *cp);
 extern krwlock_t	kcpc_cpuctx_lock;  /* lock for 'kcpc_cpuctx' below */
 extern int		kcpc_cpuctx;	   /* number of cpu-specific contexts */
 
+/*
+ * 'dtrace_cpc_in_use' contains the number of currently active cpc provider
+ * based enablings. See the block comment in uts/common/os/dtrace_subr.c for
+ * details of its actual usage.
+ */
+extern uint32_t		dtrace_cpc_in_use;
+extern void (*dtrace_cpc_fire)(uint64_t);
+
 extern void kcpc_free_set(kcpc_set_t *set);
 
 extern void *kcpc_next_config(void *token, void *current,
     uint64_t **data);
 extern void kcpc_invalidate_config(void *token);
+extern char *kcpc_list_attrs(void);
+extern char *kcpc_list_events(uint_t pic);
+extern void kcpc_free_configs(kcpc_set_t *set);
+extern uint_t kcpc_pcbe_capabilities(void);
+extern int kcpc_pcbe_loaded(void);
 
 /*
  * Called by a PCBE to determine if nonprivileged access to counters should be
diff --git a/usr/src/uts/common/xen/dtrace/xdt.c b/usr/src/uts/common/xen/dtrace/xdt.c
index bac7f100e9..85c876b479 100644
--- a/usr/src/uts/common/xen/dtrace/xdt.c
+++ b/usr/src/uts/common/xen/dtrace/xdt.c
@@ -20,7 +20,7 @@
  */
 
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -782,7 +782,7 @@ xdt_set_trace_mask(uint32_t mask)
 }
 
 /*ARGSUSED*/
-static void
+static int
 xdt_enable(void *arg, dtrace_id_t id, void *parg)
 {
 	xdt_probe_t *p = parg;
@@ -801,22 +801,15 @@ xdt_enable(void *arg, dtrace_id_t id, void *parg)
 	}
 
 	if (xdt_cyclic == CYCLIC_NONE) {
-		/*
-		 * DTrace doesn't have the notion of failing an enabling. It
-		 * works on the premise that, if you have advertised a probe
-		 * via the pops->dtps_provide() function, you can enable it.
-		 * Failure is not an option. In the case where we can't enable
-		 * Xen tracing the consumer will carry on regardless and
-		 * think all is OK except the probes will never fire.
-		 */
 		tbuf_op.cmd = XEN_SYSCTL_TBUFOP_enable;
 		if (xdt_sysctl_tbuf(&tbuf_op) != 0) {
 			cmn_err(CE_NOTE, "Couldn't enable hypervisor tracing.");
-			return;
+			return (-1);
 		}
 
 		xdt_cyclic_enable();
 	}
+	return (0);
 }
 
 /*ARGSUSED*/