diff options
Diffstat (limited to 'usr/src')
| -rw-r--r-- | usr/src/pkg/manifests/system-kernel-platform.mf | 2 | ||||
| -rw-r--r-- | usr/src/uts/sun4u/Makefile.files | 1 | ||||
| -rw-r--r-- | usr/src/uts/sun4u/Makefile.sun4u | 1 | ||||
| -rw-r--r-- | usr/src/uts/sun4u/daktari/os/daktari.c | 4 | ||||
| -rw-r--r-- | usr/src/uts/sun4u/io/zuluvm.c | 1495 | ||||
| -rw-r--r-- | usr/src/uts/sun4u/ml/zulu_asm.s | 325 | ||||
| -rw-r--r-- | usr/src/uts/sun4u/ml/zulu_hat_asm.s | 314 | ||||
| -rw-r--r-- | usr/src/uts/sun4u/sys/zulu_hat.h | 214 | ||||
| -rw-r--r-- | usr/src/uts/sun4u/sys/zulumod.h | 262 | ||||
| -rw-r--r-- | usr/src/uts/sun4u/sys/zuluvm.h | 121 | ||||
| -rw-r--r-- | usr/src/uts/sun4u/vm/zulu_hat.c | 1469 | ||||
| -rw-r--r-- | usr/src/uts/sun4u/zuluvm/Makefile | 133 | ||||
| -rw-r--r-- | usr/src/uts/sun4u/zuluvm/zuluvm_offsets.in | 77 |
13 files changed, 3 insertions, 4415 deletions
diff --git a/usr/src/pkg/manifests/system-kernel-platform.mf b/usr/src/pkg/manifests/system-kernel-platform.mf index 17394600c4..ccaa252a95 100644 --- a/usr/src/pkg/manifests/system-kernel-platform.mf +++ b/usr/src/pkg/manifests/system-kernel-platform.mf @@ -1081,8 +1081,6 @@ $(sparc_ONLY)file path=platform/sun4u/kernel/misc/$(ARCH64)/sbd group=sys \ mode=0755 $(sparc_ONLY)file path=platform/sun4u/kernel/misc/$(ARCH64)/vis group=sys \ mode=0755 -$(sparc_ONLY)file path=platform/sun4u/kernel/misc/$(ARCH64)/zuluvm group=sys \ - mode=0755 $(sparc_ONLY)file path=platform/sun4u/kernel/strmod/$(ARCH64)/kb group=sys \ mode=0755 $(sparc_ONLY)file path=platform/sun4u/kernel/tod/$(ARCH64)/todblade group=sys \ diff --git a/usr/src/uts/sun4u/Makefile.files b/usr/src/uts/sun4u/Makefile.files index 96021facfb..ece838c2be 100644 --- a/usr/src/uts/sun4u/Makefile.files +++ b/usr/src/uts/sun4u/Makefile.files @@ -135,7 +135,6 @@ TRAPSTAT_OBJS += trapstat.o I2BSC_OBJS += i2bsc.o GPTWOCFG_OBJS += gptwocfg.o GPTWO_CPU_OBJS += gptwo_cpu.o -ZULUVM_OBJS += zuluvm.o zulu_asm.o zulu_hat.o zulu_hat_asm.o JBUSPPM_OBJS += jbusppm.o RMC_COMM_OBJS += rmc_comm.o rmc_comm_crctab.o rmc_comm_dp.o rmc_comm_drvintf.o diff --git a/usr/src/uts/sun4u/Makefile.sun4u b/usr/src/uts/sun4u/Makefile.sun4u index 86ccfa622b..4a68d62163 100644 --- a/usr/src/uts/sun4u/Makefile.sun4u +++ b/usr/src/uts/sun4u/Makefile.sun4u @@ -428,7 +428,6 @@ MISC_KMODS += obpsym bootdev vis cpr platmod md5 sha1 i2c_svc MISC_KMODS += sbd MISC_KMODS += opl_cfg -MISC_KMODS += zuluvm MISC_KMODS += gptwo_cpu gptwocfg MISC_KMODS += pcie diff --git a/usr/src/uts/sun4u/daktari/os/daktari.c b/usr/src/uts/sun4u/daktari/os/daktari.c index ecfe08b4c2..f522c7b26b 100644 --- a/usr/src/uts/sun4u/daktari/os/daktari.c +++ b/usr/src/uts/sun4u/daktari/os/daktari.c @@ -575,7 +575,9 @@ plat_get_cpu_unum(int cpuid, char *buf, int buflen, int *lenp) } /* - * The zuluvm module requires a dmv interrupt for each installed zulu board. + * The zuluvm module required a dmv interrupt for each installed + * Zulu/XVR-4000 board. The following has not been updated during the + * removal of zuluvm and therefore it may be suboptimal. */ void plat_dmv_params(uint_t *hwint, uint_t *swint) diff --git a/usr/src/uts/sun4u/io/zuluvm.c b/usr/src/uts/sun4u/io/zuluvm.c deleted file mode 100644 index f1184f4580..0000000000 --- a/usr/src/uts/sun4u/io/zuluvm.c +++ /dev/null @@ -1,1495 +0,0 @@ -/* - * CDDL HEADER START - * - * The contents of this file are subject to the terms of the - * Common Development and Distribution License (the "License"). - * You may not use this file except in compliance with the License. - * - * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE - * or http://www.opensolaris.org/os/licensing. - * See the License for the specific language governing permissions - * and limitations under the License. - * - * When distributing Covered Code, include this CDDL HEADER in each - * file and include the License file at usr/src/OPENSOLARIS.LICENSE. - * If applicable, add the following below this CDDL HEADER, with the - * fields enclosed by brackets "[]" replaced with your own identifying - * information: Portions Copyright [yyyy] [name of copyright owner] - * - * CDDL HEADER END - */ -/* - * Copyright 2006 Sun Microsystems, Inc. All rights reserved. - * Use is subject to license terms. - */ - -/* - * zuluvm module - * - * Provides services required by the XVR-4000 graphics accelerator (zulu) - * that are not provided by the ddi. See PSARC 2002/231. - * - * Zulu has 2 dma engines with built in MMUs. zuluvm provides TLB miss - * interrupt support obtaining virtual to physical address translations - * using the XHAT interface PSARC/2003/517. - * - * The module has 3 components. This file, sun4u/vm/zulu_hat.c, and the - * assembly language routines in sun4u/ml/zulu_asm.s and - * sun4u/ml/zulu_hat_asm.s. - * - * The interrupt handler is a data bearing mondo interrupt handled at TL=1 - * If no translation is found in the zulu hat's tsb, or if the tsb is locked by - * C code, the handler posts a soft interrupt which wakes up a parked - * thread belonging to zuludaemon(1M). - */ - -#include <sys/conf.h> -#include <sys/types.h> -#include <sys/kmem.h> -#include <sys/debug.h> -#include <sys/modctl.h> -#include <sys/autoconf.h> -#include <sys/ddi_impldefs.h> -#include <sys/ddi_subrdefs.h> -#include <sys/intr.h> -#include <sys/ddi.h> -#include <sys/sunndi.h> -#include <sys/proc.h> -#include <sys/thread.h> -#include <sys/machsystm.h> -#include <sys/ivintr.h> -#include <sys/tnf_probe.h> -#include <sys/intreg.h> -#include <sys/atomic.h> -#include <vm/as.h> -#include <vm/seg_enum.h> -#include <vm/faultcode.h> -#include <sys/dmv.h> -#include <sys/zulumod.h> -#include <sys/zulu_hat.h> - -#define ZULUVM_GET_PAGE(val) \ - (caddr_t)((uintptr_t)(val) & PAGEMASK) -#define ZULUVM_GET_AS curthread->t_procp->p_as - -#define ZULUVM_LOCK mutex_enter(&(zdev->dev_lck)) -#define ZULUVM_UNLOCK mutex_exit(&(zdev->dev_lck)) - -#define ZULUVM_SET_STATE(_z, b, c) \ - atomic_cas_32((uint32_t *)&((_z)->zvm.state), c, b) -#define ZULUVM_GET_STATE(_z) \ - (_z)->zvm.state -#define ZULUVM_SET_IDLE(_z) \ - (_z)->zvm.state = ZULUVM_STATE_IDLE; - -#define ZULUVM_INO_MASK ((1<<INO_SIZE)-1) -#define ZULUVM_IGN_MASK ((1<<IGN_SIZE)-1) -#define ZULUVM_MONDO(_zdev, _n) \ - ((ZULUVM_IGN_MASK & _zdev->agentid) << INO_SIZE) | \ - (ZULUVM_INO_MASK & (_n)) - -static void zuluvm_stop(zuluvm_state_t *, int, char *); -static zuluvm_proc_t *zuluvm_find_proc(zuluvm_state_t *, struct as *); -static int zuluvm_proc_release(zuluvm_state_t *zdev, zuluvm_proc_t *proc); -static int zuluvm_get_intr_props(zuluvm_state_t *zdev, dev_info_t *devi); -static int zuluvm_driver_attach(zuluvm_state_t *); -static int zuluvm_driver_detach(zuluvm_state_t *); -static void zuluvm_retarget_intr(void *arg); -static void zuluvm_do_retarget(zuluvm_state_t *zdev); - -extern const unsigned int _mmu_pageshift; - -extern int zuluvm_base_pgsize; -static int zuluvm_pagesizes[ZULUM_MAX_PG_SIZES + 1]; - -int zuluvm_fast_tlb = 1; - -zuluvm_state_t *zuluvm_devtab[ZULUVM_MAX_DEV]; -kmutex_t zuluvm_lck; - -#ifdef DEBUG -int zuluvm_debug_state = 0; -#endif - -unsigned long zuluvm_ctx_locked = 0; - -/* - * Module linkage information for the kernel. - */ -extern struct mod_ops mod_miscops; - -static struct modlmisc modlmisc = { - &mod_miscops, - "sun4u support " ZULUVM_MOD_VERSION -}; - -static struct modlinkage modlinkage = { - MODREV_1, - (void *)&modlmisc, - NULL -}; - -int -_init(void) -{ - zuluvm_base_pgsize = (_mmu_pageshift - 13) / 3; - if (zulu_hat_init() != 0) { - return (ZULUVM_ERROR); - } - mutex_init(&zuluvm_lck, NULL, MUTEX_DEFAULT, NULL); - return (mod_install(&modlinkage)); -} - -int -_fini(void) -{ - mutex_destroy(&zuluvm_lck); - (void) zulu_hat_destroy(); - return (mod_remove(&modlinkage)); -} - -int -_info(struct modinfo *modinfop) -{ - return (mod_info(&modlinkage, modinfop)); -} - -/* - * currently the kernel driver makes the following assumptions: - * - there is only one TLB miss per zulu device handled at - * any given time - * ==> we only need local data storage per device, not per DMA - * ==> a page fault will block the DMA engine until the fault - * is resolved - * ==> a pagefault will not trigger a zulu DMA context switch - * - * If we want to implement asynnchronous zulu page fault, then we - * need to keep track of outstanding faults while zulu DMA runs - * in a different context. - */ -static int -zuluvm_write_tte(zuluvm_state_t *zdev, void *arg, caddr_t addr, - int t_pfn, int t_perm, int t_size, uint64_t tag, - int tlbtype, int *size) -{ - int error; - - (void) addr; - - ZULUVM_STATS_MISS(zdev, t_size); - - if (tag == 0) { /* not coming from preload */ - int state = ZULUVM_SET_STATE(zdev, ZULUVM_STATE_WRITE_TTE, - ZULUVM_STATE_INTR_PENDING); - if (state != ZULUVM_STATE_INTR_PENDING) { - zuluvm_stop(zdev, state, "zuluvm_write_tte"); - return (ZULUVM_MISS_CANCELED); - } - } - - if (!(tlbtype & ZULUVM_ITLB_FLAG) && - t_size != zuluvm_base_pgsize && - t_size != ZULU_TTE4M) { - t_size = zuluvm_base_pgsize; - TNF_PROBE_2(zuluvm_write_tte_new_pfn, "zuluvm", /* */, - tnf_opaque, t_pfn, t_pfn, tnf_int, pagesize, t_size); - } - TNF_PROBE_1(zuluvm_write_tte, "zuluvm", /* */, - tnf_opaque, t_pfn, t_pfn); - /* - * if the caller is zuluvm_preload, then we need to pass - * back the page size so it can add the right offset. - */ - if (size) - *size = t_size; - - error = zulud_write_tte(zdev, arg, t_size, tag, t_pfn, - t_perm, tlbtype); - - return (error); -} - -static void -zuluvm_stop(zuluvm_state_t *zdev, int state, char *tag) -{ - int ostate = state; - while (state != ZULUVM_STATE_STOPPED) { - state = ZULUVM_SET_STATE(zdev, - ZULUVM_STATE_STOPPED, state); -#ifdef DEBUG - if (zuluvm_debug_state) - cmn_err(CE_NOTE, "zuluvm_stop(%s): (loop) state %d\n", - tag, state); -#endif - } - TNF_PROBE_2(zuluvm_stop, "zuluvm", /* */, - tnf_string, tag, tag, - tnf_int, state, ostate); - ZULUVM_STATS_CANCEL(zdev); -} - -/* - * Executed with the context of the parked zulu deamon thread, - * uses zulu_hat_load to resolve the miss. - * The tte is loaded and miss done called by the function zuluvm_load_tte - * which is called from zulu_hat - * - * This function is synchronized with the zuluvm_as_free. - * zuluvm_as_free will block until miss servicing is complete. - * - * There is a race condition between as_free and the zulu tlb miss - * soft interrupt: - * - queue zulu interrupt - * - process dies, as_free runs - * - interrupt gets scheduled and runs as_fault on the - * already freed as. - * This is solved by keeping track of current zulu dma processes - * and invalidating them in zuluvm_as_free. - */ -uint_t -zuluvm_tlb_handler(caddr_t data) -{ - zuluvm_state_t *zdev = (zuluvm_state_t *)data; - int error; - int flag = 0; - int wait = 0; - zuluvm_proc_t *proc = NULL; - struct zulu_hat *zhat = NULL; - caddr_t addr; - int tlbtype; - void *arg; - int state, newstate; - - TNF_PROBE_1(zuluvm_tlb_handler_lwp, "zuluvm", /* */, - tnf_opaque, lwp, ttolwp(curthread)); - - ZULUVM_LOCK; - error = ZULUVM_GET_TLB_ERRCODE(zdev); - addr = (caddr_t)ZULUVM_GET_TLB_ADDR(zdev); - tlbtype = ZULUVM_GET_TLB_TYPE(zdev); - arg = zdev->zvm.arg; - - /* - * select the correct dma engine and remember the - * the as_free synchronization flags. - */ - switch (tlbtype) { - case ZULUVM_ITLB1: - case ZULUVM_DMA1: - proc = zdev->zvm.proc1; - flag |= ZULUVM_DO_INTR1; - wait |= ZULUVM_WAIT_INTR1; - break; - case ZULUVM_ITLB2: - case ZULUVM_DMA2: - proc = zdev->zvm.proc2; - flag |= ZULUVM_DO_INTR2; - wait |= ZULUVM_WAIT_INTR2; - break; - } - - state = ZULUVM_SET_STATE(zdev, ZULUVM_STATE_INTR_PENDING, - ZULUVM_STATE_INTR_QUEUED); - newstate = ZULUVM_GET_STATE(zdev); - - TNF_PROBE_2(zuluvm_tlb_handler_state, "zuluvm", /* */, - tnf_int, oldstate, state, - tnf_int, newstate, newstate); -#ifdef DEBUG - if (zuluvm_debug_state) - cmn_err(CE_NOTE, "zuluvm_tlb_handler: state %d\n", state); -#endif - if (state != ZULUVM_STATE_INTR_PENDING && - state != ZULUVM_STATE_INTR_QUEUED) { - ZULUVM_UNLOCK; - - zuluvm_stop(zdev, state, "softintr1"); - zulud_tlb_done(zdev, arg, tlbtype, ZULUVM_MISS_CANCELED); - return (1); - } - - /* - * block the as_free callback in case it comes in - */ - zdev->intr_flags |= flag; - ZULUVM_UNLOCK; - - mutex_enter(&zdev->proc_lck); - /* - * check if this as is still valid - */ - if (proc == NULL || proc->valid == 0 || proc->zhat == NULL) { - mutex_exit(&zdev->proc_lck); - /* - * we are on our way out, wake up the as_free - * callback if it is waiting for us - */ - ZULUVM_LOCK; - zdev->intr_flags &= ~flag; - if (zdev->intr_flags | wait) - cv_broadcast(&zdev->intr_wait); - ZULUVM_UNLOCK; - state = ZULUVM_SET_STATE(zdev, ZULUVM_STATE_IDLE, - ZULUVM_STATE_INTR_PENDING); - if (state != ZULUVM_STATE_INTR_PENDING) { - zuluvm_stop(zdev, state, "softintr3"); - } - zulud_tlb_done(zdev, arg, tlbtype, ZULUVM_NO_HAT); - return (1); - } - zhat = proc->zhat; - mutex_exit(&zdev->proc_lck); - - TNF_PROBE_1(zuluvm_tlb_handler, "zuluvm", /* */, - tnf_opaque, addr, addr); - - switch (error) { - case ZULUVM_CTX_LOCKED: - /* - * trap handler found that zulu_hat had the lock bit set - * rather than block in the fast trap handler, it punts - * in this rare instance - */ - ++zuluvm_ctx_locked; - TNF_PROBE_1(zuluvm_ctx_locked, "zuluvm", /* CSTYLED */, - tnf_ulong, zuluvm_ctx_locked, zuluvm_ctx_locked); - - /*FALLTHROUGH*/ - - case ZULUVM_TTE_DELAY: - /* - * fast tlb handler was skipped, see zuluvm_fast_tlb flag - */ - /*FALLTHROUGH*/ - - case ZULUVM_NO_TTE: - /* - * no TSB entry and TTE in the hash - */ - mutex_enter(&zdev->load_lck); - zdev->in_intr = 1; - error = zulu_hat_load(zhat, addr, - (tlbtype == ZULUVM_DMA2) ? S_WRITE : S_READ, NULL); - zdev->in_intr = 0; - mutex_exit(&zdev->load_lck); - if (error) { - - error = ZULUVM_NO_MAP; - } else { - error = ZULUVM_SUCCESS; - TNF_PROBE_1(zuluvm_tlb_handler_done, "zuluvm", /* */, - tnf_int, error, error); - return (1); - } - - default: - /* - * error case, fall through and tell zulu driver to abort DMA - */ - break; - } - - if (error != ZULUVM_MISS_CANCELED) { - state = ZULUVM_SET_STATE(zdev, ZULUVM_STATE_IDLE, - ZULUVM_STATE_WRITE_TTE); - newstate = ZULUVM_GET_STATE(zdev); - TNF_PROBE_2(zuluvm_tlb_handler_state_done, "zuluvm", /* */, - tnf_int, oldstate, state, - tnf_int, newstate, newstate); - if (state != ZULUVM_STATE_WRITE_TTE) { - zuluvm_stop(zdev, state, "softintr4"); - } - } - /* - * synchronize with as_free callback - * It will set the wait flag, in that case we send - * a wake up. - */ - ZULUVM_LOCK; - zdev->intr_flags &= ~flag; - if (zdev->intr_flags | wait) - cv_broadcast(&zdev->intr_wait); - ZULUVM_UNLOCK; - - TNF_PROBE_1(zuluvm_tlb_handler_done, "zuluvm", /* */, - tnf_int, error, error); - - zulud_tlb_done(zdev, arg, tlbtype, error); - - return (1); -} - - -void -zuluvm_load_tte(struct zulu_hat *zhat, caddr_t addr, uint64_t pfn, - int perm, int size) -{ - zuluvm_state_t *zdev = zhat->zdev; - int tlbtype = ZULUVM_GET_TLB_TYPE(zdev); - - ASSERT(MUTEX_HELD(&zdev->load_lck)); - ASSERT(pfn != 0); - - if (zdev->in_intr) { - int error; - int flag = 0; - int wait = 0; - - error = zuluvm_write_tte(zdev, zdev->zvm.arg, addr, pfn, - perm, size, 0, tlbtype, NULL); - - if (error != ZULUVM_MISS_CANCELED) { - int state, newstate; - - state = ZULUVM_SET_STATE(zdev, ZULUVM_STATE_IDLE, - ZULUVM_STATE_WRITE_TTE); - newstate = ZULUVM_GET_STATE(zdev); - TNF_PROBE_2(zuluvm_tlb_handler_state_done, "zuluvm", - /* */, tnf_int, oldstate, state, - tnf_int, newstate, newstate); - if (state != ZULUVM_STATE_WRITE_TTE) { - zuluvm_stop(zdev, state, "softintr4"); - } - } - /* - * synchronize with as_free callback - * It will set the wait flag, in that case we send - * a wake up. - */ - switch (tlbtype) { - case ZULUVM_ITLB1: - case ZULUVM_DMA1: - flag = ZULUVM_DO_INTR1; - wait = ZULUVM_WAIT_INTR1; - break; - case ZULUVM_ITLB2: - case ZULUVM_DMA2: - flag = ZULUVM_DO_INTR2; - wait = ZULUVM_WAIT_INTR2; - break; - } - - ZULUVM_LOCK; - zdev->intr_flags &= ~flag; - if (zdev->intr_flags | wait) - cv_broadcast(&zdev->intr_wait); - ZULUVM_UNLOCK; - - zulud_tlb_done(zdev, zdev->zvm.arg, tlbtype, error); - } else { - (void) zuluvm_write_tte(zdev, zdev->zvm.arg, addr, pfn, - perm, size, (uint64_t)addr | - zhat->zulu_ctx, tlbtype, NULL); - } -} - - - - -/* - * This function provides the faulting thread for zulu page faults - * It is call from the device driver in response to an ioctl issued - * by a zuludaemon thread. - * It sits in cv_wait_sig until it gets woken up by a signal or - * zulu tlb miss soft interrupt. - */ -int -zuluvm_park(zuluvm_info_t devp) -{ - int rval; - zuluvm_state_t *zdev = (zuluvm_state_t *)devp; - mutex_enter(&zdev->park_lck); - zdev->parking = 1; - for (;;) { - rval = cv_wait_sig(&zdev->park_cv, &zdev->park_lck); - if (rval == 0) - break; - rval = zuluvm_tlb_handler(devp); - } - zdev->parking = 0; - mutex_exit(&zdev->park_lck); - return (rval); -} - -/* - * zulu soft interrupt handler, just triggers the parked zulu fault - * thread - */ -/*ARGSUSED*/ -uint_t -zuluvm_softintr(caddr_t devp, caddr_t arg2) -{ - int tlbtype; - void *arg; - zuluvm_state_t *zdev = (zuluvm_state_t *)devp; - mutex_enter(&zdev->park_lck); - if (zdev->parking) { - cv_signal(&zdev->park_cv); - mutex_exit(&zdev->park_lck); - TNF_PROBE_1(zuluvm_fast_intr, "zuluvm", /* */, - tnf_opaque, devp, devp); - } else { - mutex_exit(&zdev->park_lck); - cmn_err(CE_NOTE, "zuluvm: no page fault thread\n"); - ZULUVM_LOCK; - tlbtype = ZULUVM_GET_TLB_TYPE(zdev); - arg = zdev->zvm.arg; - ZULUVM_UNLOCK; - TNF_PROBE_0(zuluvm_fast_intr, "zuluvm", /* */); - zuluvm_stop(zdev, ZULUVM_STATE_INTR_QUEUED, "fast_intr"); - zulud_tlb_done(zdev, arg, tlbtype, ZULUVM_NO_TTE); - } - return (1); -} - -/* ***** public interface for process mapping events (hat layer) ***** */ - -/* - * If the page size matches the Zulu page sizes then just pass - * it thru. If not then emulate the page demap with demaps of - * smaller page size. - */ -/* ARGSUSED */ -void -zuluvm_demap_page(void *arg, struct hat *hat_ptr, short ctx, - caddr_t vaddr, uint_t size) -{ - void *ddarg; - zuluvm_state_t *zdev = (zuluvm_state_t *)arg; - - if (arg == NULL) - return; - - ZULUVM_STATS_DEMAP_PAGE(zdev); - - ddarg = zdev->zvm.arg; - - TNF_PROBE_3(zuluvm_demap_page, "zuluvm", /* */, - tnf_opaque, addr, vaddr, - tnf_int, size, size, - tnf_int, ctx, ctx); - - if (ddarg != NULL) { - if (size != zuluvm_base_pgsize && - size != ZULU_TTE4M) { - int i; - int cnt = size - zuluvm_base_pgsize; - cnt = ZULU_HAT_SZ_SHIFT(cnt); - for (i = 0; i < cnt; i++) { - uintptr_t addr = (uintptr_t)vaddr | - i << ZULU_HAT_BP_SHIFT; - zulud_demap_page(zdev, ddarg, - (caddr_t)addr, ctx); - } - } else { - zulud_demap_page(zdev, ddarg, vaddr, ctx); - } - TNF_PROBE_0(zuluvm_demap_page_done, "zuluvm", /* */); - } else { - TNF_PROBE_0(zuluvm_demap_page_null_ddarg, "zuluvm", /* */); - } -} - -/* - * An entire context has gone away, just pass it thru - */ -void -zuluvm_demap_ctx(void *arg, short ctx) -{ - void *ddarg; - zuluvm_state_t *zdev = (zuluvm_state_t *)arg; - - if (arg == NULL) - return; - - ZULUVM_STATS_DEMAP_CTX(zdev); - - TNF_PROBE_1(zuluvm_demap_ctx, "zuluvm", /* */, - tnf_int, ctx, ctx); - ddarg = zdev->zvm.arg; - - if (ddarg != NULL) - zulud_demap_ctx(zdev, ddarg, ctx); -} - -static int -zuluvm_driver_attach(zuluvm_state_t *zdev) -{ - int i; - mutex_enter(&zuluvm_lck); - for (i = 0; i < ZULUVM_MAX_DEV; i++) { - if (zuluvm_devtab[i] == NULL) { - zuluvm_devtab[i] = zdev; - ZULUVM_SET_IDLE(zdev); - break; - } - } - mutex_exit(&zuluvm_lck); - if (i >= ZULUVM_MAX_DEV) - return (ZULUVM_ERROR); - - if (zulu_hat_attach((void *)zdev) != 0) { - return (ZULUVM_ERROR); - } - - mutex_init(&zdev->dev_lck, NULL, MUTEX_DEFAULT, NULL); - mutex_init(&zdev->load_lck, NULL, MUTEX_DEFAULT, NULL); - mutex_init(&zdev->proc_lck, NULL, MUTEX_DEFAULT, NULL); - mutex_init(&zdev->park_lck, NULL, MUTEX_DEFAULT, NULL); - cv_init(&zdev->park_cv, NULL, CV_DEFAULT, NULL); - cv_init(&zdev->intr_wait, NULL, CV_DEFAULT, NULL); - zdev->parking = 0; - -#ifdef ZULUVM_STATS - zdev->zvm.cancel = 0; - zdev->zvm.pagefault = 0; - zdev->zvm.no_mapping = 0; - zdev->zvm.preload = 0; - zdev->zvm.migrate = 0; - zdev->zvm.pagesize = 0; - zdev->zvm.tlb_miss[0] = 0; - zdev->zvm.tlb_miss[1] = 0; - zdev->zvm.tlb_miss[2] = 0; - zdev->zvm.tlb_miss[3] = 0; - zdev->zvm.itlb1miss = 0; - zdev->zvm.dtlb1miss = 0; - zdev->zvm.itlb2miss = 0; - zdev->zvm.dtlb2miss = 0; -#endif - zdev->zvm.pfncnt = 0; - for (i = 0; i < 50; i++) - zdev->zvm.pfnbuf[i] = 0; - - zdev->zvm.mmu_pa = NULL; - zdev->zvm.proc1 = NULL; - zdev->zvm.proc2 = NULL; - zdev->procs = NULL; - return (ZULUVM_SUCCESS); -} - -static int -zuluvm_driver_detach(zuluvm_state_t *zdev) -{ - int i; - cv_destroy(&zdev->intr_wait); - cv_destroy(&zdev->park_cv); - mutex_destroy(&zdev->park_lck); - mutex_destroy(&zdev->proc_lck); - mutex_destroy(&zdev->dev_lck); - mutex_destroy(&zdev->load_lck); - zdev->dops = NULL; - - mutex_enter(&zuluvm_lck); - for (i = 0; i < ZULUVM_MAX_DEV; i++) { - if (zuluvm_devtab[i] == zdev) { - zuluvm_devtab[i] = NULL; - break; - } - } - mutex_exit(&zuluvm_lck); - - if (zulu_hat_detach((void *)zdev) == 0) { - return (ZULUVM_SUCCESS); - } else { - return (ZULUVM_ERROR); - } -} - -zulud_ops_t *zuluvm_dops = NULL; - -/* - * init the zulu kernel driver (variables, locks, etc) - */ -int -zuluvm_init(zulud_ops_t *ops, int **pagesizes) -{ - int error = ZULUVM_SUCCESS; - int i; - int size = zuluvm_base_pgsize; /* MMU_PAGESIZE; */ - - if (ops->version != ZULUVM_INTERFACE_VERSION) - return (ZULUVM_VERSION_MISMATCH); - - zuluvm_dops = ops; - for (i = 0; i < ZULUM_MAX_PG_SIZES && size <= ZULU_TTE4M; i++) { - zuluvm_pagesizes[i] = size++; - } - zuluvm_pagesizes[i] = -1; - *pagesizes = zuluvm_pagesizes; - - return (error); -} - -/* - * cleanup afterwards - */ -int -zuluvm_fini(void) -{ - zuluvm_dops = NULL; - return (ZULUVM_SUCCESS); -} - -/* - * allocate a zulu kernel driver instance for this zulu device - */ -int -zuluvm_alloc_device(dev_info_t *devi, void *arg, zuluvm_info_t *devp, - caddr_t mmu, caddr_t imr) -{ - uint64_t intr_num; - zuluvm_state_t *zdev; - int error = ZULUVM_SUCCESS; - - TNF_PROBE_3(zuluvm_alloc_device, "zuluvm", /* */, - tnf_opaque, arg, arg, - tnf_opaque, mmu, mmu, - tnf_opaque, imr, imr); - - zdev = kmem_zalloc(sizeof (zuluvm_state_t), KM_SLEEP); - zdev->dip = devi; - zdev->dops = zuluvm_dops; - error = zuluvm_driver_attach(zdev); - if (error != ZULUVM_SUCCESS) { - kmem_free(zdev, sizeof (zuluvm_state_t)); - return (ZULUVM_NO_DEV); - } - - ZULUVM_LOCK; - error = zuluvm_get_intr_props(zdev, devi); - if (error != ZULUVM_SUCCESS) { - ZULUVM_UNLOCK; - error = zuluvm_driver_detach(zdev); - if (error != ZULUVM_SUCCESS) - return (error); - kmem_free(zdev, sizeof (zuluvm_state_t)); - return (ZULUVM_NO_DEV); - } - zdev->zvm.arg = arg; - zdev->zvm.mmu_pa = (uint64_t)va_to_pa((void *)mmu); - zdev->imr = (uint64_t *)imr; - zdev->zvm.dmv_intr = dmv_add_softintr(zuluvm_dmv_tlbmiss_tl1, - (void *)zdev); - zulud_set_itlb_pc(zdev, arg, DMV_MAKE_DMV(zdev->zvm.dmv_intr, - (void *)zdev)); - zulud_set_dtlb_pc(zdev, arg, DMV_MAKE_DMV(zdev->zvm.dmv_intr, - (void *)zdev)); - intr_dist_add(zuluvm_retarget_intr, (void *)zdev); - zuluvm_do_retarget(zdev); - intr_num = add_softintr(ZULUVM_PIL, zuluvm_softintr, - (caddr_t)zdev, SOFTINT_ST); - zdev->zvm.intr_num = intr_num; - *devp = (caddr_t)zdev; - ZULUVM_UNLOCK; - TNF_PROBE_1(zuluvm_alloc_device_done, "zuluvm", /* */, - tnf_opaque, devp, *devp); - return (ZULUVM_SUCCESS); -} - -/* - * free a zulu kernel driver instance - */ -int -zuluvm_free_device(zuluvm_info_t devp) -{ - int error; - zuluvm_state_t *zdev = (zuluvm_state_t *)devp; - - TNF_PROBE_1(zuluvm_free_device, "zuluvm", /* */, - tnf_opaque, zdev, zdev); - - if (zdev == NULL) - return (ZULUVM_NO_DEV); - ZULUVM_LOCK; - if (zdev->zvm.arg == NULL) { - ZULUVM_UNLOCK; - TNF_PROBE_1(zuluvm_free_device_done, "zuluvm", /* */, - tnf_int, error, ZULUVM_NO_DEV); - return (ZULUVM_NO_DEV); - } - (void) dmv_rem_intr(zdev->zvm.dmv_intr); - (void) rem_softintr(zdev->zvm.intr_num); - intr_dist_rem(zuluvm_retarget_intr, (void *)zdev); - zdev->zvm.arg = NULL; - ZULUVM_UNLOCK; - error = zuluvm_driver_detach(zdev); - if (error != ZULUVM_SUCCESS) - return (error); - zdev->dops = NULL; - kmem_free(zdev, sizeof (zuluvm_state_t)); - - TNF_PROBE_0(zuluvm_free_device_done, "zuluvm", /* */); - return (ZULUVM_SUCCESS); -} - -/* - * find the as in the list of active zulu processes - * The caller has to hold zdev->proc_lck - */ -static zuluvm_proc_t * -zuluvm_find_proc(zuluvm_state_t *zdev, struct as *asp) -{ - zuluvm_proc_t *p; - TNF_PROBE_2(zuluvm_find_proc, "zuluvm", /* */, - tnf_opaque, zdev, zdev, - tnf_opaque, asp, asp); - for (p = zdev->procs; p != NULL; p = p->next) { - if (ZULU_HAT2AS(p->zhat) == asp) { - TNF_PROBE_1(zuluvm_find_proc_done, - "zuluvm", /* */, tnf_opaque, proc, p); - return (p); - } - } - TNF_PROBE_0(zuluvm_find_proc_fail, "zuluvm", /* */); - return (NULL); -} - -void -zuluvm_as_free(struct as *as, void *arg, uint_t events) -{ - zuluvm_proc_t *proc = (zuluvm_proc_t *)arg; - zuluvm_state_t *zdev = proc->zdev; - int wait = 0; - int flag = 0; - int valid; - - (void) events; - - TNF_PROBE_1(zuluvm_as_free, "zuluvm", /* */, - tnf_opaque, arg, arg); - - (void) as_delete_callback(as, arg); - /* - * if this entry is still valid, then we need to sync - * with zuluvm_tlb_handler rountine. - */ - mutex_enter(&zdev->proc_lck); - valid = proc->valid; - proc->valid = 0; - mutex_exit(&zdev->proc_lck); - - if (valid) { - ZULUVM_LOCK; - if (proc == zdev->zvm.proc1) { - flag |= ZULUVM_WAIT_INTR1; - wait |= ZULUVM_DO_INTR1; - } - if (proc == zdev->zvm.proc2) { - flag |= ZULUVM_WAIT_INTR2; - wait |= ZULUVM_DO_INTR2; - } - if (flag) { - zdev->intr_flags |= flag; - /* - * wait until the tlb miss is resloved - */ - while (zdev->intr_flags & wait) { - cv_wait(&zdev->intr_wait, &zdev->dev_lck); - } - zdev->intr_flags &= ~flag; - } - ZULUVM_UNLOCK; - } - - if (proc->zhat != NULL) { - /* - * prevent any further tlb miss processing for this hat - */ - zulu_hat_terminate(proc->zhat); - } - - /* - * decrement the ref count and do the appropriate - * if it drops to zero. - */ - mutex_enter(&zdev->proc_lck); - (void) zuluvm_proc_release(zdev, proc); - mutex_exit(&zdev->proc_lck); -} - -/* - * notify zulu vm driver about a new process going to - * use zulu DMA. Create a zulu_hat. - */ -int -zuluvm_dma_add_proc(zuluvm_info_t devp, uint64_t *cookie) -{ - zuluvm_proc_t *proc; - int refcnt; - struct as *asp = ZULUVM_GET_AS; - zuluvm_state_t *zdev = (zuluvm_state_t *)devp; - - TNF_PROBE_1(zuluvm_dma_add_proc, "zuluvm", /* */, - tnf_opaque, zdev, zdev); - mutex_enter(&zdev->proc_lck); - proc = zuluvm_find_proc(zdev, asp); - if (proc == NULL) { - proc = kmem_zalloc(sizeof (zuluvm_proc_t), KM_SLEEP); - proc->zhat = zulu_hat_proc_attach(asp, zdev); - if (proc->zhat == NULL) { - mutex_exit(&zdev->proc_lck); - kmem_free(proc, sizeof (zuluvm_proc_t)); - TNF_PROBE_2(zuluvm_dma_add_proc_done, "zuluvm", /* */, - tnf_int, valid, 0, - tnf_int, error, ZULUVM_ERROR); - return (ZULUVM_ERROR); - } - proc->zdev = zdev; - proc->valid = 1; - proc->refcnt = 1; - proc->next = zdev->procs; - if (zdev->procs) - zdev->procs->prev = proc; - proc->prev = NULL; - zdev->procs = proc; - proc->refcnt++; - (void) as_add_callback(asp, zuluvm_as_free, proc, - AS_FREE_EVENT, 0, -1, KM_SLEEP); - } else { - if (proc->valid == 0) { - mutex_exit(&zdev->proc_lck); - TNF_PROBE_2(zuluvm_dma_add_proc_done, "zuluvm", /* */, - tnf_int, valid, 0, - tnf_int, error, ZULUVM_ERROR); - return (ZULUVM_ERROR); - } - proc->refcnt++; - } - refcnt = proc->refcnt; - mutex_exit(&zdev->proc_lck); - *cookie = (uint64_t)proc; - TNF_PROBE_2(zuluvm_dma_add_proc_done, "zuluvm", /* */, - tnf_int, refcnt, refcnt, - tnf_int, error, ZULUVM_SUCCESS); - return (ZULUVM_SUCCESS); -} - -void -zuluvm_proc_hold(zuluvm_state_t *zdev, zuluvm_proc_t *proc) -{ - mutex_enter(&zdev->proc_lck); - proc->refcnt++; - mutex_exit(&zdev->proc_lck); -} - -/* - * decrement ref count and free data if it drops to zero - */ -static int -zuluvm_proc_release(zuluvm_state_t *zdev, zuluvm_proc_t *proc) -{ - int refcnt; - ASSERT(MUTEX_HELD(&zdev->proc_lck)); - refcnt = --proc->refcnt; - TNF_PROBE_3(zuluvm_proc_release, "zuluvm", /* */, - tnf_opaque, zdev, zdev, - tnf_opaque, proc, proc, - tnf_int, refcnt, refcnt); - if (refcnt == 0) { - if (proc->next) - proc->next->prev = proc->prev; - if (proc->prev) - proc->prev->next = proc->next; - else - zdev->procs = proc->next; - kmem_free(proc, sizeof (zuluvm_proc_t)); - } - return (refcnt); -} - -/* - * this process is not longer using DMA, all entries - * have been removed from the TLB. - */ -int -zuluvm_dma_delete_proc(zuluvm_info_t devp, uint64_t cookie) -{ - int refcnt; - zuluvm_proc_t *proc = (zuluvm_proc_t *)cookie; - zuluvm_state_t *zdev = (zuluvm_state_t *)devp; - - TNF_PROBE_2(zuluvm_dma_delete_proc, "zuluvm", /* */, - tnf_opaque, zdev, zdev, - tnf_opaque, cookie, cookie); - mutex_enter(&zdev->proc_lck); - if (proc != NULL) { - TNF_PROBE_1(zuluvm_dma_delete_proc, "zuluvm", /* */, - tnf_opaque, proc, proc); - if (proc->zhat != NULL) { - zulu_hat_proc_detach(proc->zhat); - proc->zhat = NULL; - } - refcnt = zuluvm_proc_release(zdev, proc); - } - mutex_exit(&zdev->proc_lck); - - TNF_PROBE_2(zuluvm_dma_delete_proc_done, "zuluvm", /* */, - tnf_int, refcnt, refcnt, - tnf_int, error, ZULUVM_SUCCESS); - return (ZULUVM_SUCCESS); -} - -/* - * barrier sync for device driver - * blocks until zuluvm_tlbmiss_tl1 function is done - */ -void -zuluvm_fast_tlb_wait(caddr_t devp) -{ - int state; - zuluvm_state_t *zdev = (zuluvm_state_t *)devp; - int cnt = 0; - - do { - state = ZULUVM_GET_STATE(zdev); - cnt++; - } while (state == ZULUVM_STATE_TLB_PENDING); - TNF_PROBE_1(zuluvm_fast_tlb_wait, "zuluvm", /* */, - tnf_int, loop_cnt, cnt); -} - -/* - * setup DMA handling for this handle - */ -int -zuluvm_dma_alloc_ctx(zuluvm_info_t devp, int dma, short *mmuctx, - uint64_t *tsbreg) -{ - struct as *asp = ZULUVM_GET_AS; - int error = ZULUVM_NO_DEV; - zuluvm_state_t *zdev = (zuluvm_state_t *)devp; - int state, newstate; - - if (asp == NULL) { - TNF_PROBE_1(zuluvm_dma_alloc_ctx_done, "zuluvm", /* */, - tnf_int, error, ZULUVM_NO_HAT); - return (ZULUVM_NO_HAT); - } - - *tsbreg = 0; - state = ZULUVM_SET_STATE(zdev, ZULUVM_STATE_IDLE, - ZULUVM_STATE_STOPPED); - newstate = ZULUVM_GET_STATE(zdev); - TNF_PROBE_4(zuluvm_dma_alloc_ctx, "zuluvm", /* */, - tnf_opaque, devp, devp, - tnf_int, dma, dma, - tnf_int, oldstate, state, - tnf_int, newstate, newstate); -#ifdef DEBUG - if (zuluvm_debug_state) - cmn_err(CE_NOTE, "zuluvm_dma_alloc_ctx: state %d\n", state); -#endif - if (state != ZULUVM_STATE_STOPPED && state != ZULUVM_STATE_IDLE) { - while (state != ZULUVM_STATE_IDLE) { - state = ZULUVM_SET_STATE(zdev, ZULUVM_STATE_IDLE, - ZULUVM_STATE_STOPPED); -#ifdef DEBUG - if (zuluvm_debug_state) - cmn_err(CE_NOTE, "zuluvm_dma_alloc_ctx: (loop)" - " state %d\n", state); -#endif - if (state != ZULUVM_STATE_IDLE) - delay(1); - } - } - - if (zdev->zvm.arg != NULL) { - struct zulu_hat *zhat; - zuluvm_proc_t *proc; - - mutex_enter(&zdev->proc_lck); - proc = zuluvm_find_proc(zdev, asp); - if (proc != NULL) { - zhat = proc->zhat; - proc->refcnt++; - } - mutex_exit(&zdev->proc_lck); - - switch (dma) { - case ZULUVM_DMA1: - ZULUVM_LOCK; - zdev->zvm.proc1 = proc; - ZULUVM_UNLOCK; - error = ZULUVM_SUCCESS; - break; - case ZULUVM_DMA2: - ZULUVM_LOCK; - zdev->zvm.proc2 = proc; - ZULUVM_UNLOCK; - error = ZULUVM_SUCCESS; - break; - default: - mutex_enter(&zdev->proc_lck); - (void) zuluvm_proc_release(zdev, proc); - mutex_exit(&zdev->proc_lck); - } - - if (error == ZULUVM_SUCCESS) { - zulu_hat_validate_ctx(zhat); - if (zhat->zulu_ctx >= 0) { - *mmuctx = zhat->zulu_ctx; - } else { - printf("invalid context value: %d\n", - zhat->zulu_ctx); - - mutex_enter(&zdev->proc_lck); - (void) zuluvm_proc_release(zdev, proc); - mutex_exit(&zdev->proc_lck); - - error = ZULUVM_ERROR; - } - } else { - error = ZULUVM_ERROR; - } - } - TNF_PROBE_1(zuluvm_dma_alloc_ctx_done, "zuluvm", /* */, - tnf_int, error, error); - return (error); -} - -/* - * preload TLB - * this will try to pre-set the zulu tlb, mainly used for dma engine 2, - * video read-back. - */ -int -zuluvm_dma_preload(zuluvm_info_t devp, int dma, - int num, zulud_preload_t *list) -{ - int i; - int error = ZULUVM_SUCCESS; - struct zulu_hat *zhat; - zuluvm_proc_t *proc = NULL; - - zuluvm_state_t *zdev = (zuluvm_state_t *)devp; - - TNF_PROBE_4(zuluvm_dma_preload, "zuluvm", /* */, - tnf_opaque, devp, devp, - tnf_int, dma, dma, - tnf_int, num, num, - tnf_opaque, list, list); - ZULUVM_LOCK; - switch (dma) { - case ZULUVM_DMA1: - proc = zdev->zvm.proc1; - break; - case ZULUVM_DMA2: - proc = zdev->zvm.proc2; - break; - } - - mutex_enter(&zdev->proc_lck); - if (proc == NULL || proc->valid == 0 || proc->zhat == NULL) { - mutex_exit(&zdev->proc_lck); - ZULUVM_UNLOCK; - return (ZULUVM_NO_HAT); - } - mutex_exit(&zdev->proc_lck); - - zhat = proc->zhat; - /* - * need to release this to avoid recursive enter in zuluvm_load_tte - * which gets called from zulu_hat_memload() - */ - ZULUVM_UNLOCK; - - mutex_enter(&zdev->load_lck); - for (i = 0; i < num; i++) { - int pg_size; - int res; - int first = 1; - caddr_t addr = ZULUVM_GET_PAGE(list[i].addr); - int64_t size = (int64_t)list[i].len; - while (size > 0) { - if (list[i].tlbtype & ~ZULUVM_DMA_MASK) { - error = ZULUVM_INVALID_MISS; - break; - } - res = zulu_hat_load(zhat, addr, - (list[i].tlbtype == ZULUVM_DMA2) ? S_WRITE : S_READ, - &pg_size); - if ((res != 0) || (pg_size < 0)) { - error = ZULUVM_NO_MAP; - break; - } - ZULUVM_STATS_PRELOAD(zdev); - TNF_PROBE_2(zuluvm_dma_preload_addr, "zuluvm", /* */, - tnf_opaque, addr, addr, - tnf_opaque, size, size); - if (first) { - first = 0; - size -= ZULU_HAT_PGDIFF(list[i].addr, - pg_size); - } else { - size -= ZULU_HAT_PGSZ(pg_size); - } - addr += ZULU_HAT_PGSZ(pg_size); - } - } - mutex_exit(&zdev->load_lck); - TNF_PROBE_1(zuluvm_dma_preload_done, "zuluvm", /* */, - tnf_int, error, error); - return (ZULUVM_SUCCESS); -} - -/* - * destroy DMA handling for this handle - */ -int -zuluvm_dma_free_ctx(zuluvm_info_t devp, int dma) -{ - int error = ZULUVM_NO_DEV; - zuluvm_state_t *zdev = (zuluvm_state_t *)devp; - int state, newstate; - - state = ZULUVM_SET_STATE(zdev, ZULUVM_STATE_STOPPED, - ZULUVM_STATE_IDLE); - newstate = ZULUVM_GET_STATE(zdev); - TNF_PROBE_4(zuluvm_dma_free_ctx, "zuluvm", /* */, - tnf_opaque, devp, devp, - tnf_int, dma, dma, - tnf_int, oldstate, state, - tnf_int, newstate, newstate); -#ifdef DEBUG - if (zuluvm_debug_state) - cmn_err(CE_NOTE, "zuluvm_dma_free_ctx: state %d\n", state); -#endif - if (state != ZULUVM_STATE_IDLE && state != ZULUVM_STATE_STOPPED) { - int doit = 1; - while (doit) { - switch (state) { - case ZULUVM_STATE_CANCELED: - case ZULUVM_STATE_STOPPED: - doit = 0; - break; - case ZULUVM_STATE_IDLE: - state = ZULUVM_SET_STATE(zdev, - ZULUVM_STATE_STOPPED, - ZULUVM_STATE_IDLE); - break; - default: - state = ZULUVM_SET_STATE(zdev, - ZULUVM_STATE_CANCELED, state); - } - TNF_PROBE_1(zuluvm_dma_free_ctx, "zuluvm", /* */, - tnf_int, state, state); -#ifdef DEBUG - if (zuluvm_debug_state) - cmn_err(CE_NOTE, "zuluvm_dma_free_ctx: (loop1)" - " state %d\n", state); -#endif - } - } - TNF_PROBE_1(zuluvm_dma_free_ctx, "zuluvm", /* */, - tnf_int, state, state); - - error = ZULUVM_SUCCESS; - while (state != ZULUVM_STATE_STOPPED) { - state = ZULUVM_GET_STATE(zdev); -#ifdef DEBUG - if (zuluvm_debug_state) - cmn_err(CE_NOTE, "zuluvm_dma_free: (loop2) state %d\n", - state); -#endif - if (state != ZULUVM_STATE_STOPPED) - delay(1); - } - ZULUVM_LOCK; - if (zdev->zvm.arg != NULL) { - zuluvm_proc_t *proc = NULL; - switch (dma) { - case ZULUVM_DMA1: - proc = zdev->zvm.proc1; - zdev->zvm.proc1 = NULL; - break; - case ZULUVM_DMA2: - proc = zdev->zvm.proc2; - zdev->zvm.proc2 = NULL; - break; - default: - error = ZULUVM_NO_DEV; - } - ZULUVM_UNLOCK; - if (proc) { - mutex_enter(&zdev->proc_lck); - (void) zuluvm_proc_release(zdev, proc); - mutex_exit(&zdev->proc_lck); - } - } else { - ZULUVM_UNLOCK; - error = ZULUVM_NO_DEV; - } - TNF_PROBE_1(zuluvm_dma_free_ctx_done, "zuluvm", /* */, - tnf_int, error, error); - return (error); -} - -static void -zuluvm_do_retarget(zuluvm_state_t *zdev) -{ - int i, idx; - uint_t cpu; - for (i = 0; i < ZULUVM_MAX_INTR; i++) { - if (zdev->interrupts[i].ino != -1) { - cpu = intr_dist_cpuid(); - idx = zdev->interrupts[i].offset; - if (zdev->imr[idx] & ZULUVM_IMR_V_MASK) - zdev->imr[idx] = ZULUVM_IMR_V_MASK | - (cpu<<ZULUVM_IMR_TARGET_SHIFT); - else - zdev->imr[idx] = - cpu<<ZULUVM_IMR_TARGET_SHIFT; - } - } -} - -static void -zuluvm_retarget_intr(void *arg) -{ - zuluvm_state_t *zdev = (zuluvm_state_t *)arg; - ZULUVM_LOCK; - zuluvm_do_retarget(zdev); - ZULUVM_UNLOCK; -} - -int -zuluvm_add_intr(zuluvm_info_t devp, int ino, - uint_t (*handler)(caddr_t), caddr_t arg) -{ - zuluvm_state_t *zdev = (zuluvm_state_t *)devp; - if (devp == NULL) { - TNF_PROBE_1(zuluvm_add_intr_done, "zuluvm", /* */, - tnf_int, error, ZULUVM_NO_DEV); - return (ZULUVM_NO_DEV); - } - if (ddi_add_intr(zdev->dip, ino, NULL, NULL, handler, arg) - != DDI_SUCCESS) { - TNF_PROBE_1(zuluvm_add_intr_done, "zuluvm", /* */, - tnf_int, error, ZULUVM_ERROR); - return (ZULUVM_ERROR); - } - return (ZULUVM_SUCCESS); -} - -int -zuluvm_rem_intr(zuluvm_info_t devp, int ino) -{ - zuluvm_state_t *zdev = (zuluvm_state_t *)devp; - if (devp == NULL) { - TNF_PROBE_1(zuluvm_rem_intr_done, "zuluvm", /* */, - tnf_int, error, ZULUVM_NO_DEV); - return (ZULUVM_NO_DEV); - } - /* remove from distributin list */ - ZULUVM_LOCK; - zdev->imr[zdev->interrupts[ino].offset] &= ~ZULUVM_IMR_V_MASK; - ZULUVM_UNLOCK; - ddi_remove_intr(zdev->dip, ino, NULL); - return (ZULUVM_SUCCESS); -} - -int -zuluvm_enable_intr(zuluvm_info_t devp, int num) -{ - zuluvm_state_t *zdev = (zuluvm_state_t *)devp; - - TNF_PROBE_2(zuluvm_enable_intr, "zuluvm_intr", /* */, - tnf_opaque, devp, devp, - tnf_int, num, num); - if (devp == NULL) { - TNF_PROBE_1(zuluvm_enable_intr_done, "zuluvm", /* */, - tnf_int, error, ZULUVM_NO_DEV); - return (ZULUVM_NO_DEV); - } - if (num < 0 || num > ZULUVM_IMR_MAX) { - TNF_PROBE_1(zuluvm_enable_intr_done, "zuluvm", /* */, - tnf_int, error, ZULUVM_BAD_IDX); - return (ZULUVM_BAD_IDX); - } - ZULUVM_LOCK; - zdev->imr[num] |= ZULUVM_IMR_V_MASK; - ZULUVM_UNLOCK; - TNF_PROBE_1(zuluvm_enable_intr_done, "zuluvm_intr", /* */, - tnf_int, error, ZULUVM_SUCCESS); - return (ZULUVM_SUCCESS); -} - -int -zuluvm_disable_intr(zuluvm_info_t devp, int num) -{ - zuluvm_state_t *zdev = (zuluvm_state_t *)devp; - - TNF_PROBE_2(zuluvm_disable_intr, "zuluvm_intr", /* */, - tnf_opaque, devp, devp, - tnf_int, num, num); - if (devp == NULL) { - TNF_PROBE_1(zuluvm_disable_intr_done, "zuluvm", /* */, - tnf_int, error, ZULUVM_NO_DEV); - return (ZULUVM_NO_DEV); - } - if (num < 0 || num > ZULUVM_IMR_MAX) { - TNF_PROBE_1(zuluvm_disable_intr_done, "zuluvm", /* */, - tnf_int, error, ZULUVM_BAD_IDX); - return (ZULUVM_BAD_IDX); - } - ZULUVM_LOCK; - zdev->imr[num] &= ~ZULUVM_IMR_V_MASK; - ZULUVM_UNLOCK; - TNF_PROBE_1(zuluvm_disable_intr_done, "zuluvm_intr", /* */, - tnf_int, error, ZULUVM_SUCCESS); - return (ZULUVM_SUCCESS); -} - -static int -zuluvm_get_intr_props(zuluvm_state_t *zdev, - dev_info_t *devi) -{ - int *intr; - int i; - uint_t nintr; - - zdev->agentid = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS, - "portid", -1); - if (zdev->agentid == -1) { - cmn_err(CE_WARN, "%s%d: no portid property", - ddi_get_name(devi), - ddi_get_instance(devi)); - return (ZULUVM_ERROR); - } - - for (i = 0; i < ZULUVM_MAX_INTR; i++) { - zdev->interrupts[i].offset = 0; - zdev->interrupts[i].ino = -1; - } - - if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS, - "interrupts", &intr, &nintr) == DDI_PROP_SUCCESS) { - - if (nintr == 0) { - cmn_err(CE_WARN, "%s%d: no interrupts in property", - ddi_get_name(devi), - ddi_get_instance(devi)); - ddi_prop_free(intr); - return (ZULUVM_ERROR); - } - if (nintr >= ZULUVM_MAX_INTR) { - cmn_err(CE_WARN, "%s%d: to many interrupts (%d)", - ddi_get_name(devi), - ddi_get_instance(devi), nintr); - ddi_prop_free(intr); - return (ZULUVM_ERROR); - } - for (i = 0; i < nintr; i++) { - zdev->interrupts[i].offset = intr[i]; - zdev->interrupts[i].ino = i; - } - ddi_prop_free(intr); - } else { - cmn_err(CE_WARN, "%s%d: no interrupts property", - ddi_get_name(devi), - ddi_get_instance(devi)); - } - return (ZULUVM_SUCCESS); -} - -/* *** enf of zulu *** */ diff --git a/usr/src/uts/sun4u/ml/zulu_asm.s b/usr/src/uts/sun4u/ml/zulu_asm.s deleted file mode 100644 index eb6c7497c6..0000000000 --- a/usr/src/uts/sun4u/ml/zulu_asm.s +++ /dev/null @@ -1,325 +0,0 @@ -/* - * CDDL HEADER START - * - * The contents of this file are subject to the terms of the - * Common Development and Distribution License (the "License"). - * You may not use this file except in compliance with the License. - * - * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE - * or http://www.opensolaris.org/os/licensing. - * See the License for the specific language governing permissions - * and limitations under the License. - * - * When distributing Covered Code, include this CDDL HEADER in each - * file and include the License file at usr/src/OPENSOLARIS.LICENSE. - * If applicable, add the following below this CDDL HEADER, with the - * fields enclosed by brackets "[]" replaced with your own identifying - * information: Portions Copyright [yyyy] [name of copyright owner] - * - * CDDL HEADER END - */ -/* - * Copyright 2006 Sun Microsystems, Inc. All rights reserved. - * Use is subject to license terms. - */ - -#pragma ident "%Z%%M% %I% %E% SMI" - -#if defined(lint) -#include <sys/types.h> -#include <sys/thread.h> -#else /* lint */ -#include "assym.h" -#endif /* lint */ - -#include <sys/asi.h> -#include <sys/sun4asi.h> -#include <sys/machasi.h> -#include <sys/asm_linkage.h> -#include <sys/pte.h> -#include <sys/mmu.h> -#include <sys/intreg.h> -#include <sys/zulumod.h> -#include <vm/hat_sfmmu.h> -#include <sys/zulu_hat.h> -#include <zuluvm_offsets.h> - -#ifdef lint -void -zuluvm_dmv_tlbmiss_tl1() -{} - -#else /* lint */ - - DGDEF(zuluvm_base_pgsize) - .word 0 - - ENTRY_NP(zuluvm_dmv_tlbmiss_tl1) - - ! g1 - zuluvm_state_t pointer - ! g2 - IRDR_0 - mov UIII_IRDR_1, %g3 - ldxa [%g3]ASI_INTR_RECEIVE, %g5 - stx %g5, [%g1 + ZULUVM_ASM_TLB_ADDR] - mov UIII_IRDR_6, %g3 - ldxa [%g3]ASI_INTR_RECEIVE, %g5 - stx %g5, [%g1 + ZULUVM_ASM_TLB_TYPE] - - stxa %g0, [%g0]ASI_INTR_RECEIVE_STATUS ! clear the BUSY bit - membar #Sync - - mov %g1, %g7 - - ! check the fast tlb miss flag - sethi %hi(zuluvm_fast_tlb), %g6 - lduw [%g6 + %lo(zuluvm_fast_tlb)], %g6 - brz,pn %g6, send_intr1 - mov ZULUVM_TTE_DELAY, %g1 -#if 1 - add %g7, ZULUVM_STATE, %g4 - mov ZULUVM_STATE_IDLE, %g1 - mov ZULUVM_STATE_TLB_PENDING, %g6 - casa [%g4]ASI_N, %g1, %g6 - cmp %g6, %g1 - be,pt %icc, 2f - nop - - mov ZULUVM_STATE_CANCELED, %g1 - cmp %g6, %g1 - be,pt %icc, 1f - mov ZULUVM_STATE_STOPPED, %g1 - retry -1: - st %g1, [%g4] -#ifdef ZULUVM_STATS - lduw [%g7 + ZULUVM_ST_TLBCANCEL], %g3 - add %g3, 1, %g3 - stuw %g3, [%g7 + ZULUVM_ST_TLBCANCEL] -#endif - retry - -2: - ldx [%g7 + ZULUVM_ASM_TLB_TYPE], %g4 - and %g4, ZULUVM_DMA_MASK, %g4 -#ifdef ZULUVM_STATS - cmp %g4, ZULUVM_DMA2 - be,a,pn %icc, 1f - add %g7, ZULUVM_ST_DTLB2MISS, %g1 - cmp %g4, ZULUVM_ITLB1 - be,a,pn %icc, 1f - add %g7, ZULUVM_ST_ITLB1MISS, %g1 - cmp %g4, ZULUVM_ITLB2 - be,a,pn %icc, 1f - add %g7, ZULUVM_ST_ITLB2MISS, %g1 - add %g7, ZULUVM_ST_DTLB1MISS, %g1 -1: - lduw [%g1], %g3 - add %g3, 1, %g3 - stuw %g3, [%g1] -#endif - /* - * lookup the tte in the tsb - * %g1 - vaddr[63:13], ctx[12:0] - * %g2 - our trap level - * %g3 - return address - * %g7 - zulu data pointer (needs to be preserved) - * return: - * %g1 - flags [63..58] and pfn [31..0] - * %g2 - status code if %g1 is null - * %g7 - zulu data pointer - */ - mov 1, %g2 - set zulu_hat_tsb_lookup_tl1, %g3 - jmpl %g3, %g3 - ldx [%g7 + ZULUVM_ASM_TLB_ADDR], %g1 ! vaddr(tag) - - /* - * did we find a tte ?? - * If not, %g2 has the error code - */ - brgez,a,pt %g1, send_intr - mov %g2, %g1 - - set zulu_tsb_hit, %g6 - ldx [%g6], %g3 - add %g3, 1, %g3 - stx %g3, [%g6] - - /* - * get flags and pfn - */ - sllx %g1, 32, %g6 - srlx %g6, 32, %g6 ! %g6 pfn - srlx %g1, 59, %g3 - and %g3, 0x7, %g2 ! %g2 page size - srlx %g3, 3, %g4 - and %g4, 1, %g4 ! %g4 write perm - mov %g6, %g1 - - /* - * check if this is a dtlb2 miss(no itlb, pgsz != 8k) - * and if the current dtlb2 pgsz != tte pgsz - */ - ldx [%g7 + ZULUVM_ASM_TLB_TYPE], %g3 - and %g3, 0x1, %g3 - brnz,pt %g3, 3f ! not 0 => itlb => handles - nop - - ! check page size, base page size is always handled by dtlb1, so we - ! only need to check against dtlb2 - sethi %hi(zuluvm_base_pgsize), %g3 - lduw [%g3 + %lo(zuluvm_base_pgsize)], %g3 - cmp %g2, %g3 - be,pt %icc, 2f - cmp %g2, ZULU_TTE4M - be,pt %icc, 2f ! TTE4M => dtlb2 => ok! - nop - -#ifdef ZULUVM_STATS - lduw [%g7 + ZULUVM_ST_PAGESIZE], %g3 - add %g3, 1, %g3 - stuw %g3, [%g7 + ZULUVM_ST_PAGESIZE] - add %g7, ZULUVM_ST_MISS, %g3 - sll %g2, 2, %g5 - add %g5, %g3, %g5 - lduw [%g5], %g3 - add %g3, 1, %g3 - stuw %g3, [%g5] -#endif - ! set tte size to ZULUVM_BASE_PGSZ - sethi %hi(zuluvm_base_pgsize), %g3 - lduw [%g3 + %lo(zuluvm_base_pgsize)], %g3 - ba,pt %icc, 3f - mov %g3, %g2 -2: - -#ifdef ZULUVM_STATS - add %g7, ZULUVM_ST_MISS, %g3 - sll %g2, 2, %g5 - add %g3, %g5, %g5 - lduw [%g5], %g3 - add %g3, 1, %g3 - stuw %g3, [%g5] -#endif - - ! we maintain data on the last pfns for the last 12 pfns that we - ! processed -3: - lduw [%g7 + ZULUVM_PFNCNT], %g5 - add %g5, 4, %g3 - cmp %g3, 48 - be,a,pn %icc, 1f - mov %g0, %g3 - -1: - stuw %g3, [%g7 + ZULUVM_PFNCNT] - sllx %g5, 3, %g5 - add %g7, ZULUVM_PFNBUF, %g3 - add %g3, %g5, %g3 - stx %g1, [%g3] - stx %g2, [%g3 + 8] - stx %g4, [%g3 + 16] - ldx [%g7 + ZULUVM_ASM_TLB_TYPE], %g5 - stx %g5, [%g3 + 24] - - ldx [%g7 + ZULUVM_ASM_TLB_TYPE], %g3 - and %g3, 0x3, %g3 ! tlbtype - ldx [%g7 + ZULUVM_ARG], %g6 - - ! write tte to zulu mmu - ! %g1 pfn - ! %g2 tte size - ! %g3 tlbtype - ! %g4 tte wrperm - ! %g6 zulu device driver arg - ! %g7 devtab pointer - - sllx %g1, ZULUVM_ZFB_MMU_TLB_D_PA_SHIFT, %g1 - mov 0x1, %g5 - sllx %g5, 63, %g5 ! ZFB_MMU_TLB_D_V_MASK - or %g1, %g5, %g1 - or %g1, ZULUVM_ZFB_MMU_TLB_D_C_MASK, %g1 - sllx %g2, ZULUVM_ZFB_MMU_TLB_D_SZ_SHIFT, %g2 - - brz,pt %g4, 3f ! write perm ?? - or %g2, %g1, %g1 - - or %g1, ZULUVM_ZFB_MMU_TLB_D_W_MASK, %g1 -3: - ! at this point %g1 is ready to be written to the corresponding - ! data_in register, let's see which if it was itlb or dtlb... - and %g3, ZULUVM_ITLB_FLAG, %g3 - ! assumption is that data miss - brz,pt %g3, 4f ! is more likely than instr miss - ldx [%g7 + ZULUVM_PAMMU], %g2 ! physical addr of zulu mmu regs - - ! instruction miss - mov ZULUVM_ZFB_MMU_TLB_CR_IMISS_MASK, %g5 - add %g2, ZULUVM_ITLB_DATA_IN, %g4 - !stxa %g1, [%g4]ASI_IO - ba,pt %xcc, 5f - stxa %g1, [%g4]ASI_IO - !ldxa [%g4]ASI_IO, %g4 -4: - ! data miss - mov ZULUVM_ZFB_MMU_TLB_CR_DMISS_MASK, %g5 - add %g2, ZULUVM_DTLB_DATA_IN, %g4 - stxa %g1, [%g4]ASI_IO - !ldxa [%g4]ASI_IO, %g4 -5: - add %g7, ZULUVM_STATE, %g4 - mov ZULUVM_STATE_TLB_PENDING, %g6 - mov ZULUVM_STATE_IDLE, %g1 - casa [%g4]ASI_N, %g6, %g1 - cmp %g6, %g1 - bne,a,pn %icc, stopped - mov ZULUVM_STATE_STOPPED, %g3 - - ldx [%g7 + ZULUVM_PAMMU], %g2 - add %g2, ZULUVM_TLB_CONTROL, %g2 - stxa %g5, [%g2]ASI_IO - !ldxa [%g2]ASI_IO, %g3 - retry - -send_intr: - add %g7, ZULUVM_STATE, %g4 - mov ZULUVM_STATE_INTR_QUEUED, %g5 - mov ZULUVM_STATE_TLB_PENDING, %g3 - casa [%g4]ASI_N, %g3, %g5 - cmp %g3, %g5 - be,pt %icc, deliver_intr - mov ZULUVM_STATE_STOPPED, %g3 - ba,pt %icc, stopped - nop -#endif - -send_intr1: - add %g7, ZULUVM_STATE, %g4 - mov ZULUVM_STATE_IDLE, %g3 - mov ZULUVM_STATE_INTR_QUEUED, %g5 - casa [%g4]ASI_N, %g3, %g5 - cmp %g3, %g5 - be,pt %icc, deliver_intr - mov ZULUVM_STATE_STOPPED, %g3 -stopped: - st %g3, [%g4] -#ifdef ZULUVM_STATS - lduw [%g7 + ZULUVM_ST_TLBCANCEL], %g3 - add %g3, 1, %g3 - stuw %g3, [%g7 + ZULUVM_ST_TLBCANCEL] -#endif - retry - -deliver_intr: - stx %g1, [%g7 + ZULUVM_ASM_TLB_ERRCODE] ! set the error field - stx %g6, [%g7 + ZULUVM_ASM_TLB_TTE] ! deliver tte in data_0 - ! %g6 is invalid if error != SUCCESS - ! setsoftint_tl1(uint64_t inum, uint64_t dummy) - set setsoftint_tl1, %g5 - jmp %g5 - ldx [%g7 + ZULUVM_INTRNUM], %g1 - - SET_SIZE(zuluvm_dmv_tlbmiss_tl1) - -#endif /* lint */ - diff --git a/usr/src/uts/sun4u/ml/zulu_hat_asm.s b/usr/src/uts/sun4u/ml/zulu_hat_asm.s deleted file mode 100644 index eba64b9368..0000000000 --- a/usr/src/uts/sun4u/ml/zulu_hat_asm.s +++ /dev/null @@ -1,314 +0,0 @@ -/* - * CDDL HEADER START - * - * The contents of this file are subject to the terms of the - * Common Development and Distribution License, Version 1.0 only - * (the "License"). You may not use this file except in compliance - * with the License. - * - * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE - * or http://www.opensolaris.org/os/licensing. - * See the License for the specific language governing permissions - * and limitations under the License. - * - * When distributing Covered Code, include this CDDL HEADER in each - * file and include the License file at usr/src/OPENSOLARIS.LICENSE. - * If applicable, add the following below this CDDL HEADER, with the - * fields enclosed by brackets "[]" replaced with your own identifying - * information: Portions Copyright [yyyy] [name of copyright owner] - * - * CDDL HEADER END - */ -/* - * Copyright 2005 Sun Microsystems, Inc. All rights reserved. - * Use is subject to license terms. - */ - -#pragma ident "%Z%%M% %I% %E% SMI" - -#if defined(lint) -#include <sys/types.h> -#include <sys/thread.h> -#else /* lint */ -#include "assym.h" -#endif /* lint */ - -#include <sys/asi.h> -#include <sys/machasi.h> -#include <sys/asm_linkage.h> -#include <zuluvm_offsets.h> -#include <sys/zulu_hat.h> -#include <sys/zuluvm.h> - -/* - * function to look up ttes in zulu_hat TSB. - * - * zulu_hat_tsb_lookup_tl1 is called from the zuluvm dmv interrupt handler - * so we can only use the global registers. - * - * zulu_hat_tsb_lookup_tl0 is called from TL=0 - */ -#ifdef lint - -/* ARGSUSED */ -uint64_t -zulu_hat_tsb_lookup_tl1(caddr_t vaddr) -{ - return (0); -} - -/* ARGSUSED */ -uint64_t -zulu_hat_tsb_lookup_tl0(struct zulu_hat *zhat, caddr_t vaddr) -{ - return (0); -} - -#else /* lint */ - - /* - * %g1 - vaddr | ctx - * %g3 - return address - * Must preserve %g7 for caller - * - * returns: - * %g1 - pfn and flags - * %g2 - zuluvm error code if %g1 is null - */ - ENTRY_NP(zulu_hat_tsb_lookup_tl1) - set ZULU_CTX_MASK, %g4 - and %g1, %g4, %g4 - - ! we're at trap level 1, (TL=1) - ! if the context is already locked by another - ! thread, punt to the TL=0 code - ! it's not safe to spinloop now. - - set zulu_ctx_tab, %g6 - sllx %g4, 3, %g5 -#ifdef DEBUG - mov %g5, %g2 ! remember ctx * 8 -#endif - add %g5, %g6, %g6 - - ldx [%g6], %g4 - andcc %g4, 1, %g0 - bne,a,pn %icc, ctx_busy - mov ZULUVM_CTX_LOCKED, %g2 - - ! now do a compare and swap and make sure it's still not locked - or %g4, 1, %g5 - casxa [%g6]ASI_N, %g4, %g5 - cmp %g4, %g5 - bne,a,pn %icc, ctx_busy - mov ZULUVM_CTX_LOCKED, %g2 - - brz,a,pn %g4, zulu_hat_tsb_exit - mov %g0, %g1 - - ! we have the lock now proceed - - ! set lsb of g3 to indicate that we need to unlock the context - ! before returning - ba,pt %xcc, zulu_hat_tsb_lookup - or %g3, 1, %g3 - -ctx_busy: - mov %g0, %g1 - jmpl %g3+8, %g0 - nop - - - /* - * zulu_hat_tsb_lookup_tl0 jumps here - * - * %g1 vaddr | ctx - * %g3 return address | unlock flag (bit zero) - * %g4 has the zulu hat ptr (locked) - */ -zulu_hat_tsb_lookup: - mov %g1, %g2 - mov %g4, %g1 - - add %g1, ZULU_HAT_TSB_SZ, %g5 - lduh [%g5], %g5 ! tsb size - sub %g5, 1, %g5 - - srlx %g2, 22, %g4 ! 4m page hash - and %g5, %g4, %g4 ! hash index - sllx %g4, 4, %g4 - add %g1, ZULU_HAT_TSB, %g5 - ldx [%g5], %g5 - add %g5, %g4, %g4 ! ptr to struct zulu_tte - ldx [%g4], %g5 ! get the tag - - set (0x1ff << 13), %g6 - andn %g5, %g6, %g5 - andn %g2, %g6, %g6 - cmp %g5, %g6 - bne,pn %xcc, zulu_hat_tsb_try_512k - nop - - ldx [%g4 + 8], %g4 ! flags and pfn - brgez,pn %g4, zulu_hat_tsb_try_512k ! check if entry is valid - nop - - sllx %g4, 2, %g5 - srlx %g5, 61, %g5 ! tte size - cmp %g5, ZULU_TTE4M - be,pn %xcc, zulu_hat_tsb_found - nop - -zulu_hat_tsb_try_512k: - add %g1, ZULU_HAT_TSB_SZ, %g5 - lduh [%g5], %g5 ! tsb size - sub %g5, 1, %g5 - - srlx %g2, 19, %g4 ! 4m page hash - and %g5, %g4, %g4 ! hash index - sllx %g4, 4, %g4 - add %g1, ZULU_HAT_TSB, %g5 - ldx [%g5], %g5 - add %g5, %g4, %g4 ! ptr to struct zulu_tte - ldx [%g4], %g5 ! get the tag - - set (0x3f << 13), %g6 - andn %g5, %g6, %g5 - andn %g2, %g6, %g6 - cmp %g5, %g6 - bne,pn %xcc, zulu_hat_tsb_try_64k - nop - - ldx [%g4 + 8], %g4 ! flags and pfn - brgez,pn %g4, zulu_hat_tsb_try_64k ! check if entry is valid - nop - - sllx %g4, 2, %g5 - srlx %g5, 61, %g5 ! tte size - cmp %g5, ZULU_TTE512K - be,pn %xcc, zulu_hat_tsb_found - nop - -zulu_hat_tsb_try_64k: - add %g1, ZULU_HAT_TSB_SZ, %g5 - lduh [%g5], %g5 ! tsb size - sub %g5, 1, %g5 - - srlx %g2, 16, %g4 ! 4m page hash - and %g5, %g4, %g4 ! hash index - sllx %g4, 4, %g4 - add %g1, ZULU_HAT_TSB, %g5 - ldx [%g5], %g5 - add %g5, %g4, %g4 ! ptr to struct zulu_tte - ldx [%g4], %g5 ! get the tag - - set (0x7 << 13), %g6 - andn %g5, %g6, %g5 - andn %g2, %g6, %g6 - cmp %g5, %g6 - bne,pn %xcc, zulu_hat_tsb_try_8k - nop - - ldx [%g4 + 8], %g4 ! flags and pfn - brgez,pn %g4, zulu_hat_tsb_try_8k ! check if entry is valid - nop - - sllx %g4, 2, %g5 - srlx %g5, 61, %g5 ! tte size - cmp %g5, ZULU_TTE64K - be,pn %xcc, zulu_hat_tsb_found - nop - -zulu_hat_tsb_try_8k: - add %g1, ZULU_HAT_TSB_SZ, %g5 - lduh [%g5], %g5 ! tsb size - sub %g5, 1, %g5 - - srlx %g2, 13, %g4 ! calc hash - and %g5, %g4, %g4 ! hash index - sllx %g4, 4, %g4 - add %g1, ZULU_HAT_TSB, %g5 - ldx [%g5], %g5 ! tsb ptr - add %g5, %g4, %g4 ! ptr to struct tte - ldx [%g4], %g5 ! get the tag - cmp %g5, %g2 - bne,pn %xcc, zulu_hat_tsb_exit - mov %g0, %g1 - - ldx [%g4 + 8], %g4 ! flags and pfn - brgez,pn %g4, zulu_hat_tsb_exit ! check if entry is valid - mov %g0, %g1 - - sllx %g4, 2, %g5 - srlx %g5, 61, %g5 ! tte size - brnz,pn %g5, zulu_hat_tsb_exit - mov %g0, %g1 - -zulu_hat_tsb_found: - ! expect the tte size in %g5 - mulx %g5, 3, %g5 - mov 1, %g1 - sllx %g1, %g5, %g1 - sub %g1, 1, %g1 - andn %g4, %g1, %g4 - srlx %g2, 13, %g5 - and %g1, %g5, %g5 - or %g5, %g4, %g4 - mov %g4, %g1 - - ! now fall through to exit - -zulu_hat_tsb_exit: - ! if bit zero of %g3 is set, we're at TL=1 and need to unlock - ! the context here - andcc %g3, 1, %g0 - be,pn %xcc, after_unlock - nop - - ! clear the context unlock flag - andn %g3, 1, %g3 - - set ZULU_CTX_MASK, %g6 - and %g2, %g6, %g6 ! ctx num - - sllx %g6, 3, %g6 - set zulu_ctx_tab, %g5 - add %g6, %g5, %g5 ! %g5 = &zulu_ctx_tab[ctx_num] - ldx [%g5], %g6 - andn %g6, 1, %g6 - stx %g6, [%g5] - -after_unlock: - - ! set the status code to ZULUVM_NO_TTE in case we are running at TL=1 - ! and no tte was found. - ! - ! note: caller doesn't examine %g2 unless flags and pfn are null - jmpl %g3 + 0x8, %g0 - mov ZULUVM_NO_TTE, %g2 - - - - - SET_SIZE(zulu_hat_tsb_lookup_tl1) - - /* - * %o0 - zulu hat ptr (already locked) - * %o1 - vaddr - */ - ENTRY_NP(zulu_hat_tsb_lookup_tl0) - mov %o0, %g4 - - set zulu_hat_tsb_lookup, %g3 - - ! note bit zero of g3 is zero which tells zulu_hat_tsb_lookup - ! to not unlock tsb before returning - - jmpl %g3, %g3 - mov %o1, %g1 - - retl - mov %g1, %o0 - SET_SIZE(zulu_hat_tsb_lookup_tl0) - -#endif /* lint */ diff --git a/usr/src/uts/sun4u/sys/zulu_hat.h b/usr/src/uts/sun4u/sys/zulu_hat.h deleted file mode 100644 index 447d631944..0000000000 --- a/usr/src/uts/sun4u/sys/zulu_hat.h +++ /dev/null @@ -1,214 +0,0 @@ -/* - * CDDL HEADER START - * - * The contents of this file are subject to the terms of the - * Common Development and Distribution License, Version 1.0 only - * (the "License"). You may not use this file except in compliance - * with the License. - * - * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE - * or http://www.opensolaris.org/os/licensing. - * See the License for the specific language governing permissions - * and limitations under the License. - * - * When distributing Covered Code, include this CDDL HEADER in each - * file and include the License file at usr/src/OPENSOLARIS.LICENSE. - * If applicable, add the following below this CDDL HEADER, with the - * fields enclosed by brackets "[]" replaced with your own identifying - * information: Portions Copyright [yyyy] [name of copyright owner] - * - * CDDL HEADER END - */ -/* - * Copyright 2003 Sun Microsystems, Inc. All rights reserved. - * Use is subject to license terms. - */ - -#ifndef __ZULU_HAT_INCL__ -#define __ZULU_HAT_INCL__ - -#pragma ident "%Z%%M% %I% %E% SMI" - -#ifdef __cplusplus -extern "C" { -#endif - -#define ZULU_TTE8K 0 -#define ZULU_TTE64K 1 -#define ZULU_TTE512K 2 -#define ZULU_TTE4M 3 -#define ZULUM_MAX_PG_SIZES 4 - -#define ZULU_CTX_MASK 0x1fff - -#ifndef _ASM - -#include <sys/types.h> -#include <sys/atomic.h> -#include <vm/xhat.h> -#include <sys/avl.h> - - -#define ZULU_HAT_BP_SHIFT 13 -#define ZULU_HAT_SZ_SHIFT(sz) ((sz) * 3) -#define ZULU_HAT_NUM_PGS(sz) (1<<ZULU_HAT_SZ_SHIFT(sz)) -#define ZULU_HAT_PGSHIFT(s) (ZULU_HAT_BP_SHIFT + \ - ZULU_HAT_SZ_SHIFT(s)) -#define ZULU_HAT_PGSZ(s) ((uint64_t)1<<ZULU_HAT_PGSHIFT(s)) -#define ZULU_HAT_PGOFFSET(s) (ZULU_HAT_PGSZ(s) - 1) -#define ZULU_HAT_PGMASK(s) (~ZULU_HAT_PGOFFSET((uint64_t)s)) -#define ZULU_HAT_PGADDR(a, s) ((uintptr_t)(a) & ZULU_HAT_PGMASK(s)) -#define ZULU_HAT_PGADDROFF(a, s) ((uintptr_t)(a) & ZULU_HAT_PGOFFSET(s)) -#define ZULU_HAT_PGDIFF(a, s) (ZULU_HAT_PGSZ(s) - \ - ZULU_HAT_PGADDROFF(a, s)) - -#define ZULU_HAT_PFN_MASK(sz) ((1 << ZULU_HAT_SZ_SHIFT(sz)) - 1) -#define ZULU_HAT_ADJ_PFN(ttep, vaddr) \ - ((ttep->zulu_tte_pfn & ~ZULU_HAT_PFN_MASK(ttep->zulu_tte_size)) | \ - (((uintptr_t)vaddr >> ZULU_HAT_BP_SHIFT) & \ - ZULU_HAT_PFN_MASK(ttep->zulu_tte_size))) - -/* - * zulu_ctx_tab is an array of pointers to zulu hat structures. - * since the addresses are 8 byte aligned we use bit 0 as a lock flag. - * This will synchronize TL1 access to the tsb and the mappings. - */ - -#define ZULU_CTX_LOCK 0x1 - -#define ZULU_CTX_LOCK_INIT(c) zulu_ctx_tab[c] = NULL -#define ZULU_CTX_IS_FREE(c) (zulu_ctx_tab[c] == NULL) -#define ZULU_CTX_SET_HAT(c, h) zulu_ctx_tab[c] = h - -#define ZULU_CTX_GET_HAT(c) (struct zulu_hat *)((uint64_t) \ - zulu_ctx_tab[c] & ~ZULU_CTX_LOCK) - -struct zulu_tag { - uint64_t zulu_tag_page:51; /* [63:13] vpage */ -}; - -struct zulu_tte { - union { - struct zulu_tag zulu_tte_tag; - uint64_t zulu_tte_addr; - } un; - uint_t zulu_tte_valid :1; - uint_t zulu_tte_perm :1; - uint_t zulu_tte_size :3; - uint_t zulu_tte_locked :1; - uint_t zulu_tte_pfn; -}; - -/* - * zulu hat stores its list of translation in a hash table. - * TODO: size this table. 256 buckets may be too small. - */ -#define ZULU_HASH_TBL_NUM 0x100 -#define ZULU_HASH_TBL_MASK (ZULU_HASH_TBL_NUM - 1) -#define ZULU_HASH_TBL_SHIFT(_s) (ZULU_HAT_BP_SHIFT + (3 * _s)) -#define ZULU_HASH_TBL_SZ (ZULU_HASH_TBL_NUM * sizeof (struct zulu_hat_blk *)) -#define ZULU_MAP_HASH_VAL(_v, _s) (((_v) >> ZULU_HASH_TBL_SHIFT(_s)) & \ - ZULU_HASH_TBL_MASK) -#define ZULU_MAP_HASH_HEAD(_zh, _v, _s) \ - (_zh->hash_tbl[ZULU_MAP_HASH_VAL(_v, _s)]) - -/* - * - * TODO: need finalize the number of entries in the TSB - * 32K tsb entries caused us to never get a tsb miss that didn't cause - * a page fault. - * - * Reducing TSB_NUM to 512 entries caused tsb_miss > tsb_hit - * in a yoyo run. - */ -#define ZULU_TSB_NUM 4096 -#define ZULU_TSB_SZ (ZULU_TSB_NUM * sizeof (struct zulu_tte)) -#define ZULU_TSB_HASH(a, ts, s) (((uintptr_t)(a) >> ZULU_HAT_PGSHIFT(ts)) & \ - (s-1)) - -#define ZULU_VADDR(tag) (tag & ~ZULU_CTX_MASK) -#define ZULU_TTE_TO_PAGE(a) a.un.zulu_tte_tag.zulu_tag_page - - -struct zulu_hat_blk { - struct zulu_hat_blk *zulu_hash_next; - struct zulu_hat_blk *zulu_hash_prev; - struct zulu_shadow_blk *zulu_shadow_blk; - struct zulu_tte zulu_hat_blk_tte; -}; - -#define zulu_hat_blk_vaddr zulu_hat_blk_tte.un.zulu_tte_addr -#define zulu_hat_blk_pfn zulu_hat_blk_tte.zulu_tte_pfn -#define zulu_hat_blk_page ZULU_TTE_TO_PAGE(zulu_hat_blk_tte) -#define zulu_hat_blk_perm zulu_hat_blk_tte.zulu_tte_perm -#define zulu_hat_blk_size zulu_hat_blk_tte.zulu_tte_size -#define zulu_hat_blk_valid zulu_hat_blk_tte.zulu_tte_valid - -/* - * for fast lookups by address, len we use an avl list to shadow occupied - * 4Mb regions that have mappings. - */ -#define ZULU_SHADOW_BLK_RANGE 0x400000 -#define ZULU_SHADOW_BLK_MASK (~(ZULU_SHADOW_BLK_RANGE - 1)) - -struct zulu_shadow_blk { - avl_node_t link; /* must be at beginning of struct */ - uint64_t ivaddr; /* base address of this node */ - uint64_t ref_count; - uint64_t min_addr; - uint64_t max_addr; -}; -#define ZULU_SHADOW_BLK_LINK_OFFSET (0) - -struct zulu_hat { - struct xhat zulu_xhat; - kmutex_t lock; - avl_tree_t shadow_tree; - char magic; /* =42 to mark our data for mdb */ - unsigned in_fault : 1; - unsigned freed : 1; - unsigned map8k : 1; - unsigned map64k : 1; - unsigned map512k : 1; - unsigned map4m : 1; - short zulu_ctx; - unsigned short zulu_tsb_size; /* TODO why not a constant? */ - struct zulu_hat_blk **hash_tbl; - struct zulu_tte *zulu_tsb; - struct zulu_shadow_blk *sblk_last; /* last sblk looked up */ - uint64_t fault_ivaddr_last; /* last translation loaded */ - caddr_t vaddr_max; - hrtime_t last_used; - void *zdev; -}; - -#define ZULU_HAT2AS(h) ((h)->zulu_xhat.xhat_as) - -/* - * Assembly language function for TSB lookups - */ -uint64_t zulu_hat_tsb_lookup_tl0(struct zulu_hat *zhat, caddr_t vaddr); - -/* - * zuluvm's interface to zulu_hat - */ - -int zulu_hat_load(struct zulu_hat *zhat, caddr_t vaddr, enum seg_rw rw, int *); - -int zulu_hat_init(); -int zulu_hat_destroy(); -int zulu_hat_attach(void *arg); -int zulu_hat_detach(void *arg); -struct zulu_hat *zulu_hat_proc_attach(struct as *as, void *zdev); -void zulu_hat_proc_detach(struct zulu_hat *zhat); - -void zulu_hat_validate_ctx(struct zulu_hat *zhat); -void zulu_hat_terminate(struct zulu_hat *zhat); - -#endif /* _ASM */ - -#ifdef __cplusplus -} -#endif - -#endif /* __ZULU_HAT_INCL__ */ diff --git a/usr/src/uts/sun4u/sys/zulumod.h b/usr/src/uts/sun4u/sys/zulumod.h deleted file mode 100644 index cce574a1dd..0000000000 --- a/usr/src/uts/sun4u/sys/zulumod.h +++ /dev/null @@ -1,262 +0,0 @@ -/* - * CDDL HEADER START - * - * The contents of this file are subject to the terms of the - * Common Development and Distribution License (the "License"). - * You may not use this file except in compliance with the License. - * - * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE - * or http://www.opensolaris.org/os/licensing. - * See the License for the specific language governing permissions - * and limitations under the License. - * - * When distributing Covered Code, include this CDDL HEADER in each - * file and include the License file at usr/src/OPENSOLARIS.LICENSE. - * If applicable, add the following below this CDDL HEADER, with the - * fields enclosed by brackets "[]" replaced with your own identifying - * information: Portions Copyright [yyyy] [name of copyright owner] - * - * CDDL HEADER END - */ -/* - * Copyright 2006 Sun Microsystems, Inc. All rights reserved. - * Use is subject to license terms. - */ - -#ifndef _ZULUMOD_H -#define _ZULUMOD_H - -#pragma ident "%Z%%M% %I% %E% SMI" - -#ifdef __cplusplus -extern "C" { -#endif - -#include <sys/int_const.h> -#include <sys/zuluvm.h> - -#ifndef _ASM - -#include <sys/zulu_hat.h> -#include <sys/sysmacros.h> - -#define ZULUVM_VERSION_STR(a) #a -#define ZULUVM_VERSION(a) ZULUVM_VERSION_STR(a) -#define ZULUVM_MOD_VERSION \ - ZULUVM_VERSION(XHAT_PROVIDER_VERSION) "." \ - ZULUVM_VERSION(ZULUVM_INTERFACE_VERSION) - -#define ZULUDCHKFUNC(_p1, _p2, _p3) \ - ((_p1) != NULL && (_p1)->_p2 != NULL) ? \ - (_p1)->_p2 _p3 : ZULUVM_NO_SUPPORT -#define ZULUDCHKPROC(_p1, _p2, _p3) \ - if ((_p1) != NULL && (_p1)->_p2 != NULL) (_p1)->_p2 _p3 - -#define zulud_set_itlb_pc(_devp, _a, _b) \ - ZULUDCHKPROC((_devp)->dops, set_itlb_pc, (_a, _b)) -#define zulud_set_dtlb_pc(_devp, _a, _b) \ - ZULUDCHKPROC((_devp)->dops, set_dtlb_pc, (_a, _b)) -#define zulud_write_tte(_devp, _a, _b, _c, _d, _e, _f) \ - ZULUDCHKFUNC((_devp)->dops, write_tte, (_a, _b, _c, _d, _e, _f)) -#define zulud_tlb_done(_devp, _a, _b, _c) \ - ZULUDCHKPROC((_devp)->dops, tlb_done, (_a, _b, _c)) -#define zulud_demap_page(_devp, _a, _b, _c) \ - ZULUDCHKPROC((_devp)->dops, demap_page, (_a, _b, _c)) -#define zulud_demap_ctx(_devp, _a, _b) \ - ZULUDCHKPROC((_devp)->dops, demap_ctx, (_a, _b)) - -#endif - -#define ZULUVM_DATA0_IDX 0 -#define ZULUVM_DATA1_IDX 1 -#define ZULUVM_DATA2_IDX 2 -#define ZULUVM_DATA3_IDX 3 -#define ZULUVM_DATA4_IDX 4 -#define ZULUVM_DATA5_IDX 5 -#define ZULUVM_DATA6_IDX 6 -#define ZULUVM_DATA7_IDX 7 - -#define ZULUVM_IDX2FLAG(i) (1 << (7 - i)) -#define ZULUVM_DATA0_FLAG ZULUVM_IDX2FLAG(ZULUVM_DATA0_IDX) -#define ZULUVM_DATA1_FLAG ZULUVM_IDX2FLAG(ZULUVM_DATA1_IDX) -#define ZULUVM_DATA2_FLAG ZULUVM_IDX2FLAG(ZULUVM_DATA2_IDX) -#define ZULUVM_DATA3_FLAG ZULUVM_IDX2FLAG(ZULUVM_DATA3_IDX) -#define ZULUVM_DATA4_FLAG ZULUVM_IDX2FLAG(ZULUVM_DATA4_IDX) -#define ZULUVM_DATA5_FLAG ZULUVM_IDX2FLAG(ZULUVM_DATA5_IDX) -#define ZULUVM_DATA6_FLAG ZULUVM_IDX2FLAG(ZULUVM_DATA6_IDX) -#define ZULUVM_DATA7_FLAG ZULUVM_IDX2FLAG(ZULUVM_DATA7_IDX) - -#define ZULUVM_TLB_ADDR_IDX ZULUVM_DATA0_IDX -#define ZULUVM_TLB_TYPE_IDX ZULUVM_DATA1_IDX -#define ZULUVM_TLB_TTE_IDX ZULUVM_DATA2_IDX -#define ZULUVM_TLB_ERRCODE_IDX ZULUVM_DATA3_IDX - -#define ZULUVM_DATA_FLAGS (ZULUVM_DATA1_FLAG | \ - ZULUVM_DATA6_FLAG) - -#define ZULUVM_GET_TLB_TTE(devp) \ - (devp)->zvm.idata[ZULUVM_TLB_TTE_IDX] -#define ZULUVM_GET_TLB_ADDR(devp) \ - (devp)->zvm.idata[ZULUVM_TLB_ADDR_IDX] -#define ZULUVM_GET_TLB_TYPE(devp) (ZULUVM_DMA_MASK & \ - (devp)->zvm.idata[ZULUVM_TLB_TYPE_IDX]) -#define ZULUVM_GET_TLB_ERRCODE(devp) (int)(0xffffffff & \ - (devp)->zvm.idata[ZULUVM_TLB_ERRCODE_IDX]) - -#define ZULUVM_MAX_DEV 2 -#define ZULUVM_PIL PIL_2 -#define ZULUVM_NUM_PGSZS 4 - -#define ZULUVM_STATE_IDLE 0 -#define ZULUVM_STATE_STOPPED 1 -#define ZULUVM_STATE_CANCELED 2 -#define ZULUVM_STATE_TLB_PENDING 3 -#define ZULUVM_STATE_INTR_QUEUED 4 -#define ZULUVM_STATE_INTR_PENDING 5 -#define ZULUVM_STATE_WRITE_TTE 6 - -#ifndef _ASM - -typedef struct { - uint64_t idata[4]; /* mondo pkt copy area */ - void *arg; /* arg for device calls */ - uint64_t mmu_pa; /* phy. addr of MMU regs */ - struct zuluvm_proc *proc1; - struct zuluvm_proc *proc2; - volatile uint32_t state; /* state of tlb miss handling */ - uint64_t intr_num; /* our soft intr number */ - short dmv_intr; /* dmv interrupt handle */ -#ifdef ZULUVM_STATS - int cancel; - int tlb_miss[ZULUVM_NUM_PGSZS]; - int pagefault; - int no_mapping; - int preload; - int migrate; - int pagesize; - int itlb1miss; - int dtlb1miss; - int itlb2miss; - int dtlb2miss; - int demap_page; - int demap_ctx; -#endif - uint64_t pfnbuf[50]; - int pfncnt; -} zuluvm_miss_t; - -#ifdef ZULUVM_STATS -#define ZULUVM_STATS_MISS(devp, sz) (devp)->zvm.tlb_miss[sz]++ -#define ZULUVM_STATS_PAGEFAULT(devp) (devp)->zvm.pagefault++ -#define ZULUVM_STATS_NOMAP(devp) (devp)->zvm.no_mapping++ -#define ZULUVM_STATS_PRELOAD(devp) (devp)->zvm.preload++ -#define ZULUVM_STATS_MIGRATE(devp) (devp)->zvm.migrate++ -#define ZULUVM_STATS_PAGEZISE(devp) (devp)->zvm.pagesize++ -#define ZULUVM_STATS_CANCEL(devp) (devp)->zvm.cancel++ -#define ZULUVM_STATS_DEMAP_PAGE(devp) (devp)->zvm.demap_page++ -#define ZULUVM_STATS_DEMAP_CTX(devp) (devp)->zvm.demap_ctx++ -#else -#define ZULUVM_STATS_MISS(devp, sz) -#define ZULUVM_STATS_PAGEFAULT(devp) -#define ZULUVM_STATS_NOMAP(devp) -#define ZULUVM_STATS_PRELOAD(devp) -#define ZULUVM_STATS_MIGRATE(devp) -#define ZULUVM_STATS_PAGEZISE(devp) -#define ZULUVM_STATS_CANCEL(devp) -#define ZULUVM_STATS_DEMAP_PAGE(devp) -#define ZULUVM_STATS_DEMAP_CTX(devp) -#endif - -#define ZULUVM_MAX_INTR 32 - -typedef struct { - short offset; - short ino; -} zuluvm_intr_t; - -/* - * This structure contains per device data. - * It is protected by dev_lck. - */ -typedef struct { - zuluvm_miss_t zvm; /* tlb miss state */ - volatile uint64_t *imr; /* intr mapping regs */ - struct zuluvm_proc *procs; /* protected by proc_lck */ - dev_info_t *dip; /* device driver instance */ - zulud_ops_t *dops; /* device drv operations */ - kmutex_t load_lck; /* protects in_intr */ - kmutex_t dev_lck; /* protects this struct */ - kmutex_t proc_lck; /* protects active procs */ - kcondvar_t intr_wait; /* sync for as_free */ - int intr_flags; - int in_intr; - kmutex_t park_lck; /* page fault thread */ - kcondvar_t park_cv; - int parking; - int agentid; /* zulu's agent id */ - zuluvm_intr_t interrupts[ZULUVM_MAX_INTR]; -} zuluvm_state_t; - -#define ZULUVM_INTR_OFFSET offsetof(zuluvm_state_t, interrupts) -#define ZULUVM_INTR2INO(addr) (((zuluvm_intr_t *)(addr))->ino) -#define ZULUVM_INTR2ZDEV(addr) \ - (zuluvm_state_t *)((caddr_t)addr - (ZULUVM_INTR2INO(addr) * \ - sizeof (zuluvm_intr_t)) - ZULUVM_INTR_OFFSET) - -typedef struct zuluvm_proc { - struct zulu_hat *zhat; - zuluvm_state_t *zdev; /* back ptr to dev instance */ - unsigned short refcnt; /* keep this until ref == 0 */ - short valid; /* if valid is 0 then don't use */ - struct zuluvm_proc *next; - struct zuluvm_proc *prev; -} zuluvm_proc_t; - -#define ZULUVM_DO_INTR1 INT32_C(1) -#define ZULUVM_WAIT_INTR1 INT32_C(2) -#define ZULUVM_DO_INTR2 INT32_C(4) -#define ZULUVM_WAIT_INTR2 INT32_C(8) - -int zuluvm_change_state(uint32_t *state_pa, int new, int assume); -void zuluvm_demap_page(void *, struct hat *, short, caddr_t, uint_t); -void zuluvm_demap_ctx(void *, short); -void zuluvm_dmv_tlbmiss_tl1(void); -void zuluvm_load_tte(struct zulu_hat *zhat, caddr_t addr, uint64_t pfn, - int perm, int size); - - -#endif - -/* - * The following defines are copied from the ZFB and ZULU - * workspaces. We re-define them here since we can't have - * a dependency onto files outside our consolidation - */ -#define ZULUVM_IMR_V_MASK UINT64_C(0x0000000080000000) -#define ZULUVM_IMR_TARGET_SHIFT INT32_C(26) -#define ZULUVM_IMR_MAX INT32_C(0x3f) - -#define ZULUVM_ZFB_MMU_TLB_D_V_MASK 0x8000000000000000 -#define ZULUVM_ZFB_MMU_TLB_D_PA_SHIFT 0xD /* 13 bits */ -#define ZULUVM_ZFB_MMU_TLB_D_C_MASK 0x20 -#define ZULUVM_ZFB_MMU_TLB_D_SZ_SHIFT 0x3D /* 61 */ -#define ZULUVM_ZFB_MMU_TLB_D_SZ_MASK 0x6000000000000000 -#define ZULUVM_ZFB_MMU_TLB_D_W_MASK 0x2 -#define ZULUVM_ZFB_MMU_TLB_CR_IMISS_MASK 0x2 -#define ZULUVM_ZFB_MMU_TLB_CR_DMISS_MASK 0x1 -#define ZULUVM_ZFB_MMU_DTLB_PAGE_SZ_2_MASK 0xc /* DTLB2 Page size */ -#define ZULUVM_ZFB_MMU_DTLB_PAGE_SZ_2_SHIFT 2 -#define ZULUVM_DTLB_PAGE_SZ 0x8 -#define ZULUVM_ITLB_DATA_IN 0x18 -#define ZULUVM_DTLB_DATA_IN 0x28 -#define ZULUVM_TLB_CONTROL 0 -#define ZULUVM_ITLB_MISS_ICR 0x0 -#define ZULUVM_DTLB_MISS_ICR 0x8 -#define ZULUVM_DMA1_TSB_BASE 0x50 -#define ZULUVM_DMA2_TSB_BASE 0x68 - -#ifdef __cplusplus -} -#endif - -#endif /* _ZULUMOD_H */ diff --git a/usr/src/uts/sun4u/sys/zuluvm.h b/usr/src/uts/sun4u/sys/zuluvm.h deleted file mode 100644 index d36a63b9bc..0000000000 --- a/usr/src/uts/sun4u/sys/zuluvm.h +++ /dev/null @@ -1,121 +0,0 @@ -/* - * CDDL HEADER START - * - * The contents of this file are subject to the terms of the - * Common Development and Distribution License, Version 1.0 only - * (the "License"). You may not use this file except in compliance - * with the License. - * - * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE - * or http://www.opensolaris.org/os/licensing. - * See the License for the specific language governing permissions - * and limitations under the License. - * - * When distributing Covered Code, include this CDDL HEADER in each - * file and include the License file at usr/src/OPENSOLARIS.LICENSE. - * If applicable, add the following below this CDDL HEADER, with the - * fields enclosed by brackets "[]" replaced with your own identifying - * information: Portions Copyright [yyyy] [name of copyright owner] - * - * CDDL HEADER END - */ -/* - * Copyright 2003 Sun Microsystems, Inc. All rights reserved. - * Use is subject to license terms. - */ - -#ifndef __ZULUVM_INCL__ -#define __ZULUVM_INCL__ - -#pragma ident "%Z%%M% %I% %E% SMI" - -#ifdef __cplusplus -extern "C" { -#endif - -/* zulud interface */ - -#ifndef _ASM - -#include <sys/dditypes.h> - -typedef struct { - caddr_t addr; - size_t len; - int tlbtype; -} zulud_preload_t; - -typedef struct { - int version; - int (*set_itlb_pc)(void *handle, uint64_t mondo); - int (*set_dtlb_pc)(void *handle, uint64_t mondo); - int (*set_suspendAck_pc)(void *handle, uint64_t mondo); - int (*write_tte)(void *handle, int ttesize, uint64_t tag, - pfn_t pfn, int permission, int tlbtype); - int (*tlb_done)(void *handle, int tlbtype, int status); - int (*demap_page)(void *handle, caddr_t vaddr, short ctx); - int (*demap_ctx)(void *handle, short ctx); - int (*dma_suspend_ack)(void *handle); - int (*set_tsb)(void *handle, int tlbtype, uint64_t tsbreg); -} zulud_ops_t; - -#endif - -#define ZULUVM_SUCCESS 0 -#define ZULUVM_ERROR 1 -#define ZULUVM_NO_TTE 2 -#define ZULUVM_INVALID_MISS 3 -#define ZULUVM_NO_DEV 4 -#define ZULUVM_NO_HAT 5 -#define ZULUVM_NO_MAP 6 -#define ZULUVM_VERSION_MISMATCH 7 -#define ZULUVM_TTE_DELAY 8 -#define ZULUVM_MISS_CANCELED 9 -#define ZULUVM_BAD_IDX 10 -#define ZULUVM_WATCH_POINT 11 -#define ZULUVM_NO_SUPPORT 12 -#define ZULUVM_CTX_LOCKED 13 - -#define ZULUVM_ITLB_FLAG 0x1 -#define ZULUVM_DMA_FLAG 0x2 -#define ZULUVM_DMA_MASK 0x3 - -#define ZULUVM_DMA1 0 -#define ZULUVM_DMA2 ZULUVM_DMA_FLAG -#define ZULUVM_ITLB1 ZULUVM_ITLB_FLAG -#define ZULUVM_ITLB2 (ZULUVM_ITLB_FLAG | ZULUVM_DMA_FLAG) -#define ZULUVM_INVAL 0x4 - -#ifndef _ASM - -/* zuluvm interface */ - -#define ZULUVM_INTERFACE_VERSION 1 /* inc with every intf change */ - -typedef void * zuluvm_info_t; -int zuluvm_init(zulud_ops_t *ops, int **pagesizes); -int zuluvm_fini(void); -int zuluvm_alloc_device(dev_info_t *devi, void *arg, zuluvm_info_t *devp, - caddr_t mmu, caddr_t imr); -int zuluvm_free_device(zuluvm_info_t devp); -int zuluvm_dma_add_proc(zuluvm_info_t devp, uint64_t *cookie); -int zuluvm_dma_delete_proc(zuluvm_info_t devp, uint64_t cookie); -int zuluvm_dma_alloc_ctx(zuluvm_info_t devp, int dma, short *ctx, - uint64_t *tsb); -int zuluvm_dma_preload(zuluvm_info_t devp, int dma, int num, - zulud_preload_t *list); -int zuluvm_dma_free_ctx(zuluvm_info_t devp, int dma); -int zuluvm_add_intr(zuluvm_info_t devp, int ino, uint_t (*handler)(caddr_t), - caddr_t arg); -int zuluvm_rem_intr(zuluvm_info_t devp, int ino); -int zuluvm_enable_intr(zuluvm_info_t devp, int num); -int zuluvm_disable_intr(zuluvm_info_t devp, int num); -int zuluvm_park(zuluvm_info_t devp); - -#endif - -#ifdef __cplusplus -} -#endif - -#endif /* __ZULUVM_INCL__ */ diff --git a/usr/src/uts/sun4u/vm/zulu_hat.c b/usr/src/uts/sun4u/vm/zulu_hat.c deleted file mode 100644 index 5ecadc028f..0000000000 --- a/usr/src/uts/sun4u/vm/zulu_hat.c +++ /dev/null @@ -1,1469 +0,0 @@ -/* - * CDDL HEADER START - * - * The contents of this file are subject to the terms of the - * Common Development and Distribution License, Version 1.0 only - * (the "License"). You may not use this file except in compliance - * with the License. - * - * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE - * or http://www.opensolaris.org/os/licensing. - * See the License for the specific language governing permissions - * and limitations under the License. - * - * When distributing Covered Code, include this CDDL HEADER in each - * file and include the License file at usr/src/OPENSOLARIS.LICENSE. - * If applicable, add the following below this CDDL HEADER, with the - * fields enclosed by brackets "[]" replaced with your own identifying - * information: Portions Copyright [yyyy] [name of copyright owner] - * - * CDDL HEADER END - */ -/* - * Copyright 2005 Sun Microsystems, Inc. All rights reserved. - * Use is subject to license terms. - */ - -#include <sys/types.h> -#include <sys/cmn_err.h> -#include <sys/mman.h> -#include <sys/sunddi.h> -#include <sys/tnf_probe.h> -#include <vm/hat_sfmmu.h> -#include <vm/as.h> -#include <vm/xhat.h> -#include <vm/xhat_sfmmu.h> -#include <sys/zulu_hat.h> -#include <sys/zulumod.h> - -/* - * This file contains the implementation of zulu_hat: an XHAT provider - * to support the MMU for the XVR-4000 graphics accelerator (code name zulu). - * - * The zulu hat is linked into the kernel misc module zuluvm. - * zuluvm provides services that the zulu device driver module requires - * that are not part of the standard ddi. See PSARC 2002/231. - * - * The zulu driver is delivered by the graphics consolidation. - * zuluvm is in ON workspace. - * - * There are two types of interfaces provided by zulu_hat - * 1. The set of functions and data structures used by zuluvm to obtain - * tte entries for the zulu MMU and to manage the association between - * user process's address spaces and zulu graphics contexts. - * - * 2. The entry points required for an XHAT provider: zulu_hat_ops - */ - -/* - * zulu_ctx_tab contains an array of pointers to the zulu_hats. - * - * During zulu graphics context switch, the zulu MMU's current context register - * is set to the index of the process's zulu hat's location in the array - * zulu_ctx_tab. - * - * This allows the TL=1 TLB miss handler to quickly find the zulu hat and - * lookup a tte in the zulu hat's TSB. - * - * To synchronize with the trap handler we use bit zero of - * the pointer as a lock bit. See the function zulu_ctx_tsb_lock_enter(). - * - * If the trap handler finds the ctx locked it doesn't wait, it - * posts a soft interrupt which is handled at TL=0. - */ - -#define ZULU_HAT_MAX_CTX 32 -struct zulu_hat *zulu_ctx_tab[ZULU_HAT_MAX_CTX]; - -/* - * To avoid searching through the whole zulu_ctx_tab for a free slot, - * we maintain the value of zulu_ctx_search_start. - * - * This value is a guess as to where a free slot in the context table might be. - * All slots < zulu_ctx_search_start are definitely occupied. - */ -static int zulu_ctx_search_start = 0; - - -/* - * this mutex protects the zulu_ctx_tab and zulu_ctx_search_start - */ -static kmutex_t zulu_ctx_lock; - - -uint64_t zulu_tsb_hit = 0; /* assembly code increments this */ -static uint64_t zulu_tsb_miss = 0; -static uint64_t zulu_as_fault = 0; - -/* - * The zulu device has two zulu data mmus. - * We use the base pagesize for one of them and the and 4M for the other. - */ -extern int zuluvm_base_pgsize; - - - -/* - * call zuluvm to remove translations for a page - */ -static void -zulu_hat_demap_page(struct zulu_hat *zhat, caddr_t vaddr, int size) -{ - if (zhat->zulu_ctx < 0) { - /* context has been stolen, so page is already demapped */ - return; - } - zuluvm_demap_page(zhat->zdev, NULL, zhat->zulu_ctx, vaddr, size); -} - -static void -zulu_hat_demap_ctx(void *zdev, int zulu_ctx) -{ - if (zulu_ctx < 0) { - /* context has been stolen */ - return; - } - zuluvm_demap_ctx(zdev, zulu_ctx); -} - - -/* - * steal the least recently used context slot. - */ -static int -zulu_hat_steal_ctx() -{ - int ctx; - hrtime_t delta = INT64_MAX; - struct zulu_hat *zhat_oldest = NULL; - - ASSERT(mutex_owned(&zulu_ctx_lock)); - - for (ctx = 0; ctx < ZULU_HAT_MAX_CTX; ctx++) { - struct zulu_hat *zhat = ZULU_CTX_GET_HAT(ctx); - - /* - * we shouldn't be here unless all slots are occupied - */ - ASSERT(zhat != NULL); - - TNF_PROBE_3(steal_ctx_loop, "zulu_hat", /* CSTYLED */, - tnf_int, ctx, ctx, - tnf_long, last_used, zhat->last_used, - tnf_long, oldest, delta); - - if (zhat->last_used < delta) { - zhat_oldest = zhat; - delta = zhat->last_used; - } - } - - ASSERT(zhat_oldest != NULL); - - mutex_enter(&zhat_oldest->lock); - - /* Nobody should have the tsb lock bit set here */ - ASSERT(((uint64_t)zulu_ctx_tab[zhat_oldest->zulu_ctx] & ZULU_CTX_LOCK) - == 0); - - ctx = zhat_oldest->zulu_ctx; - zhat_oldest->zulu_ctx = -1; - - ZULU_CTX_SET_HAT(ctx, NULL); - - zulu_hat_demap_ctx(zhat_oldest->zdev, ctx); - - mutex_exit(&zhat_oldest->lock); - - TNF_PROBE_1(zulu_hat_steal_ctx, "zulu_hat", /* CSTYLED */, - tnf_int, ctx, ctx); - - return (ctx); -} - -/* - * find a slot in the context table for a zulu_hat - */ -static void -zulu_hat_ctx_alloc(struct zulu_hat *zhat) -{ - int ctx; - - mutex_enter(&zulu_ctx_lock); - - for (ctx = zulu_ctx_search_start; ctx < ZULU_HAT_MAX_CTX; ctx++) { - if (ZULU_CTX_IS_FREE(ctx)) { - zulu_ctx_search_start = ctx + 1; - break; - } - } - - if (ctx == ZULU_HAT_MAX_CTX) { - /* table is full need to steal an entry */ - zulu_ctx_search_start = ZULU_HAT_MAX_CTX; - ctx = zulu_hat_steal_ctx(); - } - - mutex_enter(&zhat->lock); - - ZULU_CTX_SET_HAT(ctx, zhat); - zhat->zulu_ctx = ctx; - - mutex_exit(&zhat->lock); - - mutex_exit(&zulu_ctx_lock); - - TNF_PROBE_2(zulu_hat_ctx_alloc, "zulu_hat", /* CSTYLED */, - tnf_opaque, zhat, zhat, tnf_int, ctx, ctx); -} - -/* - * zulu_hat_validate_ctx: Called before the graphics context associated - * with a given zulu hat becomes the current zulu graphics context. - * Make sure that the hat has a slot in zulu_ctx_tab. - */ -void -zulu_hat_validate_ctx(struct zulu_hat *zhat) -{ - if (zhat->zulu_ctx < 0) { - zulu_hat_ctx_alloc(zhat); - } - zhat->last_used = gethrtime(); -} - - -static void -zulu_hat_ctx_free(struct zulu_hat *zhat) -{ - TNF_PROBE_1(zulu_hat_ctx_free, "zulu_hat", /* CSTYLED */, - tnf_int, ctx, zhat->zulu_ctx); - - mutex_enter(&zulu_ctx_lock); - - mutex_enter(&zhat->lock); - if (zhat->zulu_ctx >= 0) { - ZULU_CTX_SET_HAT(zhat->zulu_ctx, NULL); - - if (zulu_ctx_search_start > zhat->zulu_ctx) { - zulu_ctx_search_start = zhat->zulu_ctx; - } - } - mutex_exit(&zhat->lock); - mutex_exit(&zulu_ctx_lock); -} - -/* - * Lock the zulu tsb for a given zulu_hat. - * - * We're just protecting against the TLB trap handler here. Other operations - * on the zulu_hat require entering the zhat's lock. - */ -static void -zulu_ctx_tsb_lock_enter(struct zulu_hat *zhat) -{ - uint64_t lck; - uint64_t *plck; - - ASSERT(mutex_owned(&zhat->lock)); - - if (zhat->zulu_ctx < 0) { - return; - } - plck = (uint64_t *)&zulu_ctx_tab[zhat->zulu_ctx]; - - for (; ; ) { - lck = *plck; - if (!(lck & ZULU_CTX_LOCK)) { - uint64_t old_lck, new_lck; - - new_lck = lck | ZULU_CTX_LOCK; - - old_lck = atomic_cas_64(plck, lck, new_lck); - - if (old_lck == lck) { - /* - * success - */ - break; - } - } - } -} - -static void -zulu_ctx_tsb_lock_exit(struct zulu_hat *zhat) -{ - uint64_t lck; - int zulu_ctx = zhat->zulu_ctx; - - if (zulu_ctx < 0) { - return; - } - lck = (uint64_t)zulu_ctx_tab[zulu_ctx]; - ASSERT(lck & ZULU_CTX_LOCK); - lck &= ~ZULU_CTX_LOCK; - zulu_ctx_tab[zulu_ctx] = (struct zulu_hat *)lck; -} - -/* - * Each zulu hat has a "shadow tree" which is a table of 4MB address regions - * for which the zhat has mappings. - * - * This table is maintained in an avl tree. - * Nodes in the tree are called shadow blocks (or sblks) - * - * This data structure allows unload operations by (address, range) to be - * much more efficent. - * - * We get called a lot for address ranges that have never been supplied - * to zulu. - */ - -/* - * compare the base address of two nodes in the shadow tree - */ -static int -zulu_shadow_tree_compare(const void *a, const void *b) -{ - struct zulu_shadow_blk *zba = (struct zulu_shadow_blk *)a; - struct zulu_shadow_blk *zbb = (struct zulu_shadow_blk *)b; - uint64_t addr_a = zba->ivaddr; - uint64_t addr_b = zbb->ivaddr; - - TNF_PROBE_2(zulu_shadow_tree_compare, "zulu_shadow_tree", /* CSTYLED */, - tnf_opaque, addr_a, addr_a, tnf_opaque, addr_b, addr_b); - - if (addr_a < addr_b) { - return (-1); - } else if (addr_a > addr_b) { - return (1); - } else { - return (0); - } -} - -/* - * lookup the entry in the shadow tree for a given virtual address - */ -static struct zulu_shadow_blk * -zulu_shadow_tree_lookup(struct zulu_hat *zhat, uint64_t ivaddr, - avl_index_t *where) -{ - struct zulu_shadow_blk proto; - struct zulu_shadow_blk *sblk; - - proto.ivaddr = ivaddr & ZULU_SHADOW_BLK_MASK; - - /* - * pages typically fault in in order so we cache the last shadow - * block that was referenced so we usually get to reduce calls to - * avl_find. - */ - if ((zhat->sblk_last != NULL) && - (proto.ivaddr == zhat->sblk_last->ivaddr)) { - sblk = zhat->sblk_last; - } else { - sblk = (struct zulu_shadow_blk *)avl_find(&zhat->shadow_tree, - &proto, where); - zhat->sblk_last = sblk; - } - - TNF_PROBE_2(zulu_shadow_tree_lookup, "zulu_shadow_tree", /* CSTYLED */, - tnf_opaque, ivaddr, proto.ivaddr, - tnf_opaque, where, where ? *where : ~0); - - return (sblk); -} - -/* - * insert a sblk into the shadow tree for a given zblk. - * If a sblk already exists, just increment it's refcount. - */ -static void -zulu_shadow_tree_insert(struct zulu_hat *zhat, struct zulu_hat_blk *zblk) -{ - avl_index_t where; - struct zulu_shadow_blk *sblk = NULL; - uint64_t ivaddr; - uint64_t end; - - ivaddr = zblk->zulu_hat_blk_vaddr & ZULU_SHADOW_BLK_MASK; - - end = zblk->zulu_hat_blk_vaddr + ZULU_HAT_PGSZ(zblk->zulu_hat_blk_size); - - sblk = zulu_shadow_tree_lookup(zhat, ivaddr, &where); - if (sblk != NULL) { - sblk->ref_count++; - - end = zblk->zulu_hat_blk_vaddr + - ZULU_HAT_PGSZ(zblk->zulu_hat_blk_size); - if (zblk->zulu_hat_blk_vaddr < sblk->min_addr) { - sblk->min_addr = zblk->zulu_hat_blk_vaddr; - } - /* - * a blk can set both the minimum and maximum when it - * is the first zblk added to a previously emptied sblk - */ - if (end > sblk->max_addr) { - sblk->max_addr = end; - } - } else { - sblk = kmem_zalloc(sizeof (*sblk), KM_SLEEP); - sblk->ref_count = 1; - sblk->ivaddr = ivaddr; - sblk->min_addr = zblk->zulu_hat_blk_vaddr; - sblk->max_addr = end; - zhat->sblk_last = sblk; - - avl_insert(&zhat->shadow_tree, sblk, where); - } - zblk->zulu_shadow_blk = sblk; - TNF_PROBE_2(zulu_shadow_tree_insert, "zulu_shadow_tree", /* CSTYLED */, - tnf_opaque, vaddr, ivaddr, - tnf_opaque, ref_count, sblk->ref_count); -} - -/* - * decrement the ref_count for the sblk that corresponds to a given zblk. - * When the ref_count goes to zero remove the sblk from the tree and free it. - */ - -static void -zulu_shadow_tree_delete(struct zulu_hat *zhat, struct zulu_hat_blk *zblk) -{ - struct zulu_shadow_blk *sblk; - - ASSERT(zblk->zulu_shadow_blk != NULL); - - sblk = zblk->zulu_shadow_blk; - - TNF_PROBE_2(zulu_shadow_tree_delete, "zulu_shadow_tree", /* CSTYLED */, - tnf_opaque, vaddr, sblk->ivaddr, - tnf_opaque, ref_count, sblk->ref_count-1); - - if (--sblk->ref_count == 0) { - if (zhat->sblk_last == sblk) { - zhat->sblk_last = NULL; - } - sblk->min_addr = sblk->ivaddr + ZULU_SHADOW_BLK_RANGE; - sblk->max_addr = sblk->ivaddr; - } else { - /* - * Update the high and low water marks for this sblk. - * These are estimates, because we don't know if the previous - * or next region are actually occupied, but we can tell - * whether the previous values have become invalid. - * - * In the most often applied case a segment is being - * unloaded, and the min_addr will be kept up to date as - * the zblks are deleted in order. - */ - uint64_t end = zblk->zulu_hat_blk_vaddr + - ZULU_HAT_PGSZ(zblk->zulu_hat_blk_size); - - if (zblk->zulu_hat_blk_vaddr == sblk->min_addr) { - sblk->min_addr = end; - } - if (end == sblk->max_addr) { - sblk->max_addr = zblk->zulu_hat_blk_vaddr; - } - } - - zblk->zulu_shadow_blk = NULL; -} - -static void -zulu_shadow_tree_destroy(struct zulu_hat *zhat) -{ - struct zulu_shadow_blk *sblk; - void *cookie = NULL; - - while ((sblk = (struct zulu_shadow_blk *)avl_destroy_nodes( - &zhat->shadow_tree, &cookie)) != NULL) { - TNF_PROBE_2(shadow_tree_destroy, "zulu_hat", /* CSTYLED */, - tnf_opaque, vaddr, sblk->ivaddr, - tnf_opaque, ref_count, sblk->ref_count); - kmem_free(sblk, sizeof (*sblk)); - } - avl_destroy(&zhat->shadow_tree); -} - -/* - * zulu_hat_insert_map: - * - * Add a zulu_hat_blk to the a zhat's mappings list. - * - * Several data stuctures are used - * tsb: for simple fast lookups by the trap handler - * hash table: for efficent lookups by address, range - * An shadow tree of 4MB ranges with mappings for unloading big regions. - */ -static void -zulu_hat_insert_map(struct zulu_hat *zhat, struct zulu_hat_blk *zblk) -{ - int tsb_hash; - - tsb_hash = ZULU_TSB_HASH(zblk->zulu_hat_blk_vaddr, - zblk->zulu_hat_blk_size, zhat->zulu_tsb_size); - - TNF_PROBE_3(zulu_hat_insert_map, "zulu_hat", /* CSTYLED */, - tnf_opaque, zblkp, zblk, - tnf_opaque, vaddr, zblk->zulu_hat_blk_vaddr, - tnf_opaque, hash, tsb_hash); - - ASSERT(tsb_hash < zhat->zulu_tsb_size); - - zulu_shadow_tree_insert(zhat, zblk); - - /* - * The hash table is an array of buckets. Each bucket is the - * head of a linked list of mappings who's address hashess to the bucket - * New entries go to the head of the list. - */ - zblk->zulu_hash_prev = NULL; - zblk->zulu_hash_next = ZULU_MAP_HASH_HEAD(zhat, - zblk->zulu_hat_blk_vaddr, zblk->zulu_hat_blk_size); - if (zblk->zulu_hash_next) { - zblk->zulu_hash_next->zulu_hash_prev = zblk; - } - ZULU_MAP_HASH_HEAD(zhat, zblk->zulu_hat_blk_vaddr, - zblk->zulu_hat_blk_size) = zblk; - - zulu_ctx_tsb_lock_enter(zhat); - zhat->zulu_tsb[tsb_hash] = zblk->zulu_hat_blk_tte; - zulu_ctx_tsb_lock_exit(zhat); -} - -/* - * remove a block from a zhat - */ -static void -zulu_hat_remove_map(struct zulu_hat *zhat, struct zulu_hat_blk *zblk) -{ - int tsb_hash = ZULU_TSB_HASH(zblk->zulu_hat_blk_vaddr, - zblk->zulu_hat_blk_size, zhat->zulu_tsb_size); - - TNF_PROBE_2(zulu_hat_remove_map, "zulu_hat", /* CSTYLED */, - tnf_opaque, vaddr, zblk->zulu_hat_blk_vaddr, - tnf_opaque, hash, tsb_hash); - - ASSERT(tsb_hash < zhat->zulu_tsb_size); - ASSERT(mutex_owned(&zhat->lock)); - - zulu_shadow_tree_delete(zhat, zblk); - - /* - * first remove zblk from hash table - */ - if (zblk->zulu_hash_prev) { - zblk->zulu_hash_prev->zulu_hash_next = zblk->zulu_hash_next; - } else { - ZULU_MAP_HASH_HEAD(zhat, zblk->zulu_hat_blk_vaddr, - zblk->zulu_hat_blk_size) = NULL; - } - if (zblk->zulu_hash_next) { - zblk->zulu_hash_next->zulu_hash_prev = zblk->zulu_hash_prev; - } - zblk->zulu_hash_next = NULL; - zblk->zulu_hash_prev = NULL; - - /* - * then remove the tsb entry - */ - zulu_ctx_tsb_lock_enter(zhat); - if (zhat->zulu_tsb[tsb_hash].un.zulu_tte_addr == - zblk->zulu_hat_blk_vaddr) { - zhat->zulu_tsb[tsb_hash].zulu_tte_valid = 0; - } - zulu_ctx_tsb_lock_exit(zhat); -} - -/* - * look for a mapping to a given vaddr and page size - */ -static struct zulu_hat_blk * -zulu_lookup_map_bysize(struct zulu_hat *zhat, caddr_t vaddr, int page_sz) -{ - struct zulu_hat_blk *zblkp; - uint64_t ivaddr = (uint64_t)vaddr; - int blks_checked = 0; - - ASSERT(mutex_owned(&zhat->lock)); - - for (zblkp = ZULU_MAP_HASH_HEAD(zhat, ivaddr, page_sz); zblkp != NULL; - zblkp = zblkp->zulu_hash_next) { - uint64_t size; - uint64_t iaddr; - - blks_checked++; - - size = ZULU_HAT_PGSZ(zblkp->zulu_hat_blk_size); - iaddr = ZULU_VADDR((uint64_t)zblkp->zulu_hat_blk_vaddr); - - if (iaddr <= ivaddr && (iaddr + size) > ivaddr) { - int tsb_hash; - - tsb_hash = ZULU_TSB_HASH(zblkp->zulu_hat_blk_vaddr, - zblkp->zulu_hat_blk_size, - zhat->zulu_tsb_size); - ASSERT(tsb_hash < zhat->zulu_tsb_size); - - zulu_ctx_tsb_lock_enter(zhat); - zhat->zulu_tsb[tsb_hash] = zblkp->zulu_hat_blk_tte; - zulu_ctx_tsb_lock_exit(zhat); - break; - } - - } - - TNF_PROBE_3(zulu_hat_lookup_map_bysz, "zulu_hat", /* CSTYLED */, - tnf_opaque, zblkp, zblkp, - tnf_int, blks_checked, blks_checked, - tnf_int, page_sz, page_sz); - - return (zblkp); -} - -/* - * Lookup a zblk for a given virtual address. - */ -static struct zulu_hat_blk * -zulu_lookup_map(struct zulu_hat *zhat, caddr_t vaddr) -{ - struct zulu_hat_blk *zblkp = NULL; - - /* - * if the hat is using 4M pages, look first for a 4M page - */ - if (zhat->map4m) { - zblkp = zulu_lookup_map_bysize(zhat, vaddr, ZULU_TTE4M); - if (zblkp != NULL) { - return (zblkp); - } - } - /* - * Otherwise look for a 8k page - * Note: if base pagesize gets increased to 64K remove this test - */ - if (zhat->map8k) { - zblkp = zulu_lookup_map_bysize(zhat, vaddr, ZULU_TTE8K); - if (zblkp != NULL) { - return (zblkp); - } - } - /* - * only if the page isn't found in the sizes that match the zulu mmus - * look for the inefficient 64K or 512K page sizes - */ - if (zhat->map64k) { - zblkp = zulu_lookup_map_bysize(zhat, vaddr, ZULU_TTE64K); - if (zblkp != NULL) { - return (zblkp); - } - } - if (zhat->map512k) { - zblkp = zulu_lookup_map_bysize(zhat, vaddr, ZULU_TTE512K); - } - - return (zblkp); -} - -/* - * zulu_hat_load: Load translation for given vaddr - */ -int -zulu_hat_load(struct zulu_hat *zhat, caddr_t vaddr, - enum seg_rw rw, int *ppg_size) -{ - faultcode_t as_err; - struct zulu_hat_blk *zblkp; - int rval; - uint64_t flags_pfn; - struct zulu_tte tte; - - TNF_PROBE_2(zulu_hat_load, "zulu_hat", /* CSTYLED */, - tnf_int, zulu_ctx, zhat->zulu_ctx, - tnf_opaque, vaddr, vaddr); - - mutex_enter(&zhat->lock); - ASSERT(zhat->zulu_ctx >= 0); - /* - * lookup in our tsb first - */ - zulu_ctx_tsb_lock_enter(zhat); - flags_pfn = zulu_hat_tsb_lookup_tl0(zhat, vaddr); - zulu_ctx_tsb_lock_exit(zhat); - - if (flags_pfn) { - uint64_t *p = (uint64_t *)&tte; - - p++; /* ignore the tag */ - *p = flags_pfn; /* load the flags */ - - zuluvm_load_tte(zhat, vaddr, flags_pfn, tte.zulu_tte_perm, - tte.zulu_tte_size); - if (ppg_size != NULL) { - *ppg_size = tte.zulu_tte_size; - } - - zulu_tsb_hit++; - mutex_exit(&zhat->lock); - return (0); - } - - zulu_tsb_miss++; - - zblkp = zulu_lookup_map(zhat, vaddr); - if (zblkp) { - tte = zblkp->zulu_hat_blk_tte; - tte.zulu_tte_pfn = ZULU_HAT_ADJ_PFN((&tte), vaddr); - zuluvm_load_tte(zhat, vaddr, tte.zulu_tte_pfn, - tte.zulu_tte_perm, tte.zulu_tte_size); - if (ppg_size != NULL) { - *ppg_size = tte.zulu_tte_size; - } - mutex_exit(&zhat->lock); - return (0); - } - - /* - * Set a flag indicating that we're processing a fault. - * See comments in zulu_hat_unload_region. - */ - zhat->in_fault = 1; - mutex_exit(&zhat->lock); - - zulu_as_fault++; - TNF_PROBE_0(calling_as_fault, "zulu_hat", /* CSTYLED */); - - as_err = as_fault((struct hat *)zhat, zhat->zulu_xhat.xhat_as, - (caddr_t)(ZULU_VADDR((uint64_t)vaddr) & PAGEMASK), - PAGESIZE, F_INVAL, rw); - - mutex_enter(&zhat->lock); - zhat->in_fault = 0; - if (ppg_size != NULL) { - /* - * caller wants to know the page size (used by preload) - */ - zblkp = zulu_lookup_map(zhat, vaddr); - if (zblkp != NULL) { - *ppg_size = zblkp->zulu_hat_blk_size; - } else { - *ppg_size = -1; - } - } - mutex_exit(&zhat->lock); - - TNF_PROBE_1(as_fault_returned, "zulu_hat", /* CSTYLED */, - tnf_int, as_err, as_err); - - if (as_err != 0) { - printf("as_fault returned %d\n", as_err); - rval = as_err; - } else if (zhat->freed) { - rval = -1; - } else { - rval = 0; - } - - return (rval); -} - -static struct xhat * -zulu_hat_alloc(void *arg) -{ - struct zulu_hat *zhat = kmem_zalloc(sizeof (struct zulu_hat), KM_SLEEP); - - (void) arg; - - zulu_hat_ctx_alloc(zhat); - - mutex_init(&zhat->lock, NULL, MUTEX_DEFAULT, NULL); - - zhat->zulu_tsb = kmem_zalloc(ZULU_TSB_SZ, KM_SLEEP); - zhat->zulu_tsb_size = ZULU_TSB_NUM; - zhat->hash_tbl = kmem_zalloc(ZULU_HASH_TBL_SZ, KM_SLEEP); - avl_create(&zhat->shadow_tree, zulu_shadow_tree_compare, - sizeof (zhat->shadow_tree), ZULU_SHADOW_BLK_LINK_OFFSET); - /* - * The zulu hat has a few opaque data structs embedded in it. - * This tag makes finding the our data easier with a debugger. - */ - zhat->magic = 0x42; - - zhat->freed = 0; - TNF_PROBE_1(zulu_hat_alloc, "zulu_hat", /* CSTYLED */, - tnf_int, zulu_ctx, zhat->zulu_ctx); - return ((struct xhat *)zhat); -} - -static void -zulu_hat_free(struct xhat *xhat) -{ - struct zulu_hat *zhat = (struct zulu_hat *)xhat; - - TNF_PROBE_1(zulu_hat_free, "zulu_hat", /* CSTYLED */, - tnf_int, zulu_ctx, zhat->zulu_ctx); - - zulu_shadow_tree_destroy(zhat); - kmem_free(zhat->hash_tbl, ZULU_HASH_TBL_SZ); - kmem_free(zhat->zulu_tsb, ZULU_TSB_SZ); - mutex_destroy(&zhat->lock); - kmem_free(xhat, sizeof (struct zulu_hat)); -} - -static void -zulu_hat_free_start(struct xhat *xhat) -{ - struct zulu_hat *zhat = (struct zulu_hat *)xhat; - - TNF_PROBE_1(zulu_hat_free_start, "zulu_hat", /* CSTYLED */, - tnf_int, zulu_ctx, zhat->zulu_ctx); - (void) xhat; -} - -/* - * zulu_hat_memload: This is the callback where the vm system gives us our - * translations - */ -static void -zulu_do_hat_memload(struct xhat *xhat, caddr_t vaddr, struct page *page, - uint_t attr, uint_t flags, int use_pszc) -{ - void *blk; - struct zulu_hat *zhat = (struct zulu_hat *)xhat; - struct zulu_hat_blk *zblk; - pfn_t pfn; - - TNF_PROBE_4(zulu_hat_memload, "zulu_hat", /* CSTYLED */, - tnf_int, zulu_ctx, zhat->zulu_ctx, - tnf_opaque, vaddr, vaddr, tnf_opaque, attr, attr, - tnf_opaque, flags, flags); - - /* - * keep track of the highest address that this zhat has had - * a mapping for. - * We use this in unload to avoid searching for regions that - * we've never seen. - * - * This is particularly useful avoiding repeated searches for - * for the process's mappings to the zulu hardware. These mappings - * are explicitly unloaded at each graphics context switch.. - * - * This takes advantage of the fact that the device addresses - * are always above than the heap where most DMA data is stored. - */ - if (vaddr > zhat->vaddr_max) { - zhat->vaddr_max = vaddr; - } - - pfn = xhat_insert_xhatblk(page, xhat, &blk); - zblk = (struct zulu_hat_blk *)blk; - zblk->zulu_hat_blk_vaddr = (uintptr_t)vaddr; - zblk->zulu_hat_blk_pfn = (uint_t)pfn; - /* - * The perm bit is actually in the tte which gets copied to the TSB - */ - zblk->zulu_hat_blk_perm = (attr & PROT_WRITE) ? 1 : 0; - zblk->zulu_hat_blk_size = use_pszc ? page->p_szc : 0; - zblk->zulu_hat_blk_valid = 1; - - switch (zblk->zulu_hat_blk_size) { - case ZULU_TTE8K: - zhat->map8k = 1; - break; - case ZULU_TTE64K: - zhat->map64k = 1; - break; - case ZULU_TTE512K: - zhat->map512k = 1; - break; - case ZULU_TTE4M: - zhat->map4m = 1; - break; - default: - panic("zulu_hat illegal page size\n"); - } - - mutex_enter(&zhat->lock); - - zulu_hat_insert_map(zhat, zblk); - if (!zhat->freed) { - zuluvm_load_tte(zhat, vaddr, zblk->zulu_hat_blk_pfn, - zblk->zulu_hat_blk_perm, zblk->zulu_hat_blk_size); - } - zhat->fault_ivaddr_last = - ZULU_VADDR((uint64_t)zblk->zulu_hat_blk_vaddr); - - mutex_exit(&zhat->lock); -} - -static void -zulu_hat_memload(struct xhat *xhat, caddr_t vaddr, struct page *page, - uint_t attr, uint_t flags) -{ - zulu_do_hat_memload(xhat, vaddr, page, attr, flags, 0); -} - -static void -zulu_hat_devload(struct xhat *xhat, caddr_t vaddr, size_t size, pfn_t pfn, - uint_t attr, int flags) -{ - struct page *pp = page_numtopp_nolock(pfn); - (void) size; - zulu_do_hat_memload(xhat, vaddr, pp, attr, (uint_t)flags, 1); -} - -static void -zulu_hat_memload_array(struct xhat *xhat, caddr_t addr, size_t len, - struct page **gen_pps, uint_t attr, uint_t flags) -{ - struct zulu_hat *zhat = (struct zulu_hat *)xhat; - - TNF_PROBE_3(zulu_hat_memload_array, "zulu_hat", /* CSTYLED */, - tnf_int, zulu_ctx, zhat->zulu_ctx, - tnf_opaque, addr, addr, - tnf_opaque, len, len); - - for (; len > 0; len -= ZULU_HAT_PGSZ((*gen_pps)->p_szc), - gen_pps += ZULU_HAT_NUM_PGS((*gen_pps)->p_szc)) { - zulu_do_hat_memload(xhat, addr, *gen_pps, attr, flags, 1); - - addr += ZULU_HAT_PGSZ((*gen_pps)->p_szc); - } -} - -static void -free_zblks(struct zulu_hat_blk *free_list) -{ - struct zulu_hat_blk *zblkp; - struct zulu_hat_blk *next; - - for (zblkp = free_list; zblkp != NULL; zblkp = next) { - next = zblkp->zulu_hash_next; - (void) xhat_delete_xhatblk((struct xhat_hme_blk *)zblkp, 0); - } -} - -static void -add_to_free_list(struct zulu_hat_blk **pfree_list, struct zulu_hat_blk *zblk) -{ - zblk->zulu_hash_next = *pfree_list; - *pfree_list = zblk; -} - -static void -zulu_hat_unload_region(struct zulu_hat *zhat, uint64_t ivaddr, size_t size, - struct zulu_shadow_blk *sblk, struct zulu_hat_blk **pfree_list) -{ - uint64_t end = ivaddr + size; - int found = 0; - - TNF_PROBE_2(zulu_hat_unload_region, "zulu_hat", /* CSTYLED */, - tnf_opaque, vaddr, ivaddr, tnf_opaque, size, size); - - /* - * check address against the low and highwater marks for mappings - * in this sblk - */ - if (ivaddr < sblk->min_addr) { - ivaddr = sblk->min_addr; - TNF_PROBE_1(zulu_hat_unload_skip, "zulu_hat", /* CSTYLED */, - tnf_opaque, ivaddr, ivaddr); - } - if (end > sblk->max_addr) { - end = sblk->max_addr; - TNF_PROBE_1(zulu_hat_unload_reg_skip, "zulu_hat", /* CSTYLED */, - tnf_opaque, end, end); - } - /* - * REMIND: It's not safe to touch the sblk after we enter this loop - * because it may get deleted. - */ - - while (ivaddr < end) { - uint64_t iaddr; - size_t pg_sz; - struct zulu_hat_blk *zblkp; - - zblkp = zulu_lookup_map(zhat, (caddr_t)ivaddr); - if (zblkp == NULL) { - ivaddr += PAGESIZE; - continue; - } - - iaddr = ZULU_VADDR((uint64_t)zblkp->zulu_hat_blk_vaddr); - pg_sz = ZULU_HAT_PGSZ(zblkp->zulu_hat_blk_size); - - found++; - - zulu_hat_remove_map(zhat, zblkp); - /* - * skip demap page if as_free has already been entered - * zuluvm demapped the context already - */ - if (!zhat->freed) { - if ((zhat->in_fault) && - (iaddr == zhat->fault_ivaddr_last)) { - /* - * We're being called from within as_fault to - * unload the last translation we loaded. - * - * This is probably due to watchpoint handling. - * Delay the demap for a millisecond - * to allow zulu to make some progress. - */ - drv_usecwait(1000); - zhat->fault_ivaddr_last = 0; - } - zulu_hat_demap_page(zhat, (caddr_t)iaddr, - zblkp->zulu_hat_blk_size); - } - - add_to_free_list(pfree_list, zblkp); - - if ((iaddr + pg_sz) >= end) { - break; - } - - ivaddr += pg_sz; - } - TNF_PROBE_1(zulu_hat_unload_region_done, "zulu_hat", /* CSTYLED */, - tnf_opaque, found, found); -} - -static void -zulu_hat_unload(struct xhat *xhat, caddr_t vaddr, size_t size, uint_t flags) -{ - struct zulu_hat *zhat = (struct zulu_hat *)xhat; - uint64_t ivaddr; - uint64_t end; - int found = 0; - struct zulu_hat_blk *free_list = NULL; - - (void) flags; - - TNF_PROBE_4(zulu_hat_unload, "zulu_hat", /* CSTYLED */, - tnf_int, zulu_ctx, zhat->zulu_ctx, - tnf_opaque, vaddr, vaddr, - tnf_opaque, vaddr_max, zhat->vaddr_max, - tnf_opaque, size, size); - - mutex_enter(&zhat->lock); - - /* - * The following test prevents us from searching for the user's - * mappings to the zulu device registers. Those mappings get unloaded - * every time a graphics context switch away from a given context - * occurs. - * - * Since the heap is located at smaller virtual addresses than the - * registers, this simple test avoids quite a bit of useless work. - */ - if (vaddr > zhat->vaddr_max) { - /* - * all existing mappings have lower addresses than vaddr - * no need to search further. - */ - mutex_exit(&zhat->lock); - return; - } - - ivaddr = (uint64_t)vaddr; - end = ivaddr + size; - - do { - struct zulu_shadow_blk *sblk; - - sblk = zulu_shadow_tree_lookup(zhat, ivaddr, NULL); - if (sblk != NULL) { - uint64_t sblk_end; - size_t region_size; - - found++; - - sblk_end = (ivaddr + ZULU_SHADOW_BLK_RANGE) & - ZULU_SHADOW_BLK_MASK; - - if (sblk_end < end) { - region_size = sblk_end - ivaddr; - } else { - region_size = end - ivaddr; - } - zulu_hat_unload_region(zhat, ivaddr, region_size, sblk, - &free_list); - - } - ivaddr += ZULU_SHADOW_BLK_RANGE; - } while (ivaddr < end); - - mutex_exit(&zhat->lock); - - free_zblks(free_list); - - TNF_PROBE_1(zulu_hat_unload_done, "zulu_hat", /* CSTYLED */, - tnf_int, found, found); -} - -static void -zulu_hat_unload_callback(struct xhat *xhat, caddr_t vaddr, size_t size, - uint_t flags, hat_callback_t *pcb) -{ - (void) size; - (void) pcb; - zulu_hat_unload(xhat, vaddr, size, flags); -} - - -/* - * unload one page - */ -static int -zulu_hat_pageunload(struct xhat *xhat, struct page *pp, uint_t flags, - void *xblk) -{ - struct zulu_hat_blk *zblk = (struct zulu_hat_blk *)xblk; - struct zulu_hat *zhat = (struct zulu_hat *)xhat; - int do_delete; - - (void) pp; - (void) flags; - - TNF_PROBE_3(zulu_hat_pageunload, "zulu_hat", /* CSTYLED */, - tnf_int, zulu_ctx, zhat->zulu_ctx, - tnf_opaque, vaddr, zblk->zulu_hat_blk_vaddr, - tnf_int, pg_size, zblk->zulu_hat_blk_size); - - mutex_enter(&zhat->lock); - if (zblk->zulu_shadow_blk != NULL) { - - do_delete = 1; - - zulu_hat_remove_map(zhat, zblk); - - /* - * now that the entry is removed from the TSB, remove the - * translation from the zulu hardware. - * - * Skip the demap if this as is in the process of being freed. - * The zuluvm as callback has demapped the whole context. - */ - if (!zhat->freed) { - zulu_hat_demap_page(zhat, - (caddr_t)(uintptr_t)(zblk->zulu_hat_blk_page << - ZULU_HAT_BP_SHIFT), - zblk->zulu_hat_blk_size); - } - } else { - /* - * This block has already been removed from the zulu_hat, - * it's on a free list waiting for our thread to release - * a mutex so it can be freed - */ - do_delete = 0; - - TNF_PROBE_0(zulu_hat_pageunload_skip, "zulu_hat", - /* CSTYLED */); - } - mutex_exit(&zhat->lock); - - if (do_delete) { - (void) xhat_delete_xhatblk(xblk, 1); - } - - return (0); -} - -static void -zulu_hat_swapout(struct xhat *xhat) -{ - struct zulu_hat *zhat = (struct zulu_hat *)xhat; - struct zulu_hat_blk *zblk; - struct zulu_hat_blk *free_list = NULL; - int i; - int nblks = 0; - - TNF_PROBE_1(zulu_hat_swapout, "zulu_hat", /* CSTYLED */, - tnf_int, zulu_ctx, zhat->zulu_ctx); - - mutex_enter(&zhat->lock); - - /* - * real swapout calls are rare so we don't do anything in - * particular to optimize them. - * - * Just loop over all buckets in the hash table and free each - * zblk. - */ - for (i = 0; i < ZULU_HASH_TBL_NUM; i++) { - struct zulu_hat_blk *next; - for (zblk = zhat->hash_tbl[i]; zblk != NULL; zblk = next) { - next = zblk->zulu_hash_next; - zulu_hat_remove_map(zhat, zblk); - add_to_free_list(&free_list, zblk); - nblks++; - } - } - - /* - * remove all mappings for this context from zulu hardware. - */ - zulu_hat_demap_ctx(zhat->zdev, zhat->zulu_ctx); - - mutex_exit(&zhat->lock); - - free_zblks(free_list); - - TNF_PROBE_1(zulu_hat_swapout_done, "zulu_hat", /* CSTYLED */, - tnf_int, nblks, nblks); -} - - -static void -zulu_hat_unshare(struct xhat *xhat, caddr_t vaddr, size_t size) -{ - TNF_PROBE_0(zulu_hat_unshare, "zulu_hat", /* CSTYLED */); - - zulu_hat_unload(xhat, vaddr, size, 0); -} - -/* - * Functions to manage changes in protections for mappings. - * - * These are rarely called in normal operation so for now just unload - * the region. - * If the mapping is still needed, it will fault in later with the new - * attrributes. - */ -typedef enum { - ZULU_HAT_CHGATTR, - ZULU_HAT_SETATTR, - ZULU_HAT_CLRATTR -} zulu_hat_prot_op; - -static void -zulu_hat_update_attr(struct xhat *xhat, caddr_t vaddr, size_t size, - uint_t flags, zulu_hat_prot_op op) -{ - struct zulu_hat *zhat = (struct zulu_hat *)xhat; - - TNF_PROBE_5(zulu_hat_changeprot, "zulu_hat", /* CSTYLED */, - tnf_int, ctx, zhat->zulu_ctx, - tnf_opaque, vaddr, vaddr, tnf_opaque, size, size, - tnf_uint, flags, flags, tnf_int, op, op); - - zulu_hat_unload(xhat, vaddr, size, 0); -} - -static void -zulu_hat_chgprot(struct xhat *xhat, caddr_t vaddr, size_t size, uint_t flags) -{ - struct zulu_hat *zhat = (struct zulu_hat *)xhat; -#ifdef DEBUG - printf("zulu_hat_chgprot: ctx: %d addr: %lx, size: %lx flags: %x\n", - zhat->zulu_ctx, (uint64_t)vaddr, size, flags); -#endif - zulu_hat_update_attr(xhat, vaddr, size, flags, ZULU_HAT_CHGATTR); -} - - -static void -zulu_hat_setattr(struct xhat *xhat, caddr_t vaddr, size_t size, uint_t flags) -{ - struct zulu_hat *zhat = (struct zulu_hat *)xhat; -#ifdef DEBUG - printf("zulu_hat_setattr: ctx: %d addr: %lx, size: %lx flags: %x\n", - zhat->zulu_ctx, (uint64_t)vaddr, size, flags); -#endif - zulu_hat_update_attr(xhat, vaddr, size, flags, ZULU_HAT_SETATTR); -} - -static void -zulu_hat_clrattr(struct xhat *xhat, caddr_t vaddr, size_t size, uint_t flags) -{ - struct zulu_hat *zhat = (struct zulu_hat *)xhat; -#ifdef DEBUG - printf("zulu_hat_clrattr: ctx: %d addr: %lx, size: %lx flags: %x\n", - zhat->zulu_ctx, (uint64_t)vaddr, size, flags); -#endif - zulu_hat_update_attr(xhat, vaddr, size, flags, ZULU_HAT_CLRATTR); -} - -static void -zulu_hat_chgattr(struct xhat *xhat, caddr_t vaddr, size_t size, uint_t flags) -{ - struct zulu_hat *zhat = (struct zulu_hat *)xhat; - TNF_PROBE_3(zulu_hat_chgattr, "zulu_hat", /* CSTYLED */, - tnf_int, ctx, zhat->zulu_ctx, - tnf_opaque, vaddr, vaddr, - tnf_opaque, flags, flags); -#ifdef DEBUG - printf("zulu_hat_chgattr: ctx: %d addr: %lx, size: %lx flags: %x\n", - zhat->zulu_ctx, (uint64_t)vaddr, size, flags); -#endif - zulu_hat_update_attr(xhat, vaddr, size, flags, ZULU_HAT_CHGATTR); -} - - -struct xhat_ops zulu_hat_ops = { - zulu_hat_alloc, /* xhat_alloc */ - zulu_hat_free, /* xhat_free */ - zulu_hat_free_start, /* xhat_free_start */ - NULL, /* xhat_free_end */ - NULL, /* xhat_dup */ - NULL, /* xhat_swapin */ - zulu_hat_swapout, /* xhat_swapout */ - zulu_hat_memload, /* xhat_memload */ - zulu_hat_memload_array, /* xhat_memload_array */ - zulu_hat_devload, /* xhat_devload */ - zulu_hat_unload, /* xhat_unload */ - zulu_hat_unload_callback, /* xhat_unload_callback */ - zulu_hat_setattr, /* xhat_setattr */ - zulu_hat_clrattr, /* xhat_clrattr */ - zulu_hat_chgattr, /* xhat_chgattr */ - zulu_hat_unshare, /* xhat_unshare */ - zulu_hat_chgprot, /* xhat_chgprot */ - zulu_hat_pageunload, /* xhat_pageunload */ -}; - -xblk_cache_t zulu_xblk_cache = { - NULL, - NULL, - NULL, - xhat_xblkcache_reclaim -}; - -xhat_provider_t zulu_hat_provider = { - XHAT_PROVIDER_VERSION, - 0, - NULL, - NULL, - "zulu_hat_provider", - &zulu_xblk_cache, - &zulu_hat_ops, - sizeof (struct zulu_hat_blk) + sizeof (struct xhat_hme_blk) -}; - -/* - * The following functions are the entry points that zuluvm uses. - */ - -/* - * initialize this module. Called from zuluvm's _init function - */ -int -zulu_hat_init() -{ - int c; - int rval; - mutex_init(&zulu_ctx_lock, NULL, MUTEX_DEFAULT, NULL); - - for (c = 0; c < ZULU_HAT_MAX_CTX; c++) { - ZULU_CTX_LOCK_INIT(c); - } - zulu_ctx_search_start = 0; - rval = xhat_provider_register(&zulu_hat_provider); - if (rval != 0) { - mutex_destroy(&zulu_ctx_lock); - } - return (rval); -} - -/* - * un-initialize this module. Called from zuluvm's _fini function - */ -int -zulu_hat_destroy() -{ - if (xhat_provider_unregister(&zulu_hat_provider) != 0) { - return (-1); - } - mutex_destroy(&zulu_ctx_lock); - return (0); -} - -int -zulu_hat_attach(void *arg) -{ - (void) arg; - return (0); -} - -int -zulu_hat_detach(void *arg) -{ - (void) arg; - return (0); -} - -/* - * create a zulu hat for this address space. - */ -struct zulu_hat * -zulu_hat_proc_attach(struct as *as, void *zdev) -{ - struct zulu_hat *zhat; - int xhat_rval; - - xhat_rval = xhat_attach_xhat(&zulu_hat_provider, as, - (struct xhat **)&zhat, NULL); - if ((xhat_rval == 0) && (zhat != NULL)) { - mutex_enter(&zhat->lock); - ZULU_HAT2AS(zhat) = as; - zhat->zdev = zdev; - mutex_exit(&zhat->lock); - } - - TNF_PROBE_3(zulu_hat_proc_attach, "zulu_hat", /* CSTYLED */, - tnf_int, xhat_rval, xhat_rval, tnf_opaque, as, as, - tnf_opaque, zhat, zhat); - - return (zhat); -} - -void -zulu_hat_proc_detach(struct zulu_hat *zhat) -{ - struct as *as = ZULU_HAT2AS(zhat); - - zulu_hat_ctx_free(zhat); - - (void) xhat_detach_xhat(&zulu_hat_provider, ZULU_HAT2AS(zhat)); - - TNF_PROBE_1(zulu_hat_proc_detach, "zulu_hat", /* CSTYLED */, - tnf_opaque, as, as); -} - -/* - * zulu_hat_terminate - * - * Disables any further TLB miss processing for this hat - * Called by zuluvm's as_free callback. The primary purpose of this - * function is to cause any pending zulu DMA to abort quickly. - */ -void -zulu_hat_terminate(struct zulu_hat *zhat) -{ - int ctx = zhat->zulu_ctx; - - TNF_PROBE_1(zulu_hat_terminate, "zulu_hat", /* CSTYLED */, - tnf_int, ctx, ctx); - - mutex_enter(&zhat->lock); - - zhat->freed = 1; - - zulu_ctx_tsb_lock_enter(zhat); - /* - * zap the tsb - */ - bzero(zhat->zulu_tsb, ZULU_TSB_SZ); - zulu_ctx_tsb_lock_exit(zhat); - - zulu_hat_demap_ctx(zhat->zdev, zhat->zulu_ctx); - - mutex_exit(&zhat->lock); - - TNF_PROBE_0(zulu_hat_terminate_done, "zulu_hat", /* CSTYLED */); -} diff --git a/usr/src/uts/sun4u/zuluvm/Makefile b/usr/src/uts/sun4u/zuluvm/Makefile deleted file mode 100644 index 719558aa46..0000000000 --- a/usr/src/uts/sun4u/zuluvm/Makefile +++ /dev/null @@ -1,133 +0,0 @@ -# -# CDDL HEADER START -# -# The contents of this file are subject to the terms of the -# Common Development and Distribution License (the "License"). -# You may not use this file except in compliance with the License. -# -# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE -# or http://www.opensolaris.org/os/licensing. -# See the License for the specific language governing permissions -# and limitations under the License. -# -# When distributing Covered Code, include this CDDL HEADER in each -# file and include the License file at usr/src/OPENSOLARIS.LICENSE. -# If applicable, add the following below this CDDL HEADER, with the -# fields enclosed by brackets "[]" replaced with your own identifying -# information: Portions Copyright [yyyy] [name of copyright owner] -# -# CDDL HEADER END -# -# -# Copyright 2006 Sun Microsystems, Inc. All rights reserved. -# Use is subject to license terms. -# -# -# This makefile drives the production of the zulunvm -# mics module -# -# sun4u implementation architecture dependent -# - -# -# Path to the base of the uts directory tree (usually /usr/src/uts). -# -UTSBASE = ../.. - -# -# Define the module and object file sets. -# -MODULE = zuluvm -OBJECTS = $(ZULUVM_OBJS:%=$(OBJS_DIR)/%) -LINTS = $(ZULUVM_OBJS:%.o=$(LINTS_DIR)/%.ln) -ROOTMODULE = $(ROOT_PSM_MISC_DIR)/$(MODULE) - -# -# Include common rules. -# -include $(UTSBASE)/sun4u/Makefile.sun4u - -# -# Override defaults to build a unique, local modstubs.o. -# -MODSTUBS_DIR = . -$(MODSTUBS_O) := AS_CPPFLAGS += -DZULU_MISC_MODULE - -CLEANFILES += $(MODSTUBS_O) - -# -# Define targets -# -ALL_TARGET = $(BINARY) -LINT_TARGET = $(MODULE).lint -INSTALL_TARGET = $(BINARY) $(ROOTMODULE) - -ZULUVM_OFFSETS = $(UTSBASE)/sun4u/zuluvm/zuluvm_offsets.in -ZULUVM_OFFSETS_H = $(OBJS_DIR)/zuluvm_offsets.h - -ZULUVM_STATS = -DZULUVM_STATS - -# -# We turn off tnf probes for opt builds. -# -PROBE_FLAGS_OBJ64 = -DNPROBE - -# -# lint pass one enforcement -# -CFLAGS += $(CCVERBOSE) $(ZULUVM_STATS) $(PROBE_FLAGS_$(BUILD_TYPE)) -ASFLAGS += $(ZULUVM_STATS) -LINTFLAGS += -I$(OBJS_DIR) - -# -# For now, disable these lint checks; maintainers should endeavor -# to investigate and remove these for maximum lint coverage. -# Please do not carry these forward to new Makefiles. -# -LINTTAGS += -erroff=E_BAD_PTR_CAST_ALIGN -LINTTAGS += -erroff=E_ASSIGN_NARROW_CONV - -CERRWARN += -_gcc=-Wno-uninitialized - -# -# Default build targets. -# -.KEEP_STATE: - -def: $(DEF_DEPS) - -all: $(ALL_DEPS) - -clean: $(CLEAN_DEPS) - -clobber: $(CLOBBER_DEPS) - -lint: $(LINT_DEPS) - -modlintlib: $(MODLINTLIB_DEPS) lint64 - -clean.lint: $(CLEAN_LINT_DEPS) - -install: $(INSTALL_DEPS) - -# -# Special rules for generating assym.h for inclusion in assembly files -# -#$(DSF_DIR)/$(OBJS_DIR)/assym.h: FRC -# @cd $(DSF_DIR); $(MAKE) all.targ -# -AS_INC_PATH += -I$(OBJS_DIR) - -ZULUVM_DEPS += zulu_hat_asm.o zulu_asm.o zulu_asm.ln zulu_hat_asm.ln - -CLEANFILES += $(ZULUVM_OFFSETS_H) $(ZULUVM_OFFSETS_OUT) - -$(ZULUVM_DEPS:%=$(OBJS_DIR)/%): $(ZULUVM_OFFSETS_H) - -$(ZULUVM_OFFSETS_H): $(ZULUVM_OFFSETS) - $(OFFSETS_CREATE) <$(ZULUVM_OFFSETS) >$@ - -# -# Include common targets. -# -include $(UTSBASE)/sun4u/Makefile.targ diff --git a/usr/src/uts/sun4u/zuluvm/zuluvm_offsets.in b/usr/src/uts/sun4u/zuluvm/zuluvm_offsets.in deleted file mode 100644 index 14470a9274..0000000000 --- a/usr/src/uts/sun4u/zuluvm/zuluvm_offsets.in +++ /dev/null @@ -1,77 +0,0 @@ -\ -\ Copyright 2005 Sun Microsystems, Inc. All rights reserved. -\ Use is subject to license terms. -\ -\ CDDL HEADER START -\ -\ The contents of this file are subject to the terms of the -\ Common Development and Distribution License, Version 1.0 only -\ (the "License"). You may not use this file except in compliance -\ with the License. -\ -\ You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE -\ or http://www.opensolaris.org/os/licensing. -\ See the License for the specific language governing permissions -\ and limitations under the License. -\ -\ When distributing Covered Code, include this CDDL HEADER in each -\ file and include the License file at usr/src/OPENSOLARIS.LICENSE. -\ If applicable, add the following below this CDDL HEADER, with the -\ fields enclosed by brackets "[]" replaced with your own identifying -\ information: Portions Copyright [yyyy] [name of copyright owner] -\ -\ CDDL HEADER END -\ - -#pragma ident "%Z%%M% %I% %E% SMI" - -#include <sys/types.h> -#include <sys/sunddi.h> -#include <sys/zulumod.h> -#include <sys/zulu_hat.h> - -zulu_hat - zulu_tsb ZULU_HAT_TSB - zulu_tsb_size ZULU_HAT_TSB_SZ - zulu_ctx ZULU_HAT_CTX - -#ifdef ZULUVM_STATS -zuluvm_miss_t - idata ZULUVM_IDATA - arg ZULUVM_ARG - mmu_pa ZULUVM_PAMMU - state ZULUVM_STATE - intr_num ZULUVM_INTRNUM - cancel ZULUVM_ST_TLBCANCEL - tlb_miss ZULUVM_ST_MISS - pagefault ZULUVM_ST_PGFAULT - no_mapping ZULUVM_ST_NOMAP - preload ZULUVM_ST_PRELOAD - migrate ZULUVM_ST_MIGRATE - pagesize ZULUVM_ST_PAGESIZE - itlb1miss ZULUVM_ST_ITLB1MISS - dtlb1miss ZULUVM_ST_DTLB1MISS - itlb2miss ZULUVM_ST_ITLB2MISS - dtlb2miss ZULUVM_ST_DTLB2MISS - demap_page ZULUVM_ST_DEMAP_PAGE - demap_ctx ZULUVM_ST_DEMAP_CTX - pfnbuf ZULUVM_PFNBUF - pfncnt ZULUVM_PFNCNT - -#else - -zuluvm_miss_t - idata ZULUVM_IDATA - arg ZULUVM_ARG - mmu_pa ZULUVM_PAMMU - state ZULUVM_STATE - intr_num ZULUVM_INTRNUM - pfnbuf ZULUVM_PFNBUF - pfncnt ZULUVM_PFNCNT -#endif - -\#define ZULUVM_OFFSET(a) (ZULUVM_IDATA + ((a) * ZULUVM_IDATA_INCR)) -\#define ZULUVM_ASM_TLB_TTE ZULUVM_OFFSET(ZULUVM_TLB_TTE_IDX) -\#define ZULUVM_ASM_TLB_ADDR ZULUVM_OFFSET(ZULUVM_TLB_ADDR_IDX) -\#define ZULUVM_ASM_TLB_TYPE ZULUVM_OFFSET(ZULUVM_TLB_TYPE_IDX) -\#define ZULUVM_ASM_TLB_ERRCODE ZULUVM_OFFSET(ZULUVM_TLB_ERRCODE_IDX) |