diff options
Diffstat (limited to 'usr/src')
50 files changed, 2234 insertions, 2016 deletions
diff --git a/usr/src/cmd/perl/contrib/Sun/Solaris/Kstat/Kstat.xs b/usr/src/cmd/perl/contrib/Sun/Solaris/Kstat/Kstat.xs index b2c9d4ba34..d4e71d41ff 100644 --- a/usr/src/cmd/perl/contrib/Sun/Solaris/Kstat/Kstat.xs +++ b/usr/src/cmd/perl/contrib/Sun/Solaris/Kstat/Kstat.xs @@ -486,9 +486,6 @@ save_sfmmu_global_stat(HV *self, kstat_t *kp, int strip_str) SAVE_INT32(self, sfmmugp, sf_khash_searches); SAVE_INT32(self, sfmmugp, sf_khash_links); SAVE_INT32(self, sfmmugp, sf_swapout); - SAVE_INT32(self, sfmmugp, sf_ctxfree); - SAVE_INT32(self, sfmmugp, sf_ctxdirty); - SAVE_INT32(self, sfmmugp, sf_ctxsteal); SAVE_INT32(self, sfmmugp, sf_tsb_alloc); SAVE_INT32(self, sfmmugp, sf_tsb_allocfail); SAVE_INT32(self, sfmmugp, sf_tsb_sectsb_create); @@ -531,10 +528,7 @@ save_sfmmu_global_stat(HV *self, kstat_t *kp, int strip_str) SAVE_INT32(self, sfmmugp, sf_tsb_resize_failures); SAVE_INT32(self, sfmmugp, sf_tsb_reloc); SAVE_INT32(self, sfmmugp, sf_user_vtop); - SAVE_INT32(self, sfmmugp, sf_ctx_swap); - SAVE_INT32(self, sfmmugp, sf_tlbflush_all); - SAVE_INT32(self, sfmmugp, sf_tlbflush_ctx); - SAVE_INT32(self, sfmmugp, sf_tlbflush_deferred); + SAVE_INT32(self, sfmmugp, sf_ctx_inv); SAVE_INT32(self, sfmmugp, sf_tlb_reprog_pgsz); } #endif diff --git a/usr/src/uts/common/os/fork.c b/usr/src/uts/common/os/fork.c index 8f2fbb13c1..e4056ef222 100644 --- a/usr/src/uts/common/os/fork.c +++ b/usr/src/uts/common/os/fork.c @@ -242,11 +242,12 @@ cfork(int isvfork, int isfork1) while (*orphpp != cp) orphpp = &(*orphpp)->p_nextorph; *orphpp = cp->p_nextorph; - ASSERT(p->p_child == cp); - p->p_child = cp->p_sibling; - if (p->p_child) { - p->p_child->p_psibling = NULL; - } + if (p->p_child == cp) + p->p_child = cp->p_sibling; + if (cp->p_sibling) + cp->p_sibling->p_psibling = cp->p_psibling; + if (cp->p_psibling) + cp->p_psibling->p_sibling = cp->p_sibling; mutex_enter(&cp->p_lock); tk = cp->p_task; task_detach(cp); @@ -594,11 +595,12 @@ forklwperr: while (*orphpp != cp) orphpp = &(*orphpp)->p_nextorph; *orphpp = cp->p_nextorph; - ASSERT(p->p_child == cp); - p->p_child = cp->p_sibling; - if (p->p_child) { - p->p_child->p_psibling = NULL; - } + if (p->p_child == cp) + p->p_child = cp->p_sibling; + if (cp->p_sibling) + cp->p_sibling->p_psibling = cp->p_psibling; + if (cp->p_psibling) + cp->p_psibling->p_sibling = cp->p_sibling; pid_exit(cp); mutex_exit(&pidlock); diff --git a/usr/src/uts/common/vm/seg_kmem.h b/usr/src/uts/common/vm/seg_kmem.h index f95215a410..739e168f78 100644 --- a/usr/src/uts/common/vm/seg_kmem.h +++ b/usr/src/uts/common/vm/seg_kmem.h @@ -2,9 +2,8 @@ * CDDL HEADER START * * The contents of this file are subject to the terms of the - * Common Development and Distribution License, Version 1.0 only - * (the "License"). You may not use this file except in compliance - * with the License. + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. @@ -20,7 +19,7 @@ * CDDL HEADER END */ /* - * Copyright 2005 Sun Microsystems, Inc. All rights reserved. + * Copyright 2006 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ @@ -58,7 +57,6 @@ extern vmem_t *hat_memload_arena; /* HAT translation arena */ extern struct seg kvseg32; /* 32-bit kernel heap segment */ extern vmem_t *heap32_arena; /* 32-bit kernel heap arena */ extern vmem_t *heaptext_arena; /* kernel text arena, from heap */ -extern struct ctx *kctx; /* kernel context */ extern struct as kas; /* kernel address space */ extern struct vnode kvp; /* vnode for all segkmem pages */ extern int segkmem_reloc; /* enable/disable segkmem relocatable pages */ diff --git a/usr/src/uts/sfmmu/ml/sfmmu_asm.s b/usr/src/uts/sfmmu/ml/sfmmu_asm.s index ae27cf92ef..6e26c226ee 100644 --- a/usr/src/uts/sfmmu/ml/sfmmu_asm.s +++ b/usr/src/uts/sfmmu/ml/sfmmu_asm.s @@ -482,6 +482,19 @@ label/**/2: /* * sfmmu related subroutines */ +uint_t +sfmmu_disable_intrs() +{ return(0); } + +/* ARGSUSED */ +void +sfmmu_enable_intrs(uint_t pstate_save) +{} + +/* ARGSUSED */ +void +sfmmu_alloc_ctx(sfmmu_t *sfmmup, int allocflag, struct cpu *cp) +{} /* * Use cas, if tte has changed underneath us then reread and try again. @@ -534,6 +547,280 @@ sfmmu_panic4: sfmmu_panic5: .asciz "sfmmu_asm: no unlocked TTEs in TLB 0" + .global sfmmu_panic6 +sfmmu_panic6: + .asciz "sfmmu_asm: interrupts not disabled" + + .global sfmmu_panic7 +sfmmu_panic7: + .asciz "sfmmu_asm: kernel as" + + .global sfmmu_panic8 +sfmmu_panic8: + .asciz "sfmmu_asm: gnum is zero" + + .global sfmmu_panic9 +sfmmu_panic9: + .asciz "sfmmu_asm: cnum is greater than MAX_SFMMU_CTX_VAL" + + ENTRY(sfmmu_disable_intrs) + rdpr %pstate, %o0 +#ifdef DEBUG + PANIC_IF_INTR_DISABLED_PSTR(%o0, sfmmu_di_l0, %g1) +#endif /* DEBUG */ + retl + wrpr %o0, PSTATE_IE, %pstate + SET_SIZE(sfmmu_disable_intrs) + + ENTRY(sfmmu_enable_intrs) + retl + wrpr %g0, %o0, %pstate + SET_SIZE(sfmmu_enable_intrs) + +/* + * This routine is called both by resume() and sfmmu_get_ctx() to + * allocate a new context for the process on a MMU. + * if allocflag == 1, then alloc ctx when HAT mmu cnum == INVALID . + * if allocflag == 0, then do not alloc ctx if HAT mmu cnum == INVALID, which + * is the case when sfmmu_alloc_ctx is called from resume(). + * + * The caller must disable interrupts before entering this routine. + * To reduce ctx switch overhead, the code contains both 'fast path' and + * 'slow path' code. The fast path code covers the common case where only + * a quick check is needed and the real ctx allocation is not required. + * It can be done without holding the per-process (PP) lock. + * The 'slow path' code must be protected by the PP Lock and performs ctx + * allocation. + * Hardware context register and HAT mmu cnum are updated accordingly. + * + * %o0 - sfmmup + * %o1 - allocflag + * %o2 - CPU + */ + ENTRY_NP(sfmmu_alloc_ctx) + +#ifdef DEBUG + sethi %hi(ksfmmup), %o3 + ldx [%o3 + %lo(ksfmmup)], %o3 + cmp %o3, %o0 + bne,pt %xcc, 0f + nop + + sethi %hi(panicstr), %g1 ! if kernel as, panic + ldx [%g1 + %lo(panicstr)], %g1 + tst %g1 + bnz,pn %icc, 7f + nop + + sethi %hi(sfmmu_panic7), %o0 + call panic + or %o0, %lo(sfmmu_panic7), %o0 + +7: + retl + nop + +0: + PANIC_IF_INTR_ENABLED_PSTR(sfmmu_ei_l1, %g1) +#endif /* DEBUG */ + + ! load global mmu_ctxp info + ldx [%o2 + CPU_MMU_CTXP], %o3 ! %o3 = mmu_ctx_t ptr + lduw [%o2 + CPU_MMU_IDX], %g2 ! %g2 = mmu index + + ! load global mmu_ctxp gnum + ldx [%o3 + MMU_CTX_GNUM], %o4 ! %o4 = mmu_ctxp->gnum + +#ifdef DEBUG + cmp %o4, %g0 ! mmu_ctxp->gnum should never be 0 + bne,pt %xcc, 3f + nop + + sethi %hi(panicstr), %g1 ! test if panicstr is already set + ldx [%g1 + %lo(panicstr)], %g1 + tst %g1 + bnz,pn %icc, 3f + nop + + sethi %hi(sfmmu_panic8), %o0 + call panic + or %o0, %lo(sfmmu_panic8), %o0 +3: +#endif + + ! load HAT sfmmu_ctxs[mmuid] gnum, cnum + + sllx %g2, SFMMU_MMU_CTX_SHIFT, %g2 + add %o0, %g2, %g2 ! %g2 = &sfmmu_ctxs[mmuid] - SFMMU_CTXS + + /* + * %g5 = sfmmu gnum returned + * %g6 = sfmmu cnum returned + * %g2 = &sfmmu_ctxs[mmuid] - SFMMU_CTXS + * %g4 = scratch + * + * Fast path code, do a quick check. + */ + SFMMU_MMUID_GNUM_CNUM(%g2, %g5, %g6, %g4) + + cmp %g6, INVALID_CONTEXT ! hat cnum == INVALID ?? + bne,pt %icc, 1f ! valid hat cnum, check gnum + nop + + ! cnum == INVALID, check allocflag + brz,pt %o1, 8f ! allocflag == 0, skip ctx allocation, bail + mov %g6, %o1 + + ! (invalid HAT cnum) && (allocflag == 1) + ba,pt %icc, 2f + nop +1: + ! valid HAT cnum, check gnum + cmp %g5, %o4 + be,a,pt %icc, 8f ! gnum unchanged, go to done + mov %g6, %o1 + +2: + /* + * Grab per process (PP) sfmmu_ctx_lock spinlock, + * followed by the 'slow path' code. + */ + ldstub [%o0 + SFMMU_CTX_LOCK], %g3 ! %g3 = per process (PP) lock +3: + brz %g3, 5f + nop +4: + brnz,a,pt %g3, 4b ! spin if lock is 1 + ldub [%o0 + SFMMU_CTX_LOCK], %g3 + ba %xcc, 3b ! retry the lock + ldstub [%o0 + SFMMU_CTX_LOCK], %g3 ! %g3 = PP lock + +5: + membar #LoadLoad + /* + * %g5 = sfmmu gnum returned + * %g6 = sfmmu cnum returned + * %g2 = &sfmmu_ctxs[mmuid] - SFMMU_CTXS + * %g4 = scratch + */ + SFMMU_MMUID_GNUM_CNUM(%g2, %g5, %g6, %g4) + + cmp %g6, INVALID_CONTEXT ! hat cnum == INVALID ?? + bne,pt %icc, 1f ! valid hat cnum, check gnum + nop + + ! cnum == INVALID, check allocflag + brz,pt %o1, 2f ! allocflag == 0, called from resume, set hw + mov %g6, %o1 + + ! (invalid HAT cnum) && (allocflag == 1) + ba,pt %icc, 6f + nop +1: + ! valid HAT cnum, check gnum + cmp %g5, %o4 + be,a,pt %icc, 2f ! gnum unchanged, go to done + mov %g6, %o1 + + ba,pt %icc, 6f + nop +2: + membar #LoadStore|#StoreStore + ba,pt %icc, 8f + clrb [%o0 + SFMMU_CTX_LOCK] +6: + /* + * We get here if we do not have a valid context, or + * the HAT gnum does not match global gnum. We hold + * sfmmu_ctx_lock spinlock. Allocate that context. + * + * %o3 = mmu_ctxp + */ + add %o3, MMU_CTX_CNUM, %g3 + ld [%o3 + MMU_CTX_NCTXS], %g4 + + /* + * %g2 = &sfmmu_ctx_t[mmuid] - SFMMU_CTXS; + * %g3 = mmu cnum address + * %g4 = mmu nctxs + * + * %o0 = sfmmup + * %o1 = mmu current cnum value (used as new cnum) + * %o4 = mmu gnum + * + * %o5 = scratch + */ + ld [%g3], %o1 +0: + cmp %o1, %g4 + bl,a,pt %icc, 1f + add %o1, 1, %o5 ! %o5 = mmu_ctxp->cnum + 1 + + /* + * cnum reachs max, update HAT with INVALID + */ + set INVALID_CONTEXT, %o1 + + /* + * update hat cnum to INVALID, sun4v sfmmu_load_mmustate checks + * hat cnum to determine if set the number of TSBs to 0. + */ + sllx %o4, SFMMU_MMU_GNUM_RSHIFT, %o4 + or %o4, %o1, %o4 + stx %o4, [%g2 + SFMMU_CTXS] + + membar #LoadStore|#StoreStore + ba,pt %icc, 8f + clrb [%o0 + SFMMU_CTX_LOCK] +1: + ! %g3 = addr of mmu_ctxp->cnum + ! %o5 = mmu_ctxp->cnum + 1 + cas [%g3], %o1, %o5 + cmp %o1, %o5 + bne,a,pn %xcc, 0b ! cas failed + ld [%g3], %o1 + +#ifdef DEBUG + set MAX_SFMMU_CTX_VAL, %o5 + cmp %o1, %o5 + ble,pt %icc, 2f + nop + + sethi %hi(sfmmu_panic9), %o0 + call panic + or %o0, %lo(sfmmu_panic9), %o0 +2: +#endif + ! update hat gnum and cnum + sllx %o4, SFMMU_MMU_GNUM_RSHIFT, %o4 + or %o4, %o1, %o4 + stx %o4, [%g2 + SFMMU_CTXS] + + membar #LoadStore|#StoreStore + clrb [%o0 + SFMMU_CTX_LOCK] + +8: + /* + * program the secondary context register + * + * %o1 = cnum + */ +#ifdef sun4u + ldub [%o0 + SFMMU_CEXT], %o2 + sll %o2, CTXREG_EXT_SHIFT, %o2 + or %o1, %o2, %o1 +#endif + + mov MMU_SCONTEXT, %o4 + sethi %hi(FLUSH_ADDR), %o5 + stxa %o1, [%o4]ASI_MMU_CTX ! set 2nd context reg. + flush %o5 + + retl + nop + + SET_SIZE(sfmmu_alloc_ctx) + ENTRY_NP(sfmmu_modifytte) ldx [%o2], %g3 /* current */ @@ -1062,21 +1349,7 @@ sfmmu_kpm_unload_tsb(caddr_t addr, int vpshift) */ rdpr %pstate, %o5 #ifdef DEBUG - andcc %o5, PSTATE_IE, %g0 /* if interrupts already */ - bnz,pt %icc, 1f /* disabled, panic */ - nop - - sethi %hi(panicstr), %g1 - ldx [%g1 + %lo(panicstr)], %g1 - tst %g1 - bnz,pt %icc, 1f - nop - - save %sp, -SA(MINFRAME), %sp - sethi %hi(sfmmu_panic1), %o0 - call panic - or %o0, %lo(sfmmu_panic1), %o0 -1: + PANIC_IF_INTR_DISABLED_PSTR(%o5, sfmmu_di_l2, %g1) #endif /* DEBUG */ wrpr %o5, PSTATE_IE, %pstate /* disable interrupts */ @@ -1117,21 +1390,7 @@ sfmmu_kpm_unload_tsb(caddr_t addr, int vpshift) */ rdpr %pstate, %o5 ! %o5 = saved pstate #ifdef DEBUG - andcc %o5, PSTATE_IE, %g0 ! if interrupts already - bnz,pt %icc, 1f ! disabled, panic - nop - - sethi %hi(panicstr), %g1 - ldx [%g1 + %lo(panicstr)], %g1 - tst %g1 - bnz,pt %icc, 1f - nop - - save %sp, -SA(MINFRAME), %sp - sethi %hi(sfmmu_panic1), %o0 - call panic - or %o0, %lo(sfmmu_panic1), %o0 -1: + PANIC_IF_INTR_DISABLED_PSTR(%o5, sfmmu_di_l3, %g1) #endif /* DEBUG */ wrpr %o5, PSTATE_IE, %pstate ! disable interrupts @@ -1367,20 +1626,7 @@ hblk_add_panic2: rdpr %pstate, %o5 #ifdef DEBUG - andcc %o5, PSTATE_IE, %g0 /* if interrupts already */ - bnz,pt %icc, 3f /* disabled, panic */ - nop - - sethi %hi(panicstr), %g1 - ldx [%g1 + %lo(panicstr)], %g1 - tst %g1 - bnz,pt %icc, 3f - nop - - sethi %hi(sfmmu_panic1), %o0 - call panic - or %o0, %lo(sfmmu_panic1), %o0 -3: + PANIC_IF_INTR_DISABLED_PSTR(%o5, sfmmu_di_l4, %g1) #endif /* DEBUG */ /* * disable interrupts, clear Address Mask to access 64 bit physaddr @@ -1846,11 +2092,6 @@ sfmmu_kpm_dtsb_miss_small(void) #else /* lint */ - -#if (CTX_SIZE != (1 << CTX_SZ_SHIFT)) -#error - size of context struct does not match with CTX_SZ_SHIFT -#endif - #if (IMAP_SEG != 0) #error - ism_map->ism_seg offset is not zero #endif @@ -3317,21 +3558,7 @@ sfmmu_vatopfn(caddr_t vaddr, sfmmu_t *sfmmup, tte_t *ttep) */ rdpr %pstate, %o3 #ifdef DEBUG - andcc %o3, PSTATE_IE, %g0 /* if interrupts already */ - bnz,pt %icc, 1f /* disabled, panic */ - nop - - sethi %hi(panicstr), %g1 - ldx [%g1 + %lo(panicstr)], %g1 - tst %g1 - bnz,pt %icc, 1f - nop - - save %sp, -SA(MINFRAME), %sp - sethi %hi(sfmmu_panic1), %o0 - call panic - or %o0, %lo(sfmmu_panic1), %o0 -1: + PANIC_IF_INTR_DISABLED_PSTR(%o3, sfmmu_di_l5, %g1) #endif /* * disable interrupts to protect the TSBMISS area diff --git a/usr/src/uts/sfmmu/ml/sfmmu_kdi.s b/usr/src/uts/sfmmu/ml/sfmmu_kdi.s index 53b5f9f938..9b110396b5 100644 --- a/usr/src/uts/sfmmu/ml/sfmmu_kdi.s +++ b/usr/src/uts/sfmmu/ml/sfmmu_kdi.s @@ -225,7 +225,7 @@ ttep_calc: /* idx in %g1 */ \ * uint64_t * kdi_vatotte(uintptr_t va, int cnum) * { - * sfmmu_t *sfmmup = ctxs[cnum].ctx_sfmmu; + * sfmmu_t *sfmmup = ksfmmup; * uint64_t hmebpa, hmetag, hmeblkpa; * int i; * @@ -265,7 +265,13 @@ kdi_trap_vatotte(void) #else /* - * Invocation in normal context as a VA-to-TTE translator. + * Invocation in normal context as a VA-to-TTE translator + * for kernel context only. This routine returns 0 on + * success and -1 on error. + * + * %o0 = VA, input register + * %o1 = KCONTEXT + * %o2 = ttep, output register */ ENTRY_NP(kdi_vatotte) mov %o0, %g1 /* VA in %g1 */ @@ -288,21 +294,21 @@ kdi_trap_vatotte(void) mov -1, %o0 SET_SIZE(kdi_vatotte) + /* + * %g1 = vaddr passed in, tte or 0 (error) when return + * %g2 = KCONTEXT + * %g7 = return address + */ ENTRY_NP(kdi_trap_vatotte) - set nctxs, %g3 - ld [%g3], %g3 - cmp %g2, %g3 - bge,a %xcc, 6f - clr %g1 - set ctxs, %g3 - ldx [%g3], %g3 - mulx %g2, CTX_SIZE, %g2 - add %g2, %g3, %g2 - add %g2, CTX_SFMMUP, %g2 - ldx [%g2], %g2 /* VA %g1, sfmmup %g2 */ + cmp %g2, KCONTEXT /* make sure called in kernel ctx */ + bne,a,pn %icc, 6f + clr %g1 + + sethi %hi(ksfmmup), %g2 + ldx [%g2 + %lo(ksfmmup)], %g2 - mov 1, %g3 /* VA %g1, sfmmup %g2, idx %g3 */ + mov 1, %g3 /* VA %g1, ksfmmup %g2, idx %g3 */ mov HBLK_RANGE_SHIFT, %g4 ba 3f nop diff --git a/usr/src/uts/sfmmu/vm/hat_sfmmu.c b/usr/src/uts/sfmmu/vm/hat_sfmmu.c index 4e507ef2a3..7a9a22b61c 100644 --- a/usr/src/uts/sfmmu/vm/hat_sfmmu.c +++ b/usr/src/uts/sfmmu/vm/hat_sfmmu.c @@ -38,6 +38,7 @@ */ #include <sys/types.h> +#include <sys/kstat.h> #include <vm/hat.h> #include <vm/hat_sfmmu.h> #include <vm/page.h> @@ -149,13 +150,7 @@ int disable_auto_large_pages = 0; * Private sfmmu data structures for hat management */ static struct kmem_cache *sfmmuid_cache; - -/* - * Private sfmmu data structures for ctx management - */ -static struct ctx *ctxhand; /* hand used while stealing ctxs */ -static struct ctx *ctxfree; /* head of free ctx list */ -static struct ctx *ctxdirty; /* head of dirty ctx list */ +static struct kmem_cache *mmuctxdom_cache; /* * Private sfmmu data structures for tsb management @@ -173,7 +168,6 @@ static struct kmem_cache *sfmmu8_cache; static struct kmem_cache *sfmmu1_cache; static struct kmem_cache *pa_hment_cache; -static kmutex_t ctx_list_lock; /* mutex for ctx free/dirty lists */ static kmutex_t ism_mlist_lock; /* mutex for ism mapping list */ /* * private data for ism @@ -315,8 +309,7 @@ static int sfmmu_vacconflict_array(caddr_t, page_t *, int *); static int tst_tnc(page_t *pp, pgcnt_t); static void conv_tnc(page_t *pp, int); -static struct ctx *sfmmu_get_ctx(sfmmu_t *); -static void sfmmu_free_ctx(sfmmu_t *, struct ctx *); +static void sfmmu_get_ctx(sfmmu_t *); static void sfmmu_free_sfmmu(sfmmu_t *); static void sfmmu_gettte(struct hat *, caddr_t, tte_t *); @@ -335,9 +328,7 @@ static void sfmmu_ismtlbcache_demap(caddr_t, sfmmu_t *, struct hme_blk *, pfn_t, int); static void sfmmu_tlb_demap(caddr_t, sfmmu_t *, struct hme_blk *, int, int); static void sfmmu_tlb_range_demap(demap_range_t *); -static void sfmmu_tlb_ctx_demap(sfmmu_t *); -static void sfmmu_tlb_all_demap(void); -static void sfmmu_tlb_swap_ctx(sfmmu_t *, struct ctx *); +static void sfmmu_invalidate_ctx(sfmmu_t *); static void sfmmu_sync_mmustate(sfmmu_t *); static void sfmmu_tsbinfo_setup_phys(struct tsb_info *, pfn_t); @@ -378,11 +369,6 @@ static void sfmmu_hblkcache_reclaim(void *); static void sfmmu_shadow_hcleanup(sfmmu_t *, struct hme_blk *, struct hmehash_bucket *); static void sfmmu_free_hblks(sfmmu_t *, caddr_t, caddr_t, int); - -static void sfmmu_reuse_ctx(struct ctx *, sfmmu_t *); -static void sfmmu_disallow_ctx_steal(sfmmu_t *); -static void sfmmu_allow_ctx_steal(sfmmu_t *); - static void sfmmu_rm_large_mappings(page_t *, int); static void hat_lock_init(void); @@ -410,12 +396,14 @@ static void sfmmu_kpm_pageunload(page_t *); static void sfmmu_kpm_vac_unload(page_t *, caddr_t); static void sfmmu_kpm_demap_large(caddr_t); static void sfmmu_kpm_demap_small(caddr_t); -static void sfmmu_kpm_demap_tlbs(caddr_t, int); +static void sfmmu_kpm_demap_tlbs(caddr_t); static void sfmmu_kpm_hme_unload(page_t *); static kpm_hlk_t *sfmmu_kpm_kpmp_enter(page_t *, pgcnt_t); static void sfmmu_kpm_kpmp_exit(kpm_hlk_t *kpmp); static void sfmmu_kpm_page_cache(page_t *, int, int); +static void sfmmu_ctx_wrap_around(mmu_ctx_t *); + /* kpm globals */ #ifdef DEBUG /* @@ -447,8 +435,13 @@ uint64_t uhme_hash_pa; /* PA of uhme_hash */ uint64_t khme_hash_pa; /* PA of khme_hash */ int uhmehash_num; /* # of buckets in user hash table */ int khmehash_num; /* # of buckets in kernel hash table */ -struct ctx *ctxs; /* used by <machine/mmu.c> */ -uint_t nctxs; /* total number of contexts */ + +uint_t max_mmu_ctxdoms = 0; /* max context domains in the system */ +mmu_ctx_t **mmu_ctxs_tbl; /* global array of context domains */ +uint64_t mmu_saved_gnum = 0; /* to init incoming MMUs' gnums */ + +#define DEFAULT_NUM_CTXS_PER_MMU 8192 +static uint_t nctxs = DEFAULT_NUM_CTXS_PER_MMU; int cache; /* describes system cache */ @@ -567,7 +560,6 @@ struct sfmmu_tsbsize_stat sfmmu_tsbsize_stat; * Global data */ sfmmu_t *ksfmmup; /* kernel's hat id */ -struct ctx *kctx; /* kernel's context */ #ifdef DEBUG static void chk_tte(tte_t *, tte_t *, tte_t *, struct hme_blk *); @@ -682,26 +674,15 @@ static pad_mutex_t sfmmu_page_lock[SPL_TABLE_SIZE]; */ #define MAX_CB_ADDR 32 -#ifdef DEBUG - -/* - * Debugging trace ring buffer for stolen and freed ctxs. The - * stolen_ctxs[] array is protected by the ctx_trace_mutex. - */ -struct ctx_trace stolen_ctxs[TRSIZE]; -struct ctx_trace *ctx_trace_first = &stolen_ctxs[0]; -struct ctx_trace *ctx_trace_last = &stolen_ctxs[TRSIZE-1]; -struct ctx_trace *ctx_trace_ptr = &stolen_ctxs[0]; -kmutex_t ctx_trace_mutex; -uint_t num_ctx_stolen = 0; - -int ism_debug = 0; - -#endif /* DEBUG */ - tte_t hw_tte; static ulong_t sfmmu_dmr_maxbit = DMR_MAXBIT; +static char *mmu_ctx_kstat_names[] = { + "mmu_ctx_tsb_exceptions", + "mmu_ctx_tsb_raise_exception", + "mmu_ctx_wrap_around", +}; + /* * kpm virtual address to physical address */ @@ -1003,9 +984,8 @@ hat_init_pagesizes() void hat_init(void) { - struct ctx *ctx; - struct ctx *cur_ctx = NULL; int i; + size_t size; hat_lock_init(); hat_kstat_init(); @@ -1030,31 +1010,64 @@ hat_init(void) uhmehash_num--; /* make sure counter starts from 0 */ /* - * Initialize ctx structures and list lock. - * We keep two lists of ctxs. The "free" list contains contexts - * ready to use. The "dirty" list contains contexts that are OK - * to use after flushing the TLBs of any stale mappings. + * Allocate context domain structures. + * + * A platform may choose to modify max_mmu_ctxdoms in + * set_platform_defaults(). If a platform does not define + * a set_platform_defaults() or does not choose to modify + * max_mmu_ctxdoms, it gets one MMU context domain for every CPU. + * + * For sun4v, there will be one global context domain, this is to + * avoid the ldom cpu substitution problem. + * + * For all platforms that have CPUs sharing MMUs, this + * value must be defined. */ - mutex_init(&ctx_list_lock, NULL, MUTEX_DEFAULT, NULL); - kctx = &ctxs[KCONTEXT]; - ctx = &ctxs[NUM_LOCKED_CTXS]; - ctxhand = ctxfree = ctx; /* head of free list */ - ctxdirty = NULL; - for (i = NUM_LOCKED_CTXS; i < nctxs; i++) { - cur_ctx = &ctxs[i]; - cur_ctx->ctx_flags = CTX_FREE_FLAG; - cur_ctx->ctx_free = &ctxs[i + 1]; + if (max_mmu_ctxdoms == 0) { +#ifndef sun4v + max_mmu_ctxdoms = max_ncpus; +#else /* sun4v */ + max_mmu_ctxdoms = 1; +#endif /* sun4v */ } - cur_ctx->ctx_free = NULL; /* tail of free list */ + + size = max_mmu_ctxdoms * sizeof (mmu_ctx_t *); + mmu_ctxs_tbl = kmem_zalloc(size, KM_SLEEP); + + /* mmu_ctx_t is 64 bytes aligned */ + mmuctxdom_cache = kmem_cache_create("mmuctxdom_cache", + sizeof (mmu_ctx_t), 64, NULL, NULL, NULL, NULL, NULL, 0); + /* + * MMU context domain initialization for the Boot CPU. + * This needs the context domains array allocated above. + */ + mutex_enter(&cpu_lock); + sfmmu_cpu_init(CPU); + mutex_exit(&cpu_lock); /* * Intialize ism mapping list lock. */ + mutex_init(&ism_mlist_lock, NULL, MUTEX_DEFAULT, NULL); - sfmmuid_cache = kmem_cache_create("sfmmuid_cache", sizeof (sfmmu_t), - 0, sfmmu_idcache_constructor, sfmmu_idcache_destructor, - NULL, NULL, NULL, 0); + /* + * Each sfmmu structure carries an array of MMU context info + * structures, one per context domain. The size of this array depends + * on the maximum number of context domains. So, the size of the + * sfmmu structure varies per platform. + * + * sfmmu is allocated from static arena, because trap + * handler at TL > 0 is not allowed to touch kernel relocatable + * memory. sfmmu's alignment is changed to 64 bytes from + * default 8 bytes, as the lower 6 bits will be used to pass + * pgcnt to vtag_flush_pgcnt_tl1. + */ + size = sizeof (sfmmu_t) + sizeof (sfmmu_ctx_t) * (max_mmu_ctxdoms - 1); + + sfmmuid_cache = kmem_cache_create("sfmmuid_cache", size, + 64, sfmmu_idcache_constructor, sfmmu_idcache_destructor, + NULL, NULL, static_arena, 0); sfmmu_tsbinfo_cache = kmem_cache_create("sfmmu_tsbinfo_cache", sizeof (struct tsb_info), 0, NULL, NULL, NULL, NULL, NULL, 0); @@ -1233,7 +1246,6 @@ static void hat_lock_init() { int i; - struct ctx *ctx; /* * initialize the array of mutexes protecting a page's mapping @@ -1256,14 +1268,6 @@ hat_lock_init() for (i = 0; i < SFMMU_NUM_LOCK; i++) mutex_init(HATLOCK_MUTEXP(&hat_lock[i]), NULL, MUTEX_DEFAULT, NULL); - -#ifdef DEBUG - mutex_init(&ctx_trace_mutex, NULL, MUTEX_DEFAULT, NULL); -#endif /* DEBUG */ - - for (ctx = ctxs, i = 0; i < nctxs; i++, ctx++) { - rw_init(&ctx->ctx_rwlock, NULL, RW_DEFAULT, NULL); - } } extern caddr_t kmem64_base, kmem64_end; @@ -1279,23 +1283,21 @@ struct hat * hat_alloc(struct as *as) { sfmmu_t *sfmmup; - struct ctx *ctx; int i; + uint64_t cnum; extern uint_t get_color_start(struct as *); ASSERT(AS_WRITE_HELD(as, &as->a_lock)); sfmmup = kmem_cache_alloc(sfmmuid_cache, KM_SLEEP); sfmmup->sfmmu_as = as; sfmmup->sfmmu_flags = 0; + LOCK_INIT_CLEAR(&sfmmup->sfmmu_ctx_lock); if (as == &kas) { - ctx = kctx; ksfmmup = sfmmup; - sfmmup->sfmmu_cnum = ctxtoctxnum(ctx); - ASSERT(sfmmup->sfmmu_cnum == KCONTEXT); sfmmup->sfmmu_cext = 0; - ctx->ctx_sfmmu = sfmmup; - ctx->ctx_flags = 0; + cnum = KCONTEXT; + sfmmup->sfmmu_clrstart = 0; sfmmup->sfmmu_tsb = NULL; /* @@ -1311,8 +1313,9 @@ hat_alloc(struct as *as) * we fault when we try to run and so have to get * another ctx. */ - sfmmup->sfmmu_cnum = INVALID_CONTEXT; sfmmup->sfmmu_cext = 0; + cnum = INVALID_CONTEXT; + /* initialize original physical page coloring bin */ sfmmup->sfmmu_clrstart = get_color_start(as); #ifdef DEBUG @@ -1331,6 +1334,13 @@ hat_alloc(struct as *as) sfmmup->sfmmu_flags = HAT_SWAPPED; ASSERT(sfmmup->sfmmu_tsb != NULL); } + + ASSERT(max_mmu_ctxdoms > 0); + for (i = 0; i < max_mmu_ctxdoms; i++) { + sfmmup->sfmmu_ctxs[i].cnum = cnum; + sfmmup->sfmmu_ctxs[i].gnum = 0; + } + sfmmu_setup_tsbinfo(sfmmup); for (i = 0; i < max_mmu_page_sizes; i++) { sfmmup->sfmmu_ttecnt[i] = 0; @@ -1355,6 +1365,164 @@ hat_alloc(struct as *as) } /* + * Create per-MMU context domain kstats for a given MMU ctx. + */ +static void +sfmmu_mmu_kstat_create(mmu_ctx_t *mmu_ctxp) +{ + mmu_ctx_stat_t stat; + kstat_t *mmu_kstat; + + ASSERT(MUTEX_HELD(&cpu_lock)); + ASSERT(mmu_ctxp->mmu_kstat == NULL); + + mmu_kstat = kstat_create("unix", mmu_ctxp->mmu_idx, "mmu_ctx", + "hat", KSTAT_TYPE_NAMED, MMU_CTX_NUM_STATS, KSTAT_FLAG_VIRTUAL); + + if (mmu_kstat == NULL) { + cmn_err(CE_WARN, "kstat_create for MMU %d failed", + mmu_ctxp->mmu_idx); + } else { + mmu_kstat->ks_data = mmu_ctxp->mmu_kstat_data; + for (stat = 0; stat < MMU_CTX_NUM_STATS; stat++) + kstat_named_init(&mmu_ctxp->mmu_kstat_data[stat], + mmu_ctx_kstat_names[stat], KSTAT_DATA_INT64); + mmu_ctxp->mmu_kstat = mmu_kstat; + kstat_install(mmu_kstat); + } +} + +/* + * plat_cpuid_to_mmu_ctx_info() is a platform interface that returns MMU + * context domain information for a given CPU. If a platform does not + * specify that interface, then the function below is used instead to return + * default information. The defaults are as follows: + * + * - For sun4u systems there's one MMU context domain per CPU. + * This default is used by all sun4u systems except OPL. OPL systems + * provide platform specific interface to map CPU ids to MMU ids + * because on OPL more than 1 CPU shares a single MMU. + * Note that on sun4v, there is one global context domain for + * the entire system. This is to avoid running into potential problem + * with ldom physical cpu substitution feature. + * - The number of MMU context IDs supported on any CPU in the + * system is 8K. + */ +/*ARGSUSED*/ +static void +sfmmu_cpuid_to_mmu_ctx_info(processorid_t cpuid, mmu_ctx_info_t *infop) +{ + infop->mmu_nctxs = nctxs; +#ifndef sun4v + infop->mmu_idx = cpu[cpuid]->cpu_seqid; +#else /* sun4v */ + infop->mmu_idx = 0; +#endif /* sun4v */ +} + +/* + * Called during CPU initialization to set the MMU context-related information + * for a CPU. + * + * cpu_lock serializes accesses to mmu_ctxs and mmu_saved_gnum. + */ +void +sfmmu_cpu_init(cpu_t *cp) +{ + mmu_ctx_info_t info; + mmu_ctx_t *mmu_ctxp; + + ASSERT(MUTEX_HELD(&cpu_lock)); + + if (&plat_cpuid_to_mmu_ctx_info == NULL) + sfmmu_cpuid_to_mmu_ctx_info(cp->cpu_id, &info); + else + plat_cpuid_to_mmu_ctx_info(cp->cpu_id, &info); + + ASSERT(info.mmu_idx < max_mmu_ctxdoms); + + if ((mmu_ctxp = mmu_ctxs_tbl[info.mmu_idx]) == NULL) { + /* Each mmu_ctx is cacheline aligned. */ + mmu_ctxp = kmem_cache_alloc(mmuctxdom_cache, KM_SLEEP); + bzero(mmu_ctxp, sizeof (mmu_ctx_t)); + + mutex_init(&mmu_ctxp->mmu_lock, NULL, MUTEX_SPIN, + (void *)ipltospl(DISP_LEVEL)); + mmu_ctxp->mmu_idx = info.mmu_idx; + mmu_ctxp->mmu_nctxs = info.mmu_nctxs; + /* + * Globally for lifetime of a system, + * gnum must always increase. + * mmu_saved_gnum is protected by the cpu_lock. + */ + mmu_ctxp->mmu_gnum = mmu_saved_gnum + 1; + mmu_ctxp->mmu_cnum = NUM_LOCKED_CTXS; + + sfmmu_mmu_kstat_create(mmu_ctxp); + + mmu_ctxs_tbl[info.mmu_idx] = mmu_ctxp; + } else { + ASSERT(mmu_ctxp->mmu_idx == info.mmu_idx); + } + + /* + * The mmu_lock is acquired here to prevent races with + * the wrap-around code. + */ + mutex_enter(&mmu_ctxp->mmu_lock); + + + mmu_ctxp->mmu_ncpus++; + CPUSET_ADD(mmu_ctxp->mmu_cpuset, cp->cpu_id); + CPU_MMU_IDX(cp) = info.mmu_idx; + CPU_MMU_CTXP(cp) = mmu_ctxp; + + mutex_exit(&mmu_ctxp->mmu_lock); +} + +/* + * Called to perform MMU context-related cleanup for a CPU. + */ +void +sfmmu_cpu_cleanup(cpu_t *cp) +{ + mmu_ctx_t *mmu_ctxp; + + ASSERT(MUTEX_HELD(&cpu_lock)); + + mmu_ctxp = CPU_MMU_CTXP(cp); + ASSERT(mmu_ctxp != NULL); + + /* + * The mmu_lock is acquired here to prevent races with + * the wrap-around code. + */ + mutex_enter(&mmu_ctxp->mmu_lock); + + CPU_MMU_CTXP(cp) = NULL; + + CPUSET_DEL(mmu_ctxp->mmu_cpuset, cp->cpu_id); + if (--mmu_ctxp->mmu_ncpus == 0) { + mmu_ctxs_tbl[mmu_ctxp->mmu_idx] = NULL; + mutex_exit(&mmu_ctxp->mmu_lock); + mutex_destroy(&mmu_ctxp->mmu_lock); + + if (mmu_ctxp->mmu_kstat) + kstat_delete(mmu_ctxp->mmu_kstat); + + /* mmu_saved_gnum is protected by the cpu_lock. */ + if (mmu_saved_gnum < mmu_ctxp->mmu_gnum) + mmu_saved_gnum = mmu_ctxp->mmu_gnum; + + kmem_cache_free(mmuctxdom_cache, mmu_ctxp); + + return; + } + + mutex_exit(&mmu_ctxp->mmu_lock); +} + +/* * Hat_setup, makes an address space context the current active one. * In sfmmu this translates to setting the secondary context with the * corresponding context. @@ -1362,8 +1530,6 @@ hat_alloc(struct as *as) void hat_setup(struct hat *sfmmup, int allocflag) { - struct ctx *ctx; - uint_t ctx_num; hatlock_t *hatlockp; /* Init needs some special treatment. */ @@ -1383,24 +1549,8 @@ hat_setup(struct hat *sfmmup, int allocflag) */ sfmmu_tsb_swapin(sfmmup, hatlockp); - sfmmu_disallow_ctx_steal(sfmmup); - - kpreempt_disable(); - - ctx = sfmmutoctx(sfmmup); - CPUSET_ADD(sfmmup->sfmmu_cpusran, CPU->cpu_id); - ctx_num = ctxtoctxnum(ctx); - ASSERT(sfmmup == ctx->ctx_sfmmu); - ASSERT(ctx_num >= NUM_LOCKED_CTXS); - sfmmu_setctx_sec(ctx_num); - sfmmu_load_mmustate(sfmmup); - - kpreempt_enable(); + sfmmu_get_ctx(sfmmup); - /* - * Allow ctx to be stolen. - */ - sfmmu_allow_ctx_steal(sfmmup); sfmmu_hat_exit(hatlockp); } else { ASSERT(allocflag == HAT_ALLOC); @@ -1409,6 +1559,12 @@ hat_setup(struct hat *sfmmup, int allocflag) kpreempt_disable(); CPUSET_ADD(sfmmup->sfmmu_cpusran, CPU->cpu_id); + + /* + * sfmmu_setctx_sec takes <pgsz|cnum> as a parameter, + * pagesize bits don't matter in this case since we are passing + * INVALID_CONTEXT to it. + */ sfmmu_setctx_sec(INVALID_CONTEXT); sfmmu_clear_utsbinfo(); @@ -1455,13 +1611,7 @@ hat_free_end(struct hat *sfmmup) if (sfmmup->sfmmu_rmstat) { hat_freestat(sfmmup->sfmmu_as, NULL); } - if (!delay_tlb_flush) { - sfmmu_tlb_ctx_demap(sfmmup); - xt_sync(sfmmup->sfmmu_cpusran); - } else { - SFMMU_STAT(sf_tlbflush_deferred); - } - sfmmu_free_ctx(sfmmup, sfmmutoctx(sfmmup)); + while (sfmmup->sfmmu_tsb != NULL) { struct tsb_info *next = sfmmup->sfmmu_tsb->tsb_next; sfmmu_tsbinfo_free(sfmmup->sfmmu_tsb); @@ -1495,8 +1645,6 @@ hat_swapout(struct hat *sfmmup) struct hme_blk *hmeblkp; struct hme_blk *pr_hblk = NULL; struct hme_blk *nx_hblk; - struct ctx *ctx; - int cnum; int i; uint64_t hblkpa, prevpa, nx_pa; struct hme_blk *list = NULL; @@ -1566,24 +1714,8 @@ hat_swapout(struct hat *sfmmup) * Now free up the ctx so that others can reuse it. */ hatlockp = sfmmu_hat_enter(sfmmup); - ctx = sfmmutoctx(sfmmup); - cnum = ctxtoctxnum(ctx); - if (cnum != INVALID_CONTEXT) { - rw_enter(&ctx->ctx_rwlock, RW_WRITER); - if (sfmmup->sfmmu_cnum == cnum) { - sfmmu_reuse_ctx(ctx, sfmmup); - /* - * Put ctx back to the free list. - */ - mutex_enter(&ctx_list_lock); - CTX_SET_FLAGS(ctx, CTX_FREE_FLAG); - ctx->ctx_free = ctxfree; - ctxfree = ctx; - mutex_exit(&ctx_list_lock); - } - rw_exit(&ctx->ctx_rwlock); - } + sfmmu_invalidate_ctx(sfmmup); /* * Free TSBs, but not tsbinfos, and set SWAPPED flag. @@ -4658,9 +4790,8 @@ hat_unload_large_virtual( struct hme_blk *list = NULL; int i; uint64_t hblkpa, prevpa, nx_pa; - hatlock_t *hatlockp; - struct tsb_info *tsbinfop; - struct ctx *ctx; + demap_range_t dmr, *dmrp; + cpuset_t cpuset; caddr_t endaddr = startaddr + len; caddr_t sa; caddr_t ea; @@ -4668,34 +4799,12 @@ hat_unload_large_virtual( caddr_t cb_ea[MAX_CB_ADDR]; int addr_cnt = 0; int a = 0; - int cnum; - - hatlockp = sfmmu_hat_enter(sfmmup); - - /* - * Since we know we're unmapping a huge range of addresses, - * just throw away the context and switch to another. It's - * cheaper than trying to unmap all of the TTEs we may find - * from the TLB individually, which is too expensive in terms - * of xcalls. Better yet, if we're exiting, no need to flush - * anything at all! - */ - if (!sfmmup->sfmmu_free) { - ctx = sfmmutoctx(sfmmup); - rw_enter(&ctx->ctx_rwlock, RW_WRITER); - cnum = sfmmutoctxnum(sfmmup); - if (cnum != INVALID_CONTEXT) { - sfmmu_tlb_swap_ctx(sfmmup, ctx); - } - rw_exit(&ctx->ctx_rwlock); - for (tsbinfop = sfmmup->sfmmu_tsb; tsbinfop != NULL; - tsbinfop = tsbinfop->tsb_next) { - if (tsbinfop->tsb_flags & TSB_SWAPPED) - continue; - sfmmu_inv_tsb(tsbinfop->tsb_va, - TSB_BYTES(tsbinfop->tsb_szc)); - } + if (sfmmup->sfmmu_free) { + dmrp = NULL; + } else { + dmrp = &dmr; + DEMAP_RANGE_INIT(sfmmup, dmrp); } /* @@ -4731,7 +4840,7 @@ hat_unload_large_virtual( if (hmeblkp->hblk_vcnt != 0 || hmeblkp->hblk_hmecnt != 0) (void) sfmmu_hblk_unload(sfmmup, hmeblkp, - sa, ea, NULL, flags); + sa, ea, dmrp, flags); /* * on unmap we also release the HME block itself, once @@ -4765,6 +4874,12 @@ hat_unload_large_virtual( cb_sa[addr_cnt] = sa; cb_ea[addr_cnt] = ea; if (++addr_cnt == MAX_CB_ADDR) { + if (dmrp != NULL) { + DEMAP_RANGE_FLUSH(dmrp); + cpuset = sfmmup->sfmmu_cpusran; + xt_sync(cpuset); + } + for (a = 0; a < MAX_CB_ADDR; ++a) { callback->hcb_start_addr = cb_sa[a]; callback->hcb_end_addr = cb_ea[a]; @@ -4781,6 +4896,11 @@ next_block: } sfmmu_hblks_list_purge(&list); + if (dmrp != NULL) { + DEMAP_RANGE_FLUSH(dmrp); + cpuset = sfmmup->sfmmu_cpusran; + xt_sync(cpuset); + } for (a = 0; a < addr_cnt; ++a) { callback->hcb_start_addr = cb_sa[a]; @@ -4788,8 +4908,6 @@ next_block: callback->hcb_function(callback); } - sfmmu_hat_exit(hatlockp); - /* * Check TSB and TLB page sizes if the process isn't exiting. */ @@ -4797,7 +4915,6 @@ next_block: sfmmu_check_page_sizes(sfmmup, 0); } - /* * Unload all the mappings in the range [addr..addr+len). addr and len must * be MMU_PAGESIZE aligned. @@ -5180,7 +5297,7 @@ sfmmu_hblk_unload(struct hat *sfmmup, struct hme_blk *hmeblkp, caddr_t addr, ttesz = get_hblk_ttesz(hmeblkp); use_demap_range = (do_virtual_coloring && - TTEBYTES(ttesz) == DEMAP_RANGE_PGSZ(dmrp)); + ((dmrp == NULL) || TTEBYTES(ttesz) == DEMAP_RANGE_PGSZ(dmrp))); if (use_demap_range) { DEMAP_RANGE_CONTINUE(dmrp, addr, endaddr); } else { @@ -5894,18 +6011,19 @@ again: CPUSET_DEL(cpuset, CPU->cpu_id); /* LINTED: constant in conditional context */ - SFMMU_XCALL_STATS(KCONTEXT); + SFMMU_XCALL_STATS(ksfmmup); /* * Flush TLB entry on remote CPU's */ - xt_some(cpuset, vtag_flushpage_tl1, (uint64_t)addr, KCONTEXT); + xt_some(cpuset, vtag_flushpage_tl1, (uint64_t)addr, + (uint64_t)ksfmmup); xt_sync(cpuset); /* * Flush TLB entry on local CPU */ - vtag_flushpage(addr, KCONTEXT); + vtag_flushpage(addr, (uint64_t)ksfmmup); } while (index != 0) { @@ -7710,8 +7828,7 @@ hat_unshare(struct hat *sfmmup, caddr_t addr, size_t len, uint_t ismszc) ism_ment_t *free_ment = NULL; ism_blk_t *ism_blkp; struct hat *ism_hatid; - struct ctx *ctx; - int cnum, found, i; + int found, i; hatlock_t *hatlockp; struct tsb_info *tsbinfo; uint_t ismshift = page_get_shift(ismszc); @@ -7777,7 +7894,6 @@ hat_unshare(struct hat *sfmmup, caddr_t addr, size_t len, uint_t ismszc) ism_hatid = ism_map[i].imap_ismhat; ASSERT(ism_hatid != NULL); ASSERT(ism_hatid->sfmmu_ismhat == 1); - ASSERT(ism_hatid->sfmmu_cnum == INVALID_CONTEXT); /* * First remove ourselves from the ism mapping list. @@ -7793,14 +7909,9 @@ hat_unshare(struct hat *sfmmup, caddr_t addr, size_t len, uint_t ismszc) * will go to tl=0. */ hatlockp = sfmmu_hat_enter(sfmmup); - ctx = sfmmutoctx(sfmmup); - rw_enter(&ctx->ctx_rwlock, RW_WRITER); - cnum = sfmmutoctxnum(sfmmup); - if (cnum != INVALID_CONTEXT) { - sfmmu_tlb_swap_ctx(sfmmup, ctx); - } - rw_exit(&ctx->ctx_rwlock); + sfmmu_invalidate_ctx(sfmmup); + sfmmu_hat_exit(hatlockp); /* @@ -8555,291 +8666,195 @@ sfmmu_page_cache(page_t *pp, int flags, int cache_flush_flag, int bcolor) } } + /* - * This routine gets called when the system has run out of free contexts. - * This will simply choose context passed to it to be stolen and reused. + * Wrapper routine used to return a context. + * + * It's the responsibility of the caller to guarantee that the + * process serializes on calls here by taking the HAT lock for + * the hat. + * */ -/* ARGSUSED */ static void -sfmmu_reuse_ctx(struct ctx *ctx, sfmmu_t *sfmmup) +sfmmu_get_ctx(sfmmu_t *sfmmup) { - sfmmu_t *stolen_sfmmup; - cpuset_t cpuset; - ushort_t cnum = ctxtoctxnum(ctx); + mmu_ctx_t *mmu_ctxp; + uint_t pstate_save; - ASSERT(cnum != KCONTEXT); - ASSERT(rw_read_locked(&ctx->ctx_rwlock) == 0); /* write locked */ - - /* - * simply steal and reuse the ctx passed to us. - */ - stolen_sfmmup = ctx->ctx_sfmmu; ASSERT(sfmmu_hat_lock_held(sfmmup)); - ASSERT(stolen_sfmmup->sfmmu_cnum == cnum); - ASSERT(stolen_sfmmup != ksfmmup); - - TRACE_CTXS(&ctx_trace_mutex, ctx_trace_ptr, cnum, stolen_sfmmup, - sfmmup, CTX_TRC_STEAL); - SFMMU_STAT(sf_ctxsteal); + ASSERT(sfmmup != ksfmmup); - /* - * Update sfmmu and ctx structs. After this point all threads - * belonging to this hat/proc will fault and not use the ctx - * being stolen. - */ kpreempt_disable(); - /* - * Enforce reverse order of assignments from sfmmu_get_ctx(). This - * is done to prevent a race where a thread faults with the context - * but the TSB has changed. - */ - stolen_sfmmup->sfmmu_cnum = INVALID_CONTEXT; - membar_enter(); - ctx->ctx_sfmmu = NULL; + + mmu_ctxp = CPU_MMU_CTXP(CPU); + ASSERT(mmu_ctxp); + ASSERT(mmu_ctxp->mmu_idx < max_mmu_ctxdoms); + ASSERT(mmu_ctxp == mmu_ctxs_tbl[mmu_ctxp->mmu_idx]); /* - * 1. flush TLB in all CPUs that ran the process whose ctx - * we are stealing. - * 2. change context for all other CPUs to INVALID_CONTEXT, - * if they are running in the context that we are going to steal. + * Do a wrap-around if cnum reaches the max # cnum supported by a MMU. */ - cpuset = stolen_sfmmup->sfmmu_cpusran; - CPUSET_DEL(cpuset, CPU->cpu_id); - CPUSET_AND(cpuset, cpu_ready_set); - SFMMU_XCALL_STATS(cnum); - xt_some(cpuset, sfmmu_ctx_steal_tl1, cnum, INVALID_CONTEXT); - xt_sync(cpuset); + if (mmu_ctxp->mmu_cnum == mmu_ctxp->mmu_nctxs) + sfmmu_ctx_wrap_around(mmu_ctxp); /* - * flush TLB of local processor + * Let the MMU set up the page sizes to use for + * this context in the TLB. Don't program 2nd dtlb for ism hat. */ - vtag_flushctx(cnum); + if ((&mmu_set_ctx_page_sizes) && (sfmmup->sfmmu_ismhat == 0)) { + mmu_set_ctx_page_sizes(sfmmup); + } /* - * If we just stole the ctx from the current process - * on local cpu then we also invalidate his context - * here. + * sfmmu_alloc_ctx and sfmmu_load_mmustate will be performed with + * interrupts disabled to prevent race condition with wrap-around + * ctx invalidatation. In sun4v, ctx invalidation also involves + * a HV call to set the number of TSBs to 0. If interrupts are not + * disabled until after sfmmu_load_mmustate is complete TSBs may + * become assigned to INVALID_CONTEXT. This is not allowed. */ - if (sfmmu_getctx_sec() == cnum) { - sfmmu_setctx_sec(INVALID_CONTEXT); - sfmmu_clear_utsbinfo(); - } + pstate_save = sfmmu_disable_intrs(); + + sfmmu_alloc_ctx(sfmmup, 1, CPU); + sfmmu_load_mmustate(sfmmup); + + sfmmu_enable_intrs(pstate_save); kpreempt_enable(); - SFMMU_STAT(sf_tlbflush_ctx); } /* - * Returns a context with the reader lock held. - * - * We maintain 2 different list of contexts. The first list - * is the free list and it is headed by ctxfree. These contexts - * are ready to use. The second list is the dirty list and is - * headed by ctxdirty. These contexts have been freed but haven't - * been flushed from the TLB. - * - * It's the responsibility of the caller to guarantee that the - * process serializes on calls here by taking the HAT lock for - * the hat. - * - * Changing the page size is a rather complicated process, so - * rather than jump through lots of hoops to special case it, - * the easiest way to go about it is to tell the MMU we want - * to change page sizes and then switch to using a different - * context. When we program the context registers for the - * process, we can take care of setting up the (new) page size - * for that context at that point. + * When all cnums are used up in a MMU, cnum will wrap around to the + * next generation and start from 2. */ - -static struct ctx * -sfmmu_get_ctx(sfmmu_t *sfmmup) +static void +sfmmu_ctx_wrap_around(mmu_ctx_t *mmu_ctxp) { - struct ctx *ctx; - ushort_t cnum; - struct ctx *lastctx = &ctxs[nctxs-1]; - struct ctx *firstctx = &ctxs[NUM_LOCKED_CTXS]; - uint_t found_stealable_ctx; - uint_t retry_count = 0; - -#define NEXT_CTX(ctx) (((ctx) >= lastctx) ? firstctx : ((ctx) + 1)) -retry: + /* caller must have disabled the preemption */ + ASSERT(curthread->t_preempt >= 1); + ASSERT(mmu_ctxp != NULL); - ASSERT(sfmmup->sfmmu_cnum != KCONTEXT); - /* - * Check to see if this process has already got a ctx. - * In that case just set the sec-ctx, grab a readers lock, and - * return. - * - * We have to double check after we get the readers lock on the - * context, since it could be stolen in this short window. - */ - if (sfmmup->sfmmu_cnum >= NUM_LOCKED_CTXS) { - ctx = sfmmutoctx(sfmmup); - rw_enter(&ctx->ctx_rwlock, RW_READER); - if (ctx->ctx_sfmmu == sfmmup) { - return (ctx); - } else { - rw_exit(&ctx->ctx_rwlock); - } - } - - found_stealable_ctx = 0; - mutex_enter(&ctx_list_lock); - if ((ctx = ctxfree) != NULL) { - /* - * Found a ctx in free list. Delete it from the list and - * use it. There's a short window where the stealer can - * look at the context before we grab the lock on the - * context, so we have to handle that with the free flag. - */ - SFMMU_STAT(sf_ctxfree); - ctxfree = ctx->ctx_free; - ctx->ctx_sfmmu = NULL; - mutex_exit(&ctx_list_lock); - rw_enter(&ctx->ctx_rwlock, RW_WRITER); - ASSERT(ctx->ctx_sfmmu == NULL); - ASSERT((ctx->ctx_flags & CTX_FREE_FLAG) != 0); - } else if ((ctx = ctxdirty) != NULL) { - /* - * No free contexts. If we have at least one dirty ctx - * then flush the TLBs on all cpus if necessary and move - * the dirty list to the free list. - */ - SFMMU_STAT(sf_ctxdirty); - ctxdirty = NULL; - if (delay_tlb_flush) - sfmmu_tlb_all_demap(); - ctxfree = ctx->ctx_free; - ctx->ctx_sfmmu = NULL; - mutex_exit(&ctx_list_lock); - rw_enter(&ctx->ctx_rwlock, RW_WRITER); - ASSERT(ctx->ctx_sfmmu == NULL); - ASSERT((ctx->ctx_flags & CTX_FREE_FLAG) != 0); - } else { - /* - * No free context available, so steal one. - * - * The policy to choose the appropriate context is simple; - * just sweep all the ctxs using ctxhand. This will steal - * the LRU ctx. - * - * We however only steal a non-free context that can be - * write locked. Keep searching till we find a stealable - * ctx. - */ - mutex_exit(&ctx_list_lock); - ctx = ctxhand; - do { - /* - * If you get the writers lock, and the ctx isn't - * a free ctx, THEN you can steal this ctx. - */ - if ((ctx->ctx_flags & CTX_FREE_FLAG) == 0 && - rw_tryenter(&ctx->ctx_rwlock, RW_WRITER) != 0) { - if (ctx->ctx_flags & CTX_FREE_FLAG) { - /* let the first guy have it */ - rw_exit(&ctx->ctx_rwlock); - } else { - found_stealable_ctx = 1; - break; - } - } - ctx = NEXT_CTX(ctx); - } while (ctx != ctxhand); + /* acquire Per-MMU (PM) spin lock */ + mutex_enter(&mmu_ctxp->mmu_lock); - if (found_stealable_ctx) { - /* - * Try and reuse the ctx. - */ - sfmmu_reuse_ctx(ctx, sfmmup); + /* re-check to see if wrap-around is needed */ + if (mmu_ctxp->mmu_cnum < mmu_ctxp->mmu_nctxs) + goto done; - } else if (retry_count++ < GET_CTX_RETRY_CNT) { - goto retry; + SFMMU_MMU_STAT(mmu_wrap_around); - } else { - panic("Can't find any stealable context"); - } + /* update gnum */ + ASSERT(mmu_ctxp->mmu_gnum != 0); + mmu_ctxp->mmu_gnum++; + if (mmu_ctxp->mmu_gnum == 0 || + mmu_ctxp->mmu_gnum > MAX_SFMMU_GNUM_VAL) { + cmn_err(CE_PANIC, "mmu_gnum of mmu_ctx 0x%p is out of bound.", + (void *)mmu_ctxp); } - ASSERT(rw_read_locked(&ctx->ctx_rwlock) == 0); /* write locked */ - ctx->ctx_sfmmu = sfmmup; + if (mmu_ctxp->mmu_ncpus > 1) { + cpuset_t cpuset; - /* - * Clear the ctx_flags field. - */ - ctx->ctx_flags = 0; + membar_enter(); /* make sure updated gnum visible */ - cnum = ctxtoctxnum(ctx); - membar_exit(); - sfmmup->sfmmu_cnum = cnum; + SFMMU_XCALL_STATS(NULL); - /* - * Let the MMU set up the page sizes to use for - * this context in the TLB. Don't program 2nd dtlb for ism hat. - */ - if ((&mmu_set_ctx_page_sizes) && (sfmmup->sfmmu_ismhat == 0)) - mmu_set_ctx_page_sizes(sfmmup); + /* xcall to others on the same MMU to invalidate ctx */ + cpuset = mmu_ctxp->mmu_cpuset; + ASSERT(CPU_IN_SET(cpuset, CPU->cpu_id)); + CPUSET_DEL(cpuset, CPU->cpu_id); + CPUSET_AND(cpuset, cpu_ready_set); - /* - * Downgrade to reader's lock. - */ - rw_downgrade(&ctx->ctx_rwlock); + /* + * Pass in INVALID_CONTEXT as the first parameter to + * sfmmu_raise_tsb_exception, which invalidates the context + * of any process running on the CPUs in the MMU. + */ + xt_some(cpuset, sfmmu_raise_tsb_exception, + INVALID_CONTEXT, INVALID_CONTEXT); + xt_sync(cpuset); - /* - * If this value doesn't get set to what we want - * it won't matter, so don't worry about locking. - */ - ctxhand = NEXT_CTX(ctx); + SFMMU_MMU_STAT(mmu_tsb_raise_exception); + } + + if (sfmmu_getctx_sec() != INVALID_CONTEXT) { + sfmmu_setctx_sec(INVALID_CONTEXT); + sfmmu_clear_utsbinfo(); + } /* - * Better not have been stolen while we held the ctx' - * lock or we're hosed. + * No xcall is needed here. For sun4u systems all CPUs in context + * domain share a single physical MMU therefore it's enough to flush + * TLB on local CPU. On sun4v systems we use 1 global context + * domain and flush all remote TLBs in sfmmu_raise_tsb_exception + * handler. Note that vtag_flushall_uctxs() is called + * for Ultra II machine, where the equivalent flushall functionality + * is implemented in SW, and only user ctx TLB entries are flushed. */ - ASSERT(sfmmup == sfmmutoctx(sfmmup)->ctx_sfmmu); + if (&vtag_flushall_uctxs != NULL) { + vtag_flushall_uctxs(); + } else { + vtag_flushall(); + } - return (ctx); + /* reset mmu cnum, skips cnum 0 and 1 */ + mmu_ctxp->mmu_cnum = NUM_LOCKED_CTXS; -#undef NEXT_CTX +done: + mutex_exit(&mmu_ctxp->mmu_lock); } /* - * Set the process context to INVALID_CONTEXT (but - * without stealing the ctx) so that it faults and - * reloads the MMU state from TL=0. Caller must - * hold the hat lock since we don't acquire it here. + * For multi-threaded process, set the process context to INVALID_CONTEXT + * so that it faults and reloads the MMU state from TL=0. For single-threaded + * process, we can just load the MMU state directly without having to + * set context invalid. Caller must hold the hat lock since we don't + * acquire it here. */ static void sfmmu_sync_mmustate(sfmmu_t *sfmmup) { - int cnum; - cpuset_t cpuset; + uint_t cnum; + uint_t pstate_save; ASSERT(sfmmup != ksfmmup); ASSERT(sfmmu_hat_lock_held(sfmmup)); kpreempt_disable(); - cnum = sfmmutoctxnum(sfmmup); - if (cnum != INVALID_CONTEXT) { - cpuset = sfmmup->sfmmu_cpusran; - CPUSET_DEL(cpuset, CPU->cpu_id); - CPUSET_AND(cpuset, cpu_ready_set); - SFMMU_XCALL_STATS(cnum); - - xt_some(cpuset, sfmmu_raise_tsb_exception, - cnum, INVALID_CONTEXT); - xt_sync(cpuset); - + /* + * We check whether the pass'ed-in sfmmup is the same as the + * current running proc. This is to makes sure the current proc + * stays single-threaded if it already is. + */ + if ((sfmmup == curthread->t_procp->p_as->a_hat) && + (curthread->t_procp->p_lwpcnt == 1)) { + /* single-thread */ + cnum = sfmmup->sfmmu_ctxs[CPU_MMU_IDX(CPU)].cnum; + if (cnum != INVALID_CONTEXT) { + uint_t curcnum; + /* + * Disable interrupts to prevent race condition + * with sfmmu_ctx_wrap_around ctx invalidation. + * In sun4v, ctx invalidation involves setting + * TSB to NULL, hence, interrupts should be disabled + * untill after sfmmu_load_mmustate is completed. + */ + pstate_save = sfmmu_disable_intrs(); + curcnum = sfmmu_getctx_sec(); + if (curcnum == cnum) + sfmmu_load_mmustate(sfmmup); + sfmmu_enable_intrs(pstate_save); + ASSERT(curcnum == cnum || curcnum == INVALID_CONTEXT); + } + } else { /* - * If the process is running on the local CPU - * we need to update the MMU state here as well. + * multi-thread + * or when sfmmup is not the same as the curproc. */ - if (sfmmu_getctx_sec() == cnum) - sfmmu_load_mmustate(sfmmup); - - SFMMU_STAT(sf_tsb_raise_exception); + sfmmu_invalidate_ctx(sfmmup); } kpreempt_enable(); @@ -8868,9 +8883,7 @@ sfmmu_replace_tsb(sfmmu_t *sfmmup, struct tsb_info *old_tsbinfo, uint_t szc, struct tsb_info *new_tsbinfo = NULL; struct tsb_info *curtsb, *prevtsb; uint_t tte_sz_mask; - cpuset_t cpuset; - struct ctx *ctx = NULL; - int ctxnum; + int i; ASSERT(sfmmup != ksfmmup); ASSERT(sfmmup->sfmmu_ismhat == 0); @@ -8959,27 +8972,7 @@ sfmmu_replace_tsb(sfmmu_t *sfmmup, struct tsb_info *old_tsbinfo, uint_t szc, */ if ((flags & TSB_SWAPIN) != TSB_SWAPIN) { /* The TSB is either growing or shrinking. */ - ctx = sfmmutoctx(sfmmup); - rw_enter(&ctx->ctx_rwlock, RW_WRITER); - - ctxnum = sfmmutoctxnum(sfmmup); - sfmmup->sfmmu_cnum = INVALID_CONTEXT; - membar_enter(); /* make sure visible on all CPUs */ - - kpreempt_disable(); - if (ctxnum != INVALID_CONTEXT) { - cpuset = sfmmup->sfmmu_cpusran; - CPUSET_DEL(cpuset, CPU->cpu_id); - CPUSET_AND(cpuset, cpu_ready_set); - SFMMU_XCALL_STATS(ctxnum); - - xt_some(cpuset, sfmmu_raise_tsb_exception, - ctxnum, INVALID_CONTEXT); - xt_sync(cpuset); - - SFMMU_STAT(sf_tsb_raise_exception); - } - kpreempt_enable(); + sfmmu_invalidate_ctx(sfmmup); } else { /* * It is illegal to swap in TSBs from a process other @@ -8989,9 +8982,20 @@ sfmmu_replace_tsb(sfmmu_t *sfmmup, struct tsb_info *old_tsbinfo, uint_t szc, * misses. */ ASSERT(curthread->t_procp->p_as->a_hat == sfmmup); - ASSERT(sfmmutoctxnum(sfmmup) == INVALID_CONTEXT); } +#ifdef DEBUG + ASSERT(max_mmu_ctxdoms > 0); + + /* + * Process should have INVALID_CONTEXT on all MMUs + */ + for (i = 0; i < max_mmu_ctxdoms; i++) { + + ASSERT(sfmmup->sfmmu_ctxs[i].cnum == INVALID_CONTEXT); + } +#endif + new_tsbinfo->tsb_next = old_tsbinfo->tsb_next; membar_stst(); /* strict ordering required */ if (prevtsb) @@ -9008,18 +9012,6 @@ sfmmu_replace_tsb(sfmmu_t *sfmmup, struct tsb_info *old_tsbinfo, uint_t szc, if (tsb_remap_ttes && ((flags & TSB_GROW) == TSB_GROW)) sfmmu_copy_tsb(old_tsbinfo, new_tsbinfo); - if ((flags & TSB_SWAPIN) != TSB_SWAPIN) { - kpreempt_disable(); - membar_exit(); - sfmmup->sfmmu_cnum = ctxnum; - if (ctxnum != INVALID_CONTEXT && - sfmmu_getctx_sec() == ctxnum) { - sfmmu_load_mmustate(sfmmup); - } - kpreempt_enable(); - rw_exit(&ctx->ctx_rwlock); - } - SFMMU_FLAGS_CLEAR(sfmmup, HAT_BUSY); /* @@ -9040,15 +9032,15 @@ sfmmu_replace_tsb(sfmmu_t *sfmmup, struct tsb_info *old_tsbinfo, uint_t szc, } /* - * Steal context from process, forcing the process to switch to another - * context on the next TLB miss, and therefore start using the TLB that - * is reprogrammed for the new page sizes. + * This function will re-program hat pgsz array, and invalidate the + * process' context, forcing the process to switch to another + * context on the next TLB miss, and therefore start using the + * TLB that is reprogrammed for the new page sizes. */ void -sfmmu_steal_context(sfmmu_t *sfmmup, uint8_t *tmp_pgsz) +sfmmu_reprog_pgsz_arr(sfmmu_t *sfmmup, uint8_t *tmp_pgsz) { - struct ctx *ctx; - int i, cnum; + int i; hatlock_t *hatlockp = NULL; hatlockp = sfmmu_hat_enter(sfmmup); @@ -9058,14 +9050,9 @@ sfmmu_steal_context(sfmmu_t *sfmmup, uint8_t *tmp_pgsz) sfmmup->sfmmu_pgsz[i] = tmp_pgsz[i]; } SFMMU_STAT(sf_tlb_reprog_pgsz); - ctx = sfmmutoctx(sfmmup); - rw_enter(&ctx->ctx_rwlock, RW_WRITER); - cnum = sfmmutoctxnum(sfmmup); - if (cnum != INVALID_CONTEXT) { - sfmmu_tlb_swap_ctx(sfmmup, ctx); - } - rw_exit(&ctx->ctx_rwlock); + sfmmu_invalidate_ctx(sfmmup); + sfmmu_hat_exit(hatlockp); } @@ -9326,50 +9313,6 @@ hat_preferred_pgsz(struct hat *hat, caddr_t vaddr, size_t maplen, int maptype) } /* - * Free up a ctx - */ -static void -sfmmu_free_ctx(sfmmu_t *sfmmup, struct ctx *ctx) -{ - int ctxnum; - - rw_enter(&ctx->ctx_rwlock, RW_WRITER); - - TRACE_CTXS(&ctx_trace_mutex, ctx_trace_ptr, sfmmup->sfmmu_cnum, - sfmmup, 0, CTX_TRC_FREE); - - if (sfmmup->sfmmu_cnum == INVALID_CONTEXT) { - CPUSET_ZERO(sfmmup->sfmmu_cpusran); - rw_exit(&ctx->ctx_rwlock); - return; - } - - ASSERT(sfmmup == ctx->ctx_sfmmu); - - ctx->ctx_sfmmu = NULL; - ctx->ctx_flags = 0; - sfmmup->sfmmu_cnum = INVALID_CONTEXT; - membar_enter(); - CPUSET_ZERO(sfmmup->sfmmu_cpusran); - ctxnum = sfmmu_getctx_sec(); - if (ctxnum == ctxtoctxnum(ctx)) { - sfmmu_setctx_sec(INVALID_CONTEXT); - sfmmu_clear_utsbinfo(); - } - - /* - * Put the freed ctx on the dirty list - */ - mutex_enter(&ctx_list_lock); - CTX_SET_FLAGS(ctx, CTX_FREE_FLAG); - ctx->ctx_free = ctxdirty; - ctxdirty = ctx; - mutex_exit(&ctx_list_lock); - - rw_exit(&ctx->ctx_rwlock); -} - -/* * Free up a sfmmu * Since the sfmmu is currently embedded in the hat struct we simply zero * out our fields and free up the ism map blk list if any. @@ -9389,7 +9332,7 @@ sfmmu_free_sfmmu(sfmmu_t *sfmmup) ASSERT(sfmmup->sfmmu_ttecnt[TTE4M] == 0); ASSERT(sfmmup->sfmmu_ttecnt[TTE32M] == 0); ASSERT(sfmmup->sfmmu_ttecnt[TTE256M] == 0); - ASSERT(sfmmup->sfmmu_cnum == INVALID_CONTEXT); + sfmmup->sfmmu_free = 0; sfmmup->sfmmu_ismhat = 0; @@ -10531,65 +10474,6 @@ sfmmu_hmetohblk(struct sf_hment *sfhme) } /* - * Make sure that there is a valid ctx, if not get a ctx. - * Also, get a readers lock on the ctx, so that the ctx cannot - * be stolen underneath us. - */ -static void -sfmmu_disallow_ctx_steal(sfmmu_t *sfmmup) -{ - struct ctx *ctx; - - ASSERT(sfmmup != ksfmmup); - ASSERT(sfmmup->sfmmu_ismhat == 0); - - /* - * If ctx has been stolen, get a ctx. - */ - if (sfmmup->sfmmu_cnum == INVALID_CONTEXT) { - /* - * Our ctx was stolen. Get a ctx with rlock. - */ - ctx = sfmmu_get_ctx(sfmmup); - return; - } else { - ctx = sfmmutoctx(sfmmup); - } - - /* - * Get the reader lock. - */ - rw_enter(&ctx->ctx_rwlock, RW_READER); - if (ctx->ctx_sfmmu != sfmmup) { - /* - * The ctx got stolen, so spin again. - */ - rw_exit(&ctx->ctx_rwlock); - ctx = sfmmu_get_ctx(sfmmup); - } - - ASSERT(sfmmup->sfmmu_cnum >= NUM_LOCKED_CTXS); -} - -/* - * Decrement reference count for our ctx. If the reference count - * becomes 0, our ctx can be stolen by someone. - */ -static void -sfmmu_allow_ctx_steal(sfmmu_t *sfmmup) -{ - struct ctx *ctx; - - ASSERT(sfmmup != ksfmmup); - ASSERT(sfmmup->sfmmu_ismhat == 0); - ctx = sfmmutoctx(sfmmup); - - ASSERT(sfmmup == ctx->ctx_sfmmu); - ASSERT(sfmmup->sfmmu_cnum != INVALID_CONTEXT); - rw_exit(&ctx->ctx_rwlock); -} - -/* * On swapin, get appropriately sized TSB(s) and clear the HAT_SWAPPED flag. * If we can't get appropriately sized TSB(s), try for 8K TSB(s) using * KM_SLEEP allocation. @@ -10683,22 +10567,14 @@ sfmmu_tsb_swapin(sfmmu_t *sfmmup, hatlock_t *hatlockp) * * There are many scenarios that could land us here: * - * 1) Process has no context. In this case, ctx is - * INVALID_CONTEXT and sfmmup->sfmmu_cnum == 1 so - * we will acquire a context before returning. - * 2) Need to re-load our MMU state. In this case, - * ctx is INVALID_CONTEXT and sfmmup->sfmmu_cnum != 1. - * 3) ISM mappings are being updated. This is handled - * just like case #2. - * 4) We wish to program a new page size into the TLB. - * This is handled just like case #1, since changing - * TLB page size requires us to flush the TLB. - * 5) Window fault and no valid translation found. - * - * Cases 1-4, ctx is INVALID_CONTEXT so we handle it and then - * exit which will retry the trapped instruction. Case #5 we - * punt to trap() which will raise us a trap level and handle - * the fault before unwinding. + * If the context is invalid we land here. The context can be invalid + * for 3 reasons: 1) we couldn't allocate a new context and now need to + * perform a wrap around operation in order to allocate a new context. + * 2) Context was invalidated to change pagesize programming 3) ISMs or + * TSBs configuration is changeing for this process and we are forced into + * here to do a syncronization operation. If the context is valid we can + * be here from window trap hanlder. In this case just call trap to handle + * the fault. * * Note that the process will run in INVALID_CONTEXT before * faulting into here and subsequently loading the MMU registers @@ -10718,6 +10594,7 @@ sfmmu_tsbmiss_exception(struct regs *rp, uintptr_t tagaccess, uint_t traptype) struct tsb_info *tsbinfop; SFMMU_STAT(sf_tsb_exceptions); + SFMMU_MMU_STAT(mmu_tsb_exceptions); sfmmup = astosfmmu(curthread->t_procp->p_as); ctxnum = tagaccess & TAGACC_CTX_MASK; @@ -10737,8 +10614,10 @@ sfmmu_tsbmiss_exception(struct regs *rp, uintptr_t tagaccess, uint_t traptype) * locking the HAT and grabbing the rwlock on the context as a * reader temporarily. */ - if (ctxnum == INVALID_CONTEXT || - SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED)) { + ASSERT(!SFMMU_FLAGS_ISSET(sfmmup, HAT_SWAPPED) || + ctxnum == INVALID_CONTEXT); + + if (ctxnum == INVALID_CONTEXT) { /* * Must set lwp state to LWP_SYS before * trying to acquire any adaptive lock @@ -10764,7 +10643,7 @@ retry: */ if (SFMMU_FLAGS_ISSET(sfmmup, HAT_ISMBUSY)) { cv_wait(&sfmmup->sfmmu_tsb_cv, - HATLOCK_MUTEXP(hatlockp)); + HATLOCK_MUTEXP(hatlockp)); goto retry; } @@ -10779,13 +10658,8 @@ retry: goto retry; } - sfmmu_disallow_ctx_steal(sfmmup); - ctxnum = sfmmup->sfmmu_cnum; - kpreempt_disable(); - sfmmu_setctx_sec(ctxnum); - sfmmu_load_mmustate(sfmmup); - kpreempt_enable(); - sfmmu_allow_ctx_steal(sfmmup); + sfmmu_get_ctx(sfmmup); + sfmmu_hat_exit(hatlockp); /* * Must restore lwp_state if not calling @@ -10857,7 +10731,6 @@ sfmmu_ismtlbcache_demap(caddr_t addr, sfmmu_t *ism_sfmmup, caddr_t va; ism_ment_t *ment; sfmmu_t *sfmmup; - int ctxnum; int vcolor; int ttesz; @@ -10877,7 +10750,7 @@ sfmmu_ismtlbcache_demap(caddr_t addr, sfmmu_t *ism_sfmmup, for (ment = ism_sfmmup->sfmmu_iment; ment; ment = ment->iment_next) { sfmmup = ment->iment_hat; - ctxnum = sfmmup->sfmmu_cnum; + va = ment->iment_base_va; va = (caddr_t)((uintptr_t)va + (uintptr_t)addr); @@ -10895,20 +10768,15 @@ sfmmu_ismtlbcache_demap(caddr_t addr, sfmmu_t *ism_sfmmup, sfmmu_unload_tsb_range(sfmmup, sva, eva, ttesz); } - if (ctxnum != INVALID_CONTEXT) { - /* - * Flush TLBs. We don't need to do this for - * invalid context since the flushing is already - * done as part of context stealing. - */ - cpuset = sfmmup->sfmmu_cpusran; - CPUSET_AND(cpuset, cpu_ready_set); - CPUSET_DEL(cpuset, CPU->cpu_id); - SFMMU_XCALL_STATS(ctxnum); - xt_some(cpuset, vtag_flushpage_tl1, (uint64_t)va, - ctxnum); - vtag_flushpage(va, ctxnum); - } + cpuset = sfmmup->sfmmu_cpusran; + CPUSET_AND(cpuset, cpu_ready_set); + CPUSET_DEL(cpuset, CPU->cpu_id); + + SFMMU_XCALL_STATS(sfmmup); + xt_some(cpuset, vtag_flushpage_tl1, (uint64_t)va, + (uint64_t)sfmmup); + + vtag_flushpage(va, (uint64_t)sfmmup); /* * Flush D$ @@ -10918,7 +10786,8 @@ sfmmu_ismtlbcache_demap(caddr_t addr, sfmmu_t *ism_sfmmup, if (cache_flush_flag == CACHE_FLUSH) { cpuset = cpu_ready_set; CPUSET_DEL(cpuset, CPU->cpu_id); - SFMMU_XCALL_STATS(ctxnum); + + SFMMU_XCALL_STATS(sfmmup); vcolor = addr_to_vcolor(va); xt_some(cpuset, vac_flushpage_tl1, pfnum, vcolor); vac_flushpage(pfnum, vcolor); @@ -10937,7 +10806,7 @@ sfmmu_tlbcache_demap(caddr_t addr, sfmmu_t *sfmmup, struct hme_blk *hmeblkp, pfn_t pfnum, int tlb_noflush, int cpu_flag, int cache_flush_flag, int hat_lock_held) { - int ctxnum, vcolor; + int vcolor; cpuset_t cpuset; hatlock_t *hatlockp; @@ -10947,35 +10816,42 @@ sfmmu_tlbcache_demap(caddr_t addr, sfmmu_t *sfmmup, struct hme_blk *hmeblkp, */ vcolor = addr_to_vcolor(addr); + /* + * We must hold the hat lock during the flush of TLB, + * to avoid a race with sfmmu_invalidate_ctx(), where + * sfmmu_cnum on a MMU could be set to INVALID_CONTEXT, + * causing TLB demap routine to skip flush on that MMU. + * If the context on a MMU has already been set to + * INVALID_CONTEXT, we just get an extra flush on + * that MMU. + */ + if (!hat_lock_held && !tlb_noflush) + hatlockp = sfmmu_hat_enter(sfmmup); + kpreempt_disable(); if (!tlb_noflush) { /* - * Flush the TSB. + * Flush the TSB and TLB. */ - if (!hat_lock_held) - hatlockp = sfmmu_hat_enter(sfmmup); SFMMU_UNLOAD_TSB(addr, sfmmup, hmeblkp); - ctxnum = (int)sfmmutoctxnum(sfmmup); - if (!hat_lock_held) - sfmmu_hat_exit(hatlockp); - if (ctxnum != INVALID_CONTEXT) { - /* - * Flush TLBs. We don't need to do this if our - * context is invalid context. Since we hold the - * HAT lock the context must have been stolen and - * hence will be flushed before re-use. - */ - cpuset = sfmmup->sfmmu_cpusran; - CPUSET_AND(cpuset, cpu_ready_set); - CPUSET_DEL(cpuset, CPU->cpu_id); - SFMMU_XCALL_STATS(ctxnum); - xt_some(cpuset, vtag_flushpage_tl1, (uint64_t)addr, - ctxnum); - vtag_flushpage(addr, ctxnum); - } + cpuset = sfmmup->sfmmu_cpusran; + CPUSET_AND(cpuset, cpu_ready_set); + CPUSET_DEL(cpuset, CPU->cpu_id); + + SFMMU_XCALL_STATS(sfmmup); + + xt_some(cpuset, vtag_flushpage_tl1, (uint64_t)addr, + (uint64_t)sfmmup); + + vtag_flushpage(addr, (uint64_t)sfmmup); + } + if (!hat_lock_held && !tlb_noflush) + sfmmu_hat_exit(hatlockp); + + /* * Flush the D$ * @@ -10990,7 +10866,7 @@ sfmmu_tlbcache_demap(caddr_t addr, sfmmu_t *sfmmup, struct hme_blk *hmeblkp, CPUSET_AND(cpuset, cpu_ready_set); } CPUSET_DEL(cpuset, CPU->cpu_id); - SFMMU_XCALL_STATS(sfmmutoctxnum(sfmmup)); + SFMMU_XCALL_STATS(sfmmup); xt_some(cpuset, vac_flushpage_tl1, pfnum, vcolor); vac_flushpage(pfnum, vcolor); } @@ -11006,7 +10882,6 @@ static void sfmmu_tlb_demap(caddr_t addr, sfmmu_t *sfmmup, struct hme_blk *hmeblkp, int tlb_noflush, int hat_lock_held) { - int ctxnum; cpuset_t cpuset; hatlock_t *hatlockp; @@ -11022,29 +10897,23 @@ sfmmu_tlb_demap(caddr_t addr, sfmmu_t *sfmmup, struct hme_blk *hmeblkp, if (!hat_lock_held) hatlockp = sfmmu_hat_enter(sfmmup); SFMMU_UNLOAD_TSB(addr, sfmmup, hmeblkp); - ctxnum = sfmmutoctxnum(sfmmup); + + kpreempt_disable(); + + cpuset = sfmmup->sfmmu_cpusran; + CPUSET_AND(cpuset, cpu_ready_set); + CPUSET_DEL(cpuset, CPU->cpu_id); + + SFMMU_XCALL_STATS(sfmmup); + xt_some(cpuset, vtag_flushpage_tl1, (uint64_t)addr, (uint64_t)sfmmup); + + vtag_flushpage(addr, (uint64_t)sfmmup); + if (!hat_lock_held) sfmmu_hat_exit(hatlockp); - /* - * Flush TLBs. We don't need to do this if our context is invalid - * context. Since we hold the HAT lock the context must have been - * stolen and hence will be flushed before re-use. - */ - if (ctxnum != INVALID_CONTEXT) { - /* - * There is no need to protect against ctx being stolen. - * If the ctx is stolen we will simply get an extra flush. - */ - kpreempt_disable(); - cpuset = sfmmup->sfmmu_cpusran; - CPUSET_AND(cpuset, cpu_ready_set); - CPUSET_DEL(cpuset, CPU->cpu_id); - SFMMU_XCALL_STATS(ctxnum); - xt_some(cpuset, vtag_flushpage_tl1, (uint64_t)addr, ctxnum); - vtag_flushpage(addr, ctxnum); - kpreempt_enable(); - } + kpreempt_enable(); + } /* @@ -11057,10 +10926,9 @@ static void sfmmu_tlb_range_demap(demap_range_t *dmrp) { sfmmu_t *sfmmup = dmrp->dmr_sfmmup; - int ctxnum; hatlock_t *hatlockp; cpuset_t cpuset; - uint64_t ctx_pgcnt; + uint64_t sfmmu_pgcnt; pgcnt_t pgcnt = 0; int pgunload = 0; int dirtypg = 0; @@ -11098,117 +10966,64 @@ sfmmu_tlb_range_demap(demap_range_t *dmrp) pgcnt += dirtypg; } - /* - * In the case where context is invalid context, bail. - * We hold the hat lock while checking the ctx to prevent - * a race with sfmmu_replace_tsb() which temporarily sets - * the ctx to INVALID_CONTEXT to force processes to enter - * sfmmu_tsbmiss_exception(). - */ - hatlockp = sfmmu_hat_enter(sfmmup); - ctxnum = sfmmutoctxnum(sfmmup); - sfmmu_hat_exit(hatlockp); - if (ctxnum == INVALID_CONTEXT) { - dmrp->dmr_bitvec = 0; - return; - } - ASSERT((pgcnt<<MMU_PAGESHIFT) <= dmrp->dmr_endaddr - dmrp->dmr_addr); if (sfmmup->sfmmu_free == 0) { addr = dmrp->dmr_addr; bitvec = dmrp->dmr_bitvec; - ctx_pgcnt = (uint64_t)((ctxnum << 16) | pgcnt); + + /* + * make sure it has SFMMU_PGCNT_SHIFT bits only, + * as it will be used to pack argument for xt_some + */ + ASSERT((pgcnt > 0) && + (pgcnt <= (1 << SFMMU_PGCNT_SHIFT))); + + /* + * Encode pgcnt as (pgcnt -1 ), and pass (pgcnt - 1) in + * the low 6 bits of sfmmup. This is doable since pgcnt + * always >= 1. + */ + ASSERT(!((uint64_t)sfmmup & SFMMU_PGCNT_MASK)); + sfmmu_pgcnt = (uint64_t)sfmmup | + ((pgcnt - 1) & SFMMU_PGCNT_MASK); + + /* + * We must hold the hat lock during the flush of TLB, + * to avoid a race with sfmmu_invalidate_ctx(), where + * sfmmu_cnum on a MMU could be set to INVALID_CONTEXT, + * causing TLB demap routine to skip flush on that MMU. + * If the context on a MMU has already been set to + * INVALID_CONTEXT, we just get an extra flush on + * that MMU. + */ + hatlockp = sfmmu_hat_enter(sfmmup); kpreempt_disable(); + cpuset = sfmmup->sfmmu_cpusran; CPUSET_AND(cpuset, cpu_ready_set); CPUSET_DEL(cpuset, CPU->cpu_id); - SFMMU_XCALL_STATS(ctxnum); + + SFMMU_XCALL_STATS(sfmmup); xt_some(cpuset, vtag_flush_pgcnt_tl1, (uint64_t)addr, - ctx_pgcnt); + sfmmu_pgcnt); + for (; bitvec != 0; bitvec >>= 1) { if (bitvec & 1) - vtag_flushpage(addr, ctxnum); + vtag_flushpage(addr, (uint64_t)sfmmup); addr += MMU_PAGESIZE; } kpreempt_enable(); + sfmmu_hat_exit(hatlockp); + sfmmu_xcall_save += (pgunload-1); } dmrp->dmr_bitvec = 0; } /* - * Flushes only TLB. - */ -static void -sfmmu_tlb_ctx_demap(sfmmu_t *sfmmup) -{ - int ctxnum; - cpuset_t cpuset; - - ctxnum = (int)sfmmutoctxnum(sfmmup); - if (ctxnum == INVALID_CONTEXT) { - /* - * if ctx was stolen then simply return - * whoever stole ctx is responsible for flush. - */ - return; - } - ASSERT(ctxnum != KCONTEXT); - /* - * There is no need to protect against ctx being stolen. If the - * ctx is stolen we will simply get an extra flush. - */ - kpreempt_disable(); - - cpuset = sfmmup->sfmmu_cpusran; - CPUSET_DEL(cpuset, CPU->cpu_id); - CPUSET_AND(cpuset, cpu_ready_set); - SFMMU_XCALL_STATS(ctxnum); - - /* - * Flush TLB. - * RFE: it might be worth delaying the TLB flush as well. In that - * case each cpu would have to traverse the dirty list and flush - * each one of those ctx from the TLB. - */ - vtag_flushctx(ctxnum); - xt_some(cpuset, vtag_flushctx_tl1, ctxnum, 0); - - kpreempt_enable(); - SFMMU_STAT(sf_tlbflush_ctx); -} - -/* - * Flushes all TLBs. - */ -static void -sfmmu_tlb_all_demap(void) -{ - cpuset_t cpuset; - - /* - * There is no need to protect against ctx being stolen. If the - * ctx is stolen we will simply get an extra flush. - */ - kpreempt_disable(); - - cpuset = cpu_ready_set; - CPUSET_DEL(cpuset, CPU->cpu_id); - /* LINTED: constant in conditional context */ - SFMMU_XCALL_STATS(INVALID_CONTEXT); - - vtag_flushall(); - xt_some(cpuset, vtag_flushall_tl1, 0, 0); - xt_sync(cpuset); - - kpreempt_enable(); - SFMMU_STAT(sf_tlbflush_all); -} - -/* * In cases where we need to synchronize with TLB/TSB miss trap * handlers, _and_ need to flush the TLB, it's a lot easier to - * steal the context from the process and free it than to do a + * throw away the context from the process than to do a * special song and dance to keep things consistent for the * handlers. * @@ -11221,79 +11036,73 @@ sfmmu_tlb_all_demap(void) * * One added advantage of this approach is that on MMUs that * support a "flush all" operation, we will delay the flush until - * we run out of contexts, and then flush the TLB one time. This + * cnum wrap-around, and then flush the TLB one time. This * is rather rare, so it's a lot less expensive than making 8000 - * x-calls to flush the TLB 8000 times. Another is that we can do - * all of this without pausing CPUs, due to some knowledge of how - * resume() loads processes onto the processor; it sets the thread - * into cpusran, and _then_ looks at cnum. Because we do things in - * the reverse order here, we guarantee exactly one of the following - * statements is always true: + * x-calls to flush the TLB 8000 times. * - * 1) Nobody is in resume() so we have nothing to worry about anyway. - * 2) The thread in resume() isn't in cpusran when we do the xcall, - * so we know when it does set itself it'll see cnum is - * INVALID_CONTEXT. - * 3) The thread in resume() is in cpusran, and already might have - * looked at the old cnum. That's OK, because we'll xcall it - * and, if necessary, flush the TLB along with the rest of the - * crowd. + * A per-process (PP) lock is used to synchronize ctx allocations in + * resume() and ctx invalidations here. */ static void -sfmmu_tlb_swap_ctx(sfmmu_t *sfmmup, struct ctx *ctx) +sfmmu_invalidate_ctx(sfmmu_t *sfmmup) { cpuset_t cpuset; - int cnum; + int cnum, currcnum; + mmu_ctx_t *mmu_ctxp; + int i; + uint_t pstate_save; - if (sfmmup->sfmmu_cnum == INVALID_CONTEXT) - return; + SFMMU_STAT(sf_ctx_inv); - SFMMU_STAT(sf_ctx_swap); + ASSERT(sfmmu_hat_lock_held(sfmmup)); + ASSERT(sfmmup != ksfmmup); kpreempt_disable(); - ASSERT(rw_read_locked(&ctx->ctx_rwlock) == 0); - ASSERT(ctx->ctx_sfmmu == sfmmup); + mmu_ctxp = CPU_MMU_CTXP(CPU); + ASSERT(mmu_ctxp); + ASSERT(mmu_ctxp->mmu_idx < max_mmu_ctxdoms); + ASSERT(mmu_ctxp == mmu_ctxs_tbl[mmu_ctxp->mmu_idx]); + + currcnum = sfmmup->sfmmu_ctxs[mmu_ctxp->mmu_idx].cnum; + + pstate_save = sfmmu_disable_intrs(); + + lock_set(&sfmmup->sfmmu_ctx_lock); /* acquire PP lock */ + /* set HAT cnum invalid across all context domains. */ + for (i = 0; i < max_mmu_ctxdoms; i++) { - cnum = ctxtoctxnum(ctx); - ASSERT(sfmmup->sfmmu_cnum == cnum); - ASSERT(cnum >= NUM_LOCKED_CTXS); + cnum = sfmmup->sfmmu_ctxs[i].cnum; + if (cnum == INVALID_CONTEXT) { + continue; + } + + sfmmup->sfmmu_ctxs[i].cnum = INVALID_CONTEXT; + } + membar_enter(); /* make sure globally visible to all CPUs */ + lock_clear(&sfmmup->sfmmu_ctx_lock); /* release PP lock */ - sfmmup->sfmmu_cnum = INVALID_CONTEXT; - membar_enter(); /* make sure visible on all CPUs */ - ctx->ctx_sfmmu = NULL; + sfmmu_enable_intrs(pstate_save); cpuset = sfmmup->sfmmu_cpusran; CPUSET_DEL(cpuset, CPU->cpu_id); CPUSET_AND(cpuset, cpu_ready_set); - SFMMU_XCALL_STATS(cnum); - - /* - * Force anybody running this process on CPU - * to enter sfmmu_tsbmiss_exception() on the - * next TLB miss, synchronize behind us on - * the HAT lock, and grab a new context. At - * that point the new page size will become - * active in the TLB for the new context. - * See sfmmu_get_ctx() for details. - */ - if (delay_tlb_flush) { + if (!CPUSET_ISNULL(cpuset)) { + SFMMU_XCALL_STATS(sfmmup); xt_some(cpuset, sfmmu_raise_tsb_exception, - cnum, INVALID_CONTEXT); - SFMMU_STAT(sf_tlbflush_deferred); - } else { - xt_some(cpuset, sfmmu_ctx_steal_tl1, cnum, INVALID_CONTEXT); - vtag_flushctx(cnum); - SFMMU_STAT(sf_tlbflush_ctx); + (uint64_t)sfmmup, INVALID_CONTEXT); + xt_sync(cpuset); + SFMMU_STAT(sf_tsb_raise_exception); + SFMMU_MMU_STAT(mmu_tsb_raise_exception); } - xt_sync(cpuset); /* - * If we just stole the ctx from the current + * If the hat to-be-invalidated is the same as the current * process on local CPU we need to invalidate * this CPU context as well. */ - if (sfmmu_getctx_sec() == cnum) { + if ((sfmmu_getctx_sec() == currcnum) && + (currcnum != INVALID_CONTEXT)) { sfmmu_setctx_sec(INVALID_CONTEXT); sfmmu_clear_utsbinfo(); } @@ -11301,15 +11110,10 @@ sfmmu_tlb_swap_ctx(sfmmu_t *sfmmup, struct ctx *ctx) kpreempt_enable(); /* - * Now put old ctx on the dirty list since we may not - * have flushed the context out of the TLB. We'll let - * the next guy who uses this ctx flush it instead. + * we hold the hat lock, so nobody should allocate a context + * for us yet */ - mutex_enter(&ctx_list_lock); - CTX_SET_FLAGS(ctx, CTX_FREE_FLAG); - ctx->ctx_free = ctxdirty; - ctxdirty = ctx; - mutex_exit(&ctx_list_lock); + ASSERT(sfmmup->sfmmu_ctxs[mmu_ctxp->mmu_idx].cnum == INVALID_CONTEXT); } /* @@ -11322,12 +11126,11 @@ void sfmmu_cache_flush(pfn_t pfnum, int vcolor) { cpuset_t cpuset; - int ctxnum = INVALID_CONTEXT; kpreempt_disable(); cpuset = cpu_ready_set; CPUSET_DEL(cpuset, CPU->cpu_id); - SFMMU_XCALL_STATS(ctxnum); /* account to any ctx */ + SFMMU_XCALL_STATS(NULL); /* account to any ctx */ xt_some(cpuset, vac_flushpage_tl1, pfnum, vcolor); xt_sync(cpuset); vac_flushpage(pfnum, vcolor); @@ -11338,14 +11141,13 @@ void sfmmu_cache_flushcolor(int vcolor, pfn_t pfnum) { cpuset_t cpuset; - int ctxnum = INVALID_CONTEXT; ASSERT(vcolor >= 0); kpreempt_disable(); cpuset = cpu_ready_set; CPUSET_DEL(cpuset, CPU->cpu_id); - SFMMU_XCALL_STATS(ctxnum); /* account to any ctx */ + SFMMU_XCALL_STATS(NULL); /* account to any ctx */ xt_some(cpuset, vac_flushcolor_tl1, vcolor, pfnum); xt_sync(cpuset); vac_flushcolor(vcolor, pfnum); @@ -11367,8 +11169,6 @@ sfmmu_tsb_pre_relocator(caddr_t va, uint_t tsbsz, uint_t flags, void *tsbinfo) hatlock_t *hatlockp; struct tsb_info *tsbinfop = (struct tsb_info *)tsbinfo; sfmmu_t *sfmmup = tsbinfop->tsb_sfmmu; - struct ctx *ctx; - int cnum; extern uint32_t sendmondo_in_recover; if (flags != HAT_PRESUSPEND) @@ -11408,20 +11208,7 @@ sfmmu_tsb_pre_relocator(caddr_t va, uint_t tsbsz, uint_t flags, void *tsbinfo) } } - ctx = sfmmutoctx(sfmmup); - rw_enter(&ctx->ctx_rwlock, RW_WRITER); - cnum = sfmmutoctxnum(sfmmup); - - if (cnum != INVALID_CONTEXT) { - /* - * Force all threads for this sfmmu to sfmmu_tsbmiss_exception - * on their next TLB miss. - */ - sfmmu_tlb_swap_ctx(sfmmup, ctx); - } - - rw_exit(&ctx->ctx_rwlock); - + sfmmu_invalidate_ctx(sfmmup); sfmmu_hat_exit(hatlockp); return (0); @@ -13174,7 +12961,7 @@ sfmmu_kpm_mapout(page_t *pp, caddr_t vaddr) sfmmu_kpm_unload_tsb(vaddr, MMU_PAGESHIFT4M); #ifdef DEBUG if (kpm_tlb_flush) - sfmmu_kpm_demap_tlbs(vaddr, KCONTEXT); + sfmmu_kpm_demap_tlbs(vaddr); #endif } @@ -13271,7 +13058,7 @@ smallpages_mapout: sfmmu_kpm_unload_tsb(vaddr, MMU_PAGESHIFT); #ifdef DEBUG if (kpm_tlb_flush) - sfmmu_kpm_demap_tlbs(vaddr, KCONTEXT); + sfmmu_kpm_demap_tlbs(vaddr); #endif } else if (PP_ISTNC(pp)) { @@ -13968,7 +13755,7 @@ static void sfmmu_kpm_demap_large(caddr_t vaddr) { sfmmu_kpm_unload_tsb(vaddr, MMU_PAGESHIFT4M); - sfmmu_kpm_demap_tlbs(vaddr, KCONTEXT); + sfmmu_kpm_demap_tlbs(vaddr); } /* @@ -13978,14 +13765,14 @@ static void sfmmu_kpm_demap_small(caddr_t vaddr) { sfmmu_kpm_unload_tsb(vaddr, MMU_PAGESHIFT); - sfmmu_kpm_demap_tlbs(vaddr, KCONTEXT); + sfmmu_kpm_demap_tlbs(vaddr); } /* * Demap a kpm mapping in all TLB's. */ static void -sfmmu_kpm_demap_tlbs(caddr_t vaddr, int ctxnum) +sfmmu_kpm_demap_tlbs(caddr_t vaddr) { cpuset_t cpuset; @@ -13993,9 +13780,12 @@ sfmmu_kpm_demap_tlbs(caddr_t vaddr, int ctxnum) cpuset = ksfmmup->sfmmu_cpusran; CPUSET_AND(cpuset, cpu_ready_set); CPUSET_DEL(cpuset, CPU->cpu_id); - SFMMU_XCALL_STATS(ctxnum); - xt_some(cpuset, vtag_flushpage_tl1, (uint64_t)vaddr, ctxnum); - vtag_flushpage(vaddr, ctxnum); + SFMMU_XCALL_STATS(ksfmmup); + + xt_some(cpuset, vtag_flushpage_tl1, (uint64_t)vaddr, + (uint64_t)ksfmmup); + vtag_flushpage(vaddr, (uint64_t)ksfmmup); + kpreempt_enable(); } @@ -14401,7 +14191,7 @@ sfmmu_kpm_page_cache(page_t *pp, int flags, int cache_flush_tag) /* Flush vcolor in DCache */ CPUSET_DEL(cpuset, CPU->cpu_id); - SFMMU_XCALL_STATS(ksfmmup->sfmmu_cnum); + SFMMU_XCALL_STATS(ksfmmup); xt_some(cpuset, vac_flushpage_tl1, pfn, vcolor); vac_flushpage(pfn, vcolor); } @@ -14590,8 +14380,23 @@ hat_dump(void) void hat_thread_exit(kthread_t *thd) { + uint64_t pgsz_cnum; + uint_t pstate_save; + ASSERT(thd->t_procp->p_as == &kas); - sfmmu_setctx_sec(KCONTEXT); + pgsz_cnum = KCONTEXT; +#ifdef sun4u + pgsz_cnum |= (ksfmmup->sfmmu_cext << CTXREG_EXT_SHIFT); +#endif + /* + * Note that sfmmu_load_mmustate() is currently a no-op for + * kernel threads. We need to disable interrupts here, + * simply because otherwise sfmmu_load_mmustate() would panic + * if the caller does not disable interrupts. + */ + pstate_save = sfmmu_disable_intrs(); + sfmmu_setctx_sec(pgsz_cnum); sfmmu_load_mmustate(ksfmmup); + sfmmu_enable_intrs(pstate_save); } diff --git a/usr/src/uts/sfmmu/vm/hat_sfmmu.h b/usr/src/uts/sfmmu/vm/hat_sfmmu.h index c90663408a..74ddf0187c 100644 --- a/usr/src/uts/sfmmu/vm/hat_sfmmu.h +++ b/usr/src/uts/sfmmu/vm/hat_sfmmu.h @@ -259,6 +259,121 @@ struct tsb_info { #define TSB_SWAPPED 0x4 /* + * Per-MMU context domain kstats. + * + * TSB Miss Exceptions + * Number of times a TSB miss exception is handled in an MMU. See + * sfmmu_tsbmiss_exception() for more details. + * TSB Raise Exception + * Number of times the CPUs within an MMU are cross-called + * to invalidate either a specific process context (when the process + * switches MMU contexts) or the context of any process that is + * running on those CPUs (as part of the MMU context wrap-around). + * Wrap Around + * The number of times a wrap-around of MMU context happens. + */ +typedef enum mmu_ctx_stat_types { + MMU_CTX_TSB_EXCEPTIONS, /* TSB miss exceptions handled */ + MMU_CTX_TSB_RAISE_EXCEPTION, /* ctx invalidation cross calls */ + MMU_CTX_WRAP_AROUND, /* wraparounds */ + MMU_CTX_NUM_STATS +} mmu_ctx_stat_t; + +/* + * Per-MMU context domain structure. This is instantiated the first time a CPU + * belonging to the MMU context domain is configured into the system, at boot + * time or at DR time. + * + * mmu_gnum + * The current generation number for the context IDs on this MMU context + * domain. It is protected by mmu_lock. + * mmu_cnum + * The current cnum to be allocated on this MMU context domain. It + * is protected via CAS. + * mmu_nctxs + * The max number of context IDs supported on every CPU in this + * MMU context domain. It is 8K except for Rock where it is 64K. + * This is needed here in case the system supports mixed type of + * processors/MMUs. It also helps to make ctx switch code access + * fewer cache lines i.e. no need to retrieve it from some global nctxs. + * mmu_lock + * The mutex spin lock used to serialize context ID wrap around + * mmu_idx + * The index for this MMU context domain structure in the global array + * mmu_ctxdoms. + * mmu_ncpus + * The actual number of CPUs that have been configured in this + * MMU context domain. This also acts as a reference count for the + * structure. When the last CPU in an MMU context domain is unconfigured, + * the structure is freed. It is protected by mmu_lock. + * mmu_cpuset + * The CPU set of configured CPUs for this MMU context domain. Used + * to cross-call all the CPUs in the MMU context domain to invalidate + * context IDs during a wraparound operation. It is protected by mmu_lock. + */ + +typedef struct mmu_ctx { + uint64_t mmu_gnum; + uint_t mmu_cnum; + uint_t mmu_nctxs; + kmutex_t mmu_lock; + uint_t mmu_idx; + uint_t mmu_ncpus; + cpuset_t mmu_cpuset; + kstat_t *mmu_kstat; + kstat_named_t mmu_kstat_data[MMU_CTX_NUM_STATS]; +} mmu_ctx_t; + +#define mmu_tsb_exceptions \ + mmu_kstat_data[MMU_CTX_TSB_EXCEPTIONS].value.ui64 +#define mmu_tsb_raise_exception \ + mmu_kstat_data[MMU_CTX_TSB_RAISE_EXCEPTION].value.ui64 +#define mmu_wrap_around \ + mmu_kstat_data[MMU_CTX_WRAP_AROUND].value.ui64 + +extern uint_t max_mmu_ctxdoms; +extern mmu_ctx_t **mmu_ctxs_tbl; + +extern void sfmmu_cpu_init(cpu_t *); +extern void sfmmu_cpu_cleanup(cpu_t *); + +/* + * The following structure is used to get MMU context domain information for + * a CPU from the platform. + * + * mmu_idx + * The MMU context domain index within the global array mmu_ctxs + * mmu_nctxs + * The number of context IDs supported in the MMU context domain + * (64K for Rock) + */ +typedef struct mmu_ctx_info { + uint_t mmu_idx; + uint_t mmu_nctxs; +} mmu_ctx_info_t; + +#pragma weak plat_cpuid_to_mmu_ctx_info + +extern void plat_cpuid_to_mmu_ctx_info(processorid_t, mmu_ctx_info_t *); + +/* + * Each address space has an array of sfmmu_ctx_t structures, one structure + * per MMU context domain. + * + * cnum + * The context ID allocated for an address space on an MMU context domain + * gnum + * The generation number for the context ID in the MMU context domain. + * + * This structure needs to be a power-of-two in size. + */ +typedef struct sfmmu_ctx { + uint64_t gnum:48; + uint64_t cnum:16; +} sfmmu_ctx_t; + + +/* * The platform dependent hat structure. * tte counts should be protected by cas. * cpuset is protected by cas. @@ -281,16 +396,26 @@ struct hat { uchar_t sfmmu_rmstat; /* refmod stats refcnt */ uchar_t sfmmu_clrstart; /* start color bin for page coloring */ ushort_t sfmmu_clrbin; /* per as phys page coloring bin */ - short sfmmu_cnum; /* context number */ ushort_t sfmmu_flags; /* flags */ struct tsb_info *sfmmu_tsb; /* list of per as tsbs */ uint64_t sfmmu_ismblkpa; /* pa of sfmmu_iblkp, or -1 */ + lock_t sfmmu_ctx_lock; /* sync ctx alloc and invalidation */ kcondvar_t sfmmu_tsb_cv; /* signals TSB swapin or relocation */ uchar_t sfmmu_cext; /* context page size encoding */ uint8_t sfmmu_pgsz[MMU_PAGE_SIZES]; /* ranking for MMU */ #ifdef sun4v struct hv_tsb_block sfmmu_hvblock; #endif + /* + * sfmmu_ctxs is a variable length array of max_mmu_ctxdoms # of + * elements. max_mmu_ctxdoms is determined at run-time. + * sfmmu_ctxs[1] is just the fist element of an array, it always + * has to be the last field to ensure that the memory allocated + * for sfmmu_ctxs is consecutive with the memory of the rest of + * the hat data structure. + */ + sfmmu_ctx_t sfmmu_ctxs[1]; + }; #define sfmmu_iblk h_un.sfmmu_iblkp @@ -324,28 +449,6 @@ struct hat { #define FLUSH_NECESSARY_CPUS 0 #define FLUSH_ALL_CPUS 1 -/* - * Software context structure. The size of this structure is currently - * hardwired into the tsb miss handlers in assembly code through the - * CTX_SZ_SHIFT define. Since this define is used in a shift we should keep - * this structure a power of two. - * - * ctx_flags: - * Bit 0 : Free flag. - */ -struct ctx { - union _ctx_un { - sfmmu_t *ctx_sfmmup; /* back pointer to hat id */ - struct ctx *ctx_freep; /* next ctx in freelist */ - } ctx_un; - krwlock_t ctx_rwlock; /* protect context from stealer */ - uint32_t ctx_flags; /* flags */ - uint8_t pad[12]; -}; - -#define ctx_sfmmu ctx_un.ctx_sfmmup -#define ctx_free ctx_un.ctx_freep - #ifdef DEBUG /* * For debugging purpose only. Maybe removed later. @@ -744,6 +847,14 @@ struct hmehash_bucket { #endif /* !_ASM */ +/* Proc Count Project */ +#define SFMMU_PGCNT_MASK 0x3f +#define SFMMU_PGCNT_SHIFT 6 +#define INVALID_MMU_ID -1 +#define SFMMU_MMU_GNUM_RSHIFT 16 +#define SFMMU_MMU_CNUM_LSHIFT (64 - SFMMU_MMU_GNUM_RSHIFT) +#define MAX_SFMMU_CTX_VAL ((1 << 16) - 1) /* for sanity check */ +#define MAX_SFMMU_GNUM_VAL ((0x1UL << 48) - 1) /* * The tsb miss handlers written in assembly know that sfmmup @@ -874,10 +985,10 @@ struct hmehash_bucket { #define HME_HASH_FUNCTION(hatid, vaddr, shift) \ ((hatid != KHATID)? \ - (&uhme_hash[ (((uintptr_t)(hatid) ^ ((uintptr_t)vaddr >> (shift))) & \ - UHMEHASH_SZ) ]): \ - (&khme_hash[ (((uintptr_t)(hatid) ^ ((uintptr_t)vaddr >> (shift))) & \ - KHMEHASH_SZ) ])) + (&uhme_hash[ (((uintptr_t)(hatid) ^ \ + ((uintptr_t)vaddr >> (shift))) & UHMEHASH_SZ) ]): \ + (&khme_hash[ (((uintptr_t)(hatid) ^ \ + ((uintptr_t)vaddr >> (shift))) & KHMEHASH_SZ) ])) /* * This macro will traverse a hmeblk hash link list looking for an hme_blk @@ -953,9 +1064,9 @@ struct hmehash_bucket { #define SFMMU_HASH_LOCK_ISHELD(hmebp) \ (mutex_owned(&hmebp->hmehash_mutex)) -#define SFMMU_XCALL_STATS(ctxnum) \ +#define SFMMU_XCALL_STATS(sfmmup) \ { \ - if (ctxnum == KCONTEXT) { \ + if (sfmmup == ksfmmup) { \ SFMMU_STAT(sf_kernel_xcalls); \ } else { \ SFMMU_STAT(sf_user_xcalls); \ @@ -963,11 +1074,8 @@ struct hmehash_bucket { } #define astosfmmu(as) ((as)->a_hat) -#define sfmmutoctxnum(sfmmup) ((sfmmup)->sfmmu_cnum) -#define sfmmutoctx(sfmmup) (&ctxs[sfmmutoctxnum(sfmmup)]) #define hblktosfmmu(hmeblkp) ((sfmmu_t *)(hmeblkp)->hblk_tag.htag_id) #define sfmmutoas(sfmmup) ((sfmmup)->sfmmu_as) -#define ctxnumtoctx(ctxnum) (&ctxs[ctxnum]) /* * We use the sfmmu data structure to keep the per as page coloring info. */ @@ -1355,6 +1463,35 @@ extern uint_t tsb_slab_pamask; sethi %hi(0x1000000), reg /* + * Macro to get hat per-MMU cnum on this CPU. + * sfmmu - In, pass in "sfmmup" from the caller. + * cnum - Out, return 'cnum' to the caller + * scr - scratch + */ +#define SFMMU_CPU_CNUM(sfmmu, cnum, scr) \ + CPU_ADDR(scr, cnum); /* scr = load CPU struct addr */ \ + ld [scr + CPU_MMU_IDX], cnum; /* cnum = mmuid */ \ + add sfmmu, SFMMU_CTXS, scr; /* scr = sfmmup->sfmmu_ctxs[] */ \ + sllx cnum, SFMMU_MMU_CTX_SHIFT, cnum; \ + add scr, cnum, scr; /* scr = sfmmup->sfmmu_ctxs[id] */ \ + ldx [scr + SFMMU_MMU_GC_NUM], scr; /* sfmmu_ctxs[id].gcnum */ \ + sllx scr, SFMMU_MMU_CNUM_LSHIFT, scr; \ + srlx scr, SFMMU_MMU_CNUM_LSHIFT, cnum; /* cnum = sfmmu cnum */ + +/* + * Macro to get hat gnum & cnum assocaited with sfmmu_ctx[mmuid] entry + * entry - In, pass in (&sfmmu_ctxs[mmuid] - SFMMU_CTXS) from the caller. + * gnum - Out, return sfmmu gnum + * cnum - Out, return sfmmu cnum + * reg - scratch + */ +#define SFMMU_MMUID_GNUM_CNUM(entry, gnum, cnum, reg) \ + ldx [entry + SFMMU_CTXS], reg; /* reg = sfmmu (gnum | cnum) */ \ + srlx reg, SFMMU_MMU_GNUM_RSHIFT, gnum; /* gnum = sfmmu gnum */ \ + sllx reg, SFMMU_MMU_CNUM_LSHIFT, cnum; \ + srlx cnum, SFMMU_MMU_CNUM_LSHIFT, cnum; /* cnum = sfmmu cnum */ + +/* * Macro to get this CPU's tsbmiss area. */ #define CPU_TSBMISS_AREA(tsbmiss, tmp1) \ @@ -1408,6 +1545,65 @@ label3:; #endif +/* + * Macro to setup arguments with kernel sfmmup context + page size before + * calling sfmmu_setctx_sec() + */ +#ifdef sun4v +#define SET_KAS_CTXSEC_ARGS(sfmmup, arg0, arg1) \ + set KCONTEXT, arg0; \ + set 0, arg1; +#else +#define SET_KAS_CTXSEC_ARGS(sfmmup, arg0, arg1) \ + ldub [sfmmup + SFMMU_CEXT], arg1; \ + set KCONTEXT, arg0; \ + sll arg1, CTXREG_EXT_SHIFT, arg1; +#endif + +#define PANIC_IF_INTR_DISABLED_PSTR(pstatereg, label, scr) \ + andcc pstatereg, PSTATE_IE, %g0; /* panic if intrs */ \ +/*CSTYLED*/ \ + bnz,pt %icc, label; /* already disabled */ \ + nop; \ + \ + sethi %hi(panicstr), scr; \ + ldx [scr + %lo(panicstr)], scr; \ + tst scr; \ +/*CSTYLED*/ \ + bnz,pt %xcc, label; \ + nop; \ + \ + save %sp, -SA(MINFRAME), %sp; \ + sethi %hi(sfmmu_panic1), %o0; \ + call panic; \ + or %o0, %lo(sfmmu_panic1), %o0; \ +/*CSTYLED*/ \ +label: + +#define PANIC_IF_INTR_ENABLED_PSTR(label, scr) \ + /* \ + * The caller must have disabled interrupts. \ + * If interrupts are not disabled, panic \ + */ \ + rdpr %pstate, scr; \ + andcc scr, PSTATE_IE, %g0; \ +/*CSTYLED*/ \ + bz,pt %icc, label; \ + nop; \ + \ + sethi %hi(panicstr), scr; \ + ldx [scr + %lo(panicstr)], scr; \ + tst scr; \ +/*CSTYLED*/ \ + bnz,pt %xcc, label; \ + nop; \ + \ + sethi %hi(sfmmu_panic6), %o0; \ + call panic; \ + or %o0, %lo(sfmmu_panic6), %o0; \ +/*CSTYLED*/ \ +label: + #endif /* _ASM */ #ifndef _ASM @@ -1503,11 +1699,10 @@ extern struct tsbe * extern void sfmmu_load_tsbe(struct tsbe *, uint64_t, tte_t *, int); extern void sfmmu_unload_tsbe(struct tsbe *, uint64_t, int); extern void sfmmu_load_mmustate(sfmmu_t *); -extern void sfmmu_ctx_steal_tl1(uint64_t, uint64_t); extern void sfmmu_raise_tsb_exception(uint64_t, uint64_t); #ifndef sun4v -extern void sfmmu_itlb_ld(caddr_t, int, tte_t *); -extern void sfmmu_dtlb_ld(caddr_t, int, tte_t *); +extern void sfmmu_itlb_ld_kva(caddr_t, tte_t *); +extern void sfmmu_dtlb_ld_kva(caddr_t, tte_t *); #endif /* sun4v */ extern void sfmmu_copytte(tte_t *, tte_t *); extern int sfmmu_modifytte(tte_t *, tte_t *, tte_t *); @@ -1517,7 +1712,8 @@ extern void sfmmu_hblk_hash_rm(struct hmehash_bucket *, struct hme_blk *, uint64_t, struct hme_blk *); extern void sfmmu_hblk_hash_add(struct hmehash_bucket *, struct hme_blk *, uint64_t); - +extern uint_t sfmmu_disable_intrs(void); +extern void sfmmu_enable_intrs(uint_t); /* * functions exported to machine dependent VM code */ @@ -1549,7 +1745,7 @@ extern void sfmmu_cache_flushall(void); extern pgcnt_t sfmmu_tte_cnt(sfmmu_t *, uint_t); extern void *sfmmu_tsb_segkmem_alloc(vmem_t *, size_t, int); extern void sfmmu_tsb_segkmem_free(vmem_t *, void *, size_t); -extern void sfmmu_steal_context(sfmmu_t *, uint8_t *); +extern void sfmmu_reprog_pgsz_arr(sfmmu_t *, uint8_t *); extern void hat_kern_setup(void); extern int hat_page_relocate(page_t **, page_t **, spgcnt_t *); @@ -1557,6 +1753,7 @@ extern uint_t hat_preferred_pgsz(struct hat *, caddr_t, size_t, int); extern int sfmmu_get_ppvcolor(struct page *); extern int sfmmu_get_addrvcolor(caddr_t); extern int sfmmu_hat_lock_held(sfmmu_t *); +extern void sfmmu_alloc_ctx(sfmmu_t *, int, struct cpu *); /* * Functions exported to xhat_sfmmu.c @@ -1580,8 +1777,6 @@ extern uint_t mmu_preferred_pgsz(sfmmu_t *, caddr_t, size_t); extern void mmu_check_page_sizes(sfmmu_t *, uint64_t *); extern sfmmu_t *ksfmmup; -extern struct ctx *ctxs; -extern uint_t nctxs; extern caddr_t ktsb_base; extern uint64_t ktsb_pbase; extern int ktsb_sz; @@ -1700,10 +1895,6 @@ struct sfmmu_global_stat { int sf_swapout; /* # times hat swapped out */ - int sf_ctxfree; /* ctx alloc from free list */ - int sf_ctxdirty; /* ctx alloc from dirty list */ - int sf_ctxsteal; /* ctx allocated by steal */ - int sf_tsb_alloc; /* # TSB allocations */ int sf_tsb_allocfail; /* # times TSB alloc fail */ int sf_tsb_sectsb_create; /* # times second TSB added */ @@ -1756,10 +1947,7 @@ struct sfmmu_global_stat { int sf_user_vtop; /* # of user vatopfn calls */ - int sf_ctx_swap; /* # times switched MMU ctxs */ - int sf_tlbflush_all; /* # times flush all TLBs */ - int sf_tlbflush_ctx; /* # times flush TLB ctx */ - int sf_tlbflush_deferred; /* # times !flush ctx imm. */ + int sf_ctx_inv; /* #times invalidate MMU ctx */ int sf_tlb_reprog_pgsz; /* # times switch TLB pgsz */ }; @@ -1787,9 +1975,11 @@ struct sfmmu_percpu_stat { int sf_kmod_faults; /* # of mod (prot viol) flts */ }; -#define SFMMU_STAT(stat) sfmmu_global_stat.stat++; -#define SFMMU_STAT_ADD(stat, amount) sfmmu_global_stat.stat += amount; -#define SFMMU_STAT_SET(stat, count) sfmmu_global_stat.stat = count; +#define SFMMU_STAT(stat) sfmmu_global_stat.stat++ +#define SFMMU_STAT_ADD(stat, amount) sfmmu_global_stat.stat += (amount) +#define SFMMU_STAT_SET(stat, count) sfmmu_global_stat.stat = (count) + +#define SFMMU_MMU_STAT(stat) CPU->cpu_m.cpu_mmu_ctxp->stat++ #endif /* !_ASM */ diff --git a/usr/src/uts/sun4/cpu/cpu_module.c b/usr/src/uts/sun4/cpu/cpu_module.c index def445a3e4..ebb4b0ced4 100644 --- a/usr/src/uts/sun4/cpu/cpu_module.c +++ b/usr/src/uts/sun4/cpu/cpu_module.c @@ -2,9 +2,8 @@ * CDDL HEADER START * * The contents of this file are subject to the terms of the - * Common Development and Distribution License, Version 1.0 only - * (the "License"). You may not use this file except in compliance - * with the License. + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. @@ -20,7 +19,7 @@ * CDDL HEADER END */ /* - * Copyright 2005 Sun Microsystems, Inc. All rights reserved. + * Copyright 2006 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ @@ -45,7 +44,7 @@ uint_t nsec_shift; uint_t adj_shift; hrtime_t hrtime_base; int traptrace_use_stick; -uchar_t *ctx_pgsz_array; +uint_t cpu_impl_dual_pgsz; void cpu_setup(void) @@ -57,31 +56,25 @@ cpu_init_tick_freq(void) /*ARGSUSED*/ void -vtag_flushpage(caddr_t addr, uint_t ctx) -{} - -/*ARGSUSED*/ -void -vtag_flushctx(uint_t ctx) +vtag_flushpage(caddr_t addr, uint64_t sfmmup) {} void vtag_flushall(void) {} -/*ARGSUSED*/ void -vtag_flushpage_tl1(uint64_t addr, uint64_t ctx) +vtag_flushall_uctxs(void) {} /*ARGSUSED*/ void -vtag_flush_pgcnt_tl1(uint64_t addr, uint64_t ctx_pgcnt) +vtag_flushpage_tl1(uint64_t addr, uint64_t sfmmup) {} /*ARGSUSED*/ void -vtag_flushctx_tl1(uint64_t ctx, uint64_t dummy) +vtag_flush_pgcnt_tl1(uint64_t addr, uint64_t sfmmup_pgcnt) {} /*ARGSUSED*/ diff --git a/usr/src/uts/sun4/io/trapstat.c b/usr/src/uts/sun4/io/trapstat.c index 9aa25eca4f..fa7b98350b 100644 --- a/usr/src/uts/sun4/io/trapstat.c +++ b/usr/src/uts/sun4/io/trapstat.c @@ -577,12 +577,12 @@ trapstat_load_tlb(void) if (i < TSTAT_INSTR_PAGES) { tte.tte_intlo = TTE_PFN_INTLO(tcpu->tcpu_pfn[i]) | TTE_LCK_INT | TTE_CP_INT | TTE_PRIV_INT; - sfmmu_itlb_ld(va, KCONTEXT, &tte); + sfmmu_itlb_ld_kva(va, &tte); } else { tte.tte_intlo = TTE_PFN_INTLO(tcpu->tcpu_pfn[i]) | TTE_LCK_INT | TTE_CP_INT | TTE_CV_INT | TTE_PRIV_INT | TTE_HWWR_INT; - sfmmu_dtlb_ld(va, KCONTEXT, &tte); + sfmmu_dtlb_ld_kva(va, &tte); } } #else /* sun4v */ @@ -1559,7 +1559,8 @@ trapstat_teardown(processorid_t cpu) vmem_free(tstat_arena, tcpu->tcpu_data, tstat_data_size); for (i = 0; i < tstat_total_pages; i++, va += MMU_PAGESIZE) { - xt_one(cpu, vtag_flushpage_tl1, (uint64_t)va, KCONTEXT); + xt_one(cpu, vtag_flushpage_tl1, (uint64_t)va, + (uint64_t)ksfmmup); } #else xt_one(cpu, vtag_unmap_perm_tl1, (uint64_t)va, KCONTEXT); diff --git a/usr/src/uts/sun4/ml/offsets.in b/usr/src/uts/sun4/ml/offsets.in index 8c97b977f0..807cb1c845 100644 --- a/usr/src/uts/sun4/ml/offsets.in +++ b/usr/src/uts/sun4/ml/offsets.in @@ -1,13 +1,12 @@ \ offsets.in: input file to produce assym.h using the stabs program -\ Copyright 2005 Sun Microsystems, Inc. All rights reserved. +\ Copyright 2006 Sun Microsystems, Inc. All rights reserved. \ Use is subject to license terms. \ \ CDDL HEADER START \ \ The contents of this file are subject to the terms of the -\ Common Development and Distribution License, Version 1.0 only -\ (the "License"). You may not use this file except in compliance -\ with the License. +\ Common Development and Distribution License (the "License"). +\ You may not use this file except in compliance with the License. \ \ You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE \ or http://www.opensolaris.org/os/licensing. @@ -226,6 +225,14 @@ hme_blk HMEBLK_SIZE hblk_hme HMEBLK_HME1 hblk_nextpa HMEBLK_NEXTPA +mmu_ctx MMU_CTX_SIZE + mmu_gnum MMU_CTX_GNUM + mmu_cnum MMU_CTX_CNUM + mmu_nctxs MMU_CTX_NCTXS + +sfmmu_ctx SFMMU_MMU_CTX_SIZE SFMMU_MMU_CTX_SHIFT + gnum SFMMU_MMU_GC_NUM + user USIZEBYTES u_comm u_signal @@ -366,6 +373,8 @@ cpu CPUSIZE cpu_m.tmp2 CPU_TMP2 cpu_m.mpcb CPU_MPCB cpu_m.cpu_private CPU_PRIVATE + cpu_m.cpu_mmu_idx CPU_MMU_IDX + cpu_m.cpu_mmu_ctxp CPU_MMU_CTXP cpu_m.ptl1_state CPU_PTL1 cpu_core_t CPU_CORE_SIZE CPU_CORE_SHIFT diff --git a/usr/src/uts/sun4/ml/swtch.s b/usr/src/uts/sun4/ml/swtch.s index 23d9cab385..98ec1edd55 100644 --- a/usr/src/uts/sun4/ml/swtch.s +++ b/usr/src/uts/sun4/ml/swtch.s @@ -2,9 +2,8 @@ * CDDL HEADER START * * The contents of this file are subject to the terms of the - * Common Development and Distribution License, Version 1.0 only - * (the "License"). You may not use this file except in compliance - * with the License. + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. @@ -20,7 +19,7 @@ * CDDL HEADER END */ /* - * Copyright 2005 Sun Microsystems, Inc. All rights reserved. + * Copyright 2006 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ @@ -185,7 +184,6 @@ resume(kthread_id_t t) ! ! IMPORTANT: Registers at this point must be: ! %i0 = new thread - ! %i1 = flag (non-zero if unpinning from an interrupt thread) ! %i1 = cpu pointer ! %i2 = old proc pointer ! %i3 = new proc pointer @@ -200,21 +198,21 @@ resume(kthread_id_t t) cmp %i2, %i3 ! resuming the same process? be,pt %xcc, 5f ! yes. nop + ldx [%i3 + P_AS], %o0 ! load p->p_as - ldx [%o0 + A_HAT], %o3 ! load (p->p_as)->a_hat - ! %o3 is live until the call to sfmmu_setctx_sec below + ldx [%o0 + A_HAT], %i5 ! %i5 = new proc hat ! ! update cpusran field ! ld [%i1 + CPU_ID], %o4 - add %o3, SFMMU_CPUSRAN, %o5 + add %i5, SFMMU_CPUSRAN, %o5 CPU_INDEXTOSET(%o5, %o4, %g1) - ldx [%o5], %o2 ! o2 = cpusran field + ldx [%o5], %o2 ! %o2 = cpusran field mov 1, %g2 - sllx %g2, %o4, %o4 ! o4 = bit for this cpu + sllx %g2, %o4, %o4 ! %o4 = bit for this cpu andcc %o4, %o2, %g0 - bnz,pn %xcc, 4f + bnz,pn %xcc, 0f ! bit already set, go to 0 nop 3: or %o2, %o4, %o1 ! or in this cpu's bit mask @@ -224,20 +222,52 @@ resume(kthread_id_t t) ldx [%o5], %o2 ! o2 = cpusran field membar #LoadLoad|#StoreLoad +0: + ! + ! disable interrupts ! - ! Switch to different address space. + ! if resume from user to kernel thread + ! call sfmmu_setctx_sec + ! if resume from kernel (or a different user) thread to user thread + ! call sfmmu_alloc_ctx + ! sfmmu_load_mmustate ! -4: + ! enable interrupts + ! + ! %i5 = new proc hat + ! + + sethi %hi(ksfmmup), %o2 + ldx [%o2 + %lo(ksfmmup)], %o2 + rdpr %pstate, %i4 - wrpr %i4, PSTATE_IE, %pstate ! disable interrupts + cmp %i5, %o2 ! new proc hat == ksfmmup ? + bne,pt %xcc, 3f ! new proc is not kernel as, go to 3 + wrpr %i4, PSTATE_IE, %pstate - call sfmmu_setctx_sec ! switch to other ctx (maybe 0) - lduh [%o3 + SFMMU_CNUM], %o0 - call sfmmu_load_mmustate ! program MMU registers - mov %o3, %o0 + SET_KAS_CTXSEC_ARGS(%i5, %o0, %o1) - wrpr %g0, %i4, %pstate ! enable interrupts + ! new proc is kernel as + + call sfmmu_setctx_sec ! switch to kernel context + or %o0, %o1, %o0 + + ba,a,pt %icc, 4f + + ! + ! Switch to user address space. + ! +3: + mov %i5, %o0 ! %o0 = sfmmup + mov %i1, %o2 ! %o2 = CPU + call sfmmu_alloc_ctx + mov %g0, %o1 ! %o1 = allocate flag = 0 +4: + call sfmmu_load_mmustate ! program MMU registers + mov %i5, %o0 + wrpr %g0, %i4, %pstate ! enable interrupts + 5: ! ! spin until dispatched thread's mutex has diff --git a/usr/src/uts/sun4/os/mp_startup.c b/usr/src/uts/sun4/os/mp_startup.c index 0139e6a5f0..1effc88141 100644 --- a/usr/src/uts/sun4/os/mp_startup.c +++ b/usr/src/uts/sun4/os/mp_startup.c @@ -431,6 +431,12 @@ setup_cpu_common(int cpuid) chip_cpu_init(cp); cpu_intrq_setup(cp); + + /* + * Initialize MMU context domain information. + */ + sfmmu_cpu_init(cp); + } /* @@ -522,6 +528,7 @@ cleanup_cpu_common(int cpuid) */ disp_cpu_fini(cp); cpu_pa[cpuid] = 0; + sfmmu_cpu_cleanup(cp); bzero(cp, sizeof (*cp)); /* diff --git a/usr/src/uts/sun4/os/startup.c b/usr/src/uts/sun4/os/startup.c index 3a0506263c..7e38605d52 100644 --- a/usr/src/uts/sun4/os/startup.c +++ b/usr/src/uts/sun4/os/startup.c @@ -64,6 +64,7 @@ #include <sys/traptrace.h> #include <sys/memnode.h> #include <sys/mem_cage.h> +#include <sys/mmu.h> extern void setup_trap_table(void); extern void cpu_intrq_setup(struct cpu *); @@ -2684,10 +2685,8 @@ char obp_tte_str[] = "h# %p constant khme_hash " "h# %x constant UHMEHASH_SZ " "h# %x constant KHMEHASH_SZ " + "h# %p constant KCONTEXT " "h# %p constant KHATID " - "h# %x constant CTX_SIZE " - "h# %x constant CTX_SFMMU " - "h# %p constant ctxs " "h# %x constant ASI_MEM " ": PHYS-X@ ( phys -- data ) " @@ -2754,11 +2753,6 @@ char obp_tte_str[] = " until r> drop " "; " - ": CNUM_TO_SFMMUP ( cnum -- sfmmup ) " - " CTX_SIZE * ctxs + CTX_SFMMU + " - "x@ " - "; " - ": HME_HASH_TAG ( sfmmup rehash addr -- hblktag ) " " over HME_HASH_SHIFT HME_HASH_BSPAGE ( sfmmup rehash bspage ) " " HTAG_REHASHSZ << or nip ( hblktag ) " @@ -2776,7 +2770,12 @@ char obp_tte_str[] = "; " ": unix-tte ( addr cnum -- false | tte-data true ) " - " CNUM_TO_SFMMUP ( addr sfmmup ) " + " KCONTEXT = if ( addr ) " + " KHATID ( addr khatid ) " + " else ( addr ) " + " drop false exit ( false ) " + " then " + " ( addr khatid ) " " mmu_hashcnt 1+ 1 do ( addr sfmmup ) " " 2dup swap i HME_HASH_SHIFT " "( addr sfmmup sfmmup addr hmeshift ) " @@ -2835,10 +2834,8 @@ create_va_to_tte(void) (caddr_t)va_to_pa((caddr_t)khme_hash), UHMEHASH_SZ, KHMEHASH_SZ, + KCONTEXT, KHATID, - sizeof (struct ctx), - OFFSET(struct ctx, ctx_sfmmu), - ctxs, ASI_MEM); prom_interpret(bp, 0, 0, 0, 0, 0); diff --git a/usr/src/uts/sun4/vm/sfmmu.c b/usr/src/uts/sun4/vm/sfmmu.c index 4b637f2a0b..ac6a421b2d 100644 --- a/usr/src/uts/sun4/vm/sfmmu.c +++ b/usr/src/uts/sun4/vm/sfmmu.c @@ -2,9 +2,8 @@ * CDDL HEADER START * * The contents of this file are subject to the terms of the - * Common Development and Distribution License, Version 1.0 only - * (the "License"). You may not use this file except in compliance - * with the License. + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. @@ -20,7 +19,7 @@ * CDDL HEADER END */ /* - * Copyright 2005 Sun Microsystems, Inc. All rights reserved. + * Copyright 2006 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ @@ -773,7 +772,6 @@ ndata_alloc_tsbs(struct memlist *ndata, pgcnt_t npages) int ndata_alloc_hat(struct memlist *ndata, pgcnt_t npages, pgcnt_t kpm_npages) { - size_t ctx_sz; size_t mml_alloc_sz; size_t cb_alloc_sz; int max_nucuhme_buckets = MAX_NUCUHME_BUCKETS; @@ -790,20 +788,6 @@ ndata_alloc_hat(struct memlist *ndata, pgcnt_t npages, pgcnt_t kpm_npages) } /* - * Allocate ctx structures - * - * based on v_proc to calculate how many ctx structures - * is not possible; - * use whatever module_setup() assigned to nctxs - */ - PRM_DEBUG(nctxs); - ctx_sz = nctxs * sizeof (struct ctx); - if ((ctxs = ndata_alloc(ndata, ctx_sz, sizeof (struct ctx))) == NULL) - return (-1); - - PRM_DEBUG(ctxs); - - /* * The number of buckets in the hme hash tables * is a power of 2 such that the average hash chain length is * HMENT_HASHAVELEN. The number of buckets for the user hash is diff --git a/usr/src/uts/sun4u/cpu/opl_olympus.c b/usr/src/uts/sun4u/cpu/opl_olympus.c index 331758a065..f68c133740 100644 --- a/usr/src/uts/sun4u/cpu/opl_olympus.c +++ b/usr/src/uts/sun4u/cpu/opl_olympus.c @@ -77,14 +77,7 @@ static int cpu_flt_in_memory(opl_async_flt_t *, uint64_t); */ static int opl_async_check_interval = 60; /* 1 min */ -/* - * Maximum number of contexts for Olympus-C. - */ -#define MAX_NCTXS (1 << 13) - -/* Will be set !NULL for SPARC64-VI and derivatives. */ -static uchar_t ctx_pgsz_arr[MAX_NCTXS]; -uchar_t *ctx_pgsz_array = ctx_pgsz_arr; +uint_t cpu_impl_dual_pgsz = 1; /* * PA[22:0] represent Displacement in Jupiter @@ -529,16 +522,6 @@ cpu_setup(void) at_flags = EF_SPARC_32PLUS | EF_SPARC_SUN_US1 | EF_SPARC_SUN_US3; /* - * Use the maximum number of contexts available for SPARC64-VI - * unless it has been tuned for debugging. - * We are checking against 0 here since this value can be patched - * while booting. It can not be patched via /etc/system since it - * will be patched too late and thus cause the system to panic. - */ - if (nctxs == 0) - nctxs = MAX_NCTXS; - - /* * Due to the number of entries in the fully-associative tlb * this may have to be tuned lower than in spitfire. */ @@ -957,9 +940,18 @@ mmu_set_ctx_page_sizes(struct hat *hat) ASSERT(pgsz1 < mmu_page_sizes); new_cext = TAGACCEXT_MKSZPAIR(pgsz1, pgsz0); if (hat->sfmmu_cext != new_cext) { +#ifdef DEBUG + int i; + /* + * assert cnum should be invalid, this is because pagesize + * can only be changed after a proc's ctxs are invalidated. + */ + for (i = 0; i < max_mmu_ctxdoms; i++) { + ASSERT(hat->sfmmu_ctxs[i].cnum == INVALID_CONTEXT); + } +#endif /* DEBUG */ hat->sfmmu_cext = new_cext; } - ctx_pgsz_array[hat->sfmmu_cnum] = hat->sfmmu_cext; /* * sfmmu_setctx_sec() will take care of the * rest of the dirty work for us. diff --git a/usr/src/uts/sun4u/cpu/opl_olympus_asm.s b/usr/src/uts/sun4u/cpu/opl_olympus_asm.s index 44e4983586..f90e2dea01 100644 --- a/usr/src/uts/sun4u/cpu/opl_olympus_asm.s +++ b/usr/src/uts/sun4u/cpu/opl_olympus_asm.s @@ -73,7 +73,7 @@ /* ARGSUSED */ void -vtag_flushpage(caddr_t vaddr, u_int ctxnum) +vtag_flushpage(caddr_t vaddr, uint64_t sfmmup) {} #else /* lint */ @@ -83,20 +83,11 @@ vtag_flushpage(caddr_t vaddr, u_int ctxnum) * flush page from the tlb * * %o0 = vaddr - * %o1 = ctxnum + * %o1 = sfmmup */ rdpr %pstate, %o5 #ifdef DEBUG - andcc %o5, PSTATE_IE, %g0 /* if interrupts already */ - bnz,a,pt %icc, 3f /* disabled, panic */ - nop - save %sp, -SA(MINFRAME), %sp - sethi %hi(sfmmu_panic1), %o0 - call panic - or %o0, %lo(sfmmu_panic1), %o0 - ret - restore -3: + PANIC_IF_INTR_DISABLED_PSTR(%o5, opl_di_l3, %g1) #endif /* DEBUG */ /* * disable ints @@ -109,41 +100,45 @@ vtag_flushpage(caddr_t vaddr, u_int ctxnum) * Interrupts are disabled to prevent the primary ctx register * from changing underneath us. */ - brnz,pt %o1, 1f /* KCONTEXT? */ - sethi %hi(FLUSH_ADDR), %o3 + sethi %hi(ksfmmup), %o3 + ldx [%o3 + %lo(ksfmmup)], %o3 + cmp %o3, %o1 + bne,pt %xcc, 1f ! if not kernel as, go to 1 + sethi %hi(FLUSH_ADDR), %o3 /* - * For KCONTEXT demaps use primary. type = page implicitly + * For Kernel demaps use primary. type = page implicitly */ stxa %g0, [%o0]ASI_DTLB_DEMAP /* dmmu flush for KCONTEXT */ stxa %g0, [%o0]ASI_ITLB_DEMAP /* immu flush for KCONTEXT */ flush %o3 - b 5f - nop + retl + wrpr %g0, %o5, %pstate /* enable interrupts */ 1: /* * User demap. We need to set the primary context properly. * Secondary context cannot be used for SPARC64-VI IMMU. * %o0 = vaddr - * %o1 = ctxnum + * %o1 = sfmmup * %o3 = FLUSH_ADDR */ - sethi %hi(ctx_pgsz_array), %o4 - ldn [%o4 + %lo(ctx_pgsz_array)], %o4 - ldub [%o4 + %o1], %o4 + SFMMU_CPU_CNUM(%o1, %g1, %g2) ! %g1 = sfmmu cnum on this CPU + + ldub [%o1 + SFMMU_CEXT], %o4 ! %o4 = sfmmup->sfmmu_cext sll %o4, CTXREG_EXT_SHIFT, %o4 - or %o1, %o4, %o1 + or %g1, %o4, %g1 ! %g1 = pgsz | cnum + wrpr %g0, 1, %tl set MMU_PCONTEXT, %o4 or DEMAP_PRIMARY | DEMAP_PAGE_TYPE, %o0, %o0 - ldxa [%o4]ASI_DMMU, %o2 /* rd old ctxnum */ - stxa %o1, [%o4]ASI_DMMU /* wr new ctxum */ -4: + ldxa [%o4]ASI_DMMU, %o2 ! %o2 = save old ctxnum + stxa %g1, [%o4]ASI_DMMU ! wr new ctxum + stxa %g0, [%o0]ASI_DTLB_DEMAP stxa %g0, [%o0]ASI_ITLB_DEMAP stxa %o2, [%o4]ASI_DMMU /* restore old ctxnum */ flush %o3 wrpr %g0, 0, %tl -5: + retl wrpr %g0, %o5, %pstate /* enable interrupts */ SET_SIZE(vtag_flushpage) @@ -153,61 +148,6 @@ vtag_flushpage(caddr_t vaddr, u_int ctxnum) #if defined(lint) -/* ARGSUSED */ -void -vtag_flushctx(u_int ctxnum) -{} - -#else /* lint */ - - ENTRY_NP(vtag_flushctx) - /* - * flush context from the tlb - * - * %o0 = ctxnum - * We disable interrupts to prevent the primary ctx register changing - * underneath us. - */ - sethi %hi(FLUSH_ADDR), %o3 - rdpr %pstate, %o2 - -#ifdef DEBUG - andcc %o2, PSTATE_IE, %g0 /* if interrupts already */ - bnz,a,pt %icc, 1f /* disabled, panic */ - nop - sethi %hi(sfmmu_panic1), %o0 - call panic - or %o0, %lo(sfmmu_panic1), %o0 -1: -#endif /* DEBUG */ - - sethi %hi(ctx_pgsz_array), %o4 - ldn [%o4 + %lo(ctx_pgsz_array)], %o4 - ldub [%o4 + %o0], %o4 - sll %o4, CTXREG_EXT_SHIFT, %o4 - or %o0, %o4, %o0 - wrpr %o2, PSTATE_IE, %pstate /* disable interrupts */ - set MMU_PCONTEXT, %o4 - set DEMAP_CTX_TYPE | DEMAP_PRIMARY, %g1 - wrpr %g0, 1, %tl - ldxa [%o4]ASI_DMMU, %o5 /* rd old ctxnum */ - stxa %o0, [%o4]ASI_DMMU /* wr new ctxum */ -4: - stxa %g0, [%g1]ASI_DTLB_DEMAP - stxa %g0, [%g1]ASI_ITLB_DEMAP - stxa %o5, [%o4]ASI_DMMU /* restore old ctxnum */ - flush %o3 - wrpr %g0, 0, %tl -5: - retl - wrpr %g0, %o2, %pstate /* enable interrupts */ - SET_SIZE(vtag_flushctx) - -#endif /* lint */ - - -#if defined(lint) - void vtag_flushall(void) {} @@ -235,7 +175,7 @@ vtag_flushall(void) /* ARGSUSED */ void -vtag_flushpage_tl1(uint64_t vaddr, uint64_t ctxnum) +vtag_flushpage_tl1(uint64_t vaddr, uint64_t sfmmup) {} #else /* lint */ @@ -245,13 +185,17 @@ vtag_flushpage_tl1(uint64_t vaddr, uint64_t ctxnum) * x-trap to flush page from tlb and tsb * * %g1 = vaddr, zero-extended on 32-bit kernel - * %g2 = ctxnum + * %g2 = sfmmup * * assumes TSBE_TAG = 0 */ srln %g1, MMU_PAGESHIFT, %g1 - brnz,pt %g2, 1f /* KCONTEXT */ - slln %g1, MMU_PAGESHIFT, %g1 /* g1 = vaddr */ + + sethi %hi(ksfmmup), %g3 + ldx [%g3 + %lo(ksfmmup)], %g3 + cmp %g3, %g2 + bne,pt %xcc, 1f ! if not kernel as, go to 1 + slln %g1, MMU_PAGESHIFT, %g1 /* g1 = vaddr */ /* We need to demap in the kernel context */ or DEMAP_NUCLEUS | DEMAP_PAGE_TYPE, %g1, %g1 @@ -261,15 +205,16 @@ vtag_flushpage_tl1(uint64_t vaddr, uint64_t ctxnum) 1: /* We need to demap in a user context */ or DEMAP_PRIMARY | DEMAP_PAGE_TYPE, %g1, %g1 - sethi %hi(ctx_pgsz_array), %g4 - ldn [%g4 + %lo(ctx_pgsz_array)], %g4 - ldub [%g4 + %g2], %g4 + + SFMMU_CPU_CNUM(%g2, %g6, %g3) ! %g6 = sfmmu cnum on this CPU + + ldub [%g2 + SFMMU_CEXT], %g4 ! %g4 = sfmmup->cext sll %g4, CTXREG_EXT_SHIFT, %g4 - or %g2, %g4, %g2 + or %g6, %g4, %g6 ! %g6 = pgsz | cnum set MMU_PCONTEXT, %g4 ldxa [%g4]ASI_DMMU, %g5 /* rd old ctxnum */ - stxa %g2, [%g4]ASI_DMMU /* wr new ctxum */ + stxa %g6, [%g4]ASI_DMMU /* wr new ctxum */ stxa %g0, [%g1]ASI_DTLB_DEMAP stxa %g0, [%g1]ASI_ITLB_DEMAP stxa %g5, [%g4]ASI_DMMU /* restore old ctxnum */ @@ -278,11 +223,12 @@ vtag_flushpage_tl1(uint64_t vaddr, uint64_t ctxnum) #endif /* lint */ + #if defined(lint) /* ARGSUSED */ void -vtag_flush_pgcnt_tl1(uint64_t vaddr, uint64_t ctx_pgcnt) +vtag_flush_pgcnt_tl1(uint64_t vaddr, uint64_t sfmmup_pgcnt) {} #else /* lint */ @@ -292,7 +238,7 @@ vtag_flush_pgcnt_tl1(uint64_t vaddr, uint64_t ctx_pgcnt) * x-trap to flush pgcnt MMU_PAGESIZE pages from tlb * * %g1 = vaddr, zero-extended on 32-bit kernel - * %g2 = <zero32|ctx16|pgcnt16> + * %g2 = <sfmmup58|pgcnt6> * * NOTE: this handler relies on the fact that no * interrupts or traps can occur during the loop @@ -302,45 +248,63 @@ vtag_flush_pgcnt_tl1(uint64_t vaddr, uint64_t ctx_pgcnt) * * assumes TSBE_TAG = 0 */ - set 0xffff, %g4 - and %g4, %g2, %g3 /* g3 = pgcnt */ - srln %g2, 16, %g2 /* g2 = ctxnum */ + set SFMMU_PGCNT_MASK, %g4 + and %g4, %g2, %g3 /* g3 = pgcnt - 1 */ + add %g3, 1, %g3 /* g3 = pgcnt */ + + andn %g2, SFMMU_PGCNT_MASK, %g2 /* g2 = sfmmup */ srln %g1, MMU_PAGESHIFT, %g1 - brnz,pt %g2, 1f /* KCONTEXT? */ + + sethi %hi(ksfmmup), %g4 + ldx [%g4 + %lo(ksfmmup)], %g4 + cmp %g4, %g2 + bne,pn %xcc, 1f /* if not kernel as, go to 1 */ slln %g1, MMU_PAGESHIFT, %g1 /* g1 = vaddr */ /* We need to demap in the kernel context */ or DEMAP_NUCLEUS | DEMAP_PAGE_TYPE, %g1, %g1 set MMU_PAGESIZE, %g2 /* g2 = pgsize */ + sethi %hi(FLUSH_ADDR), %g5 4: stxa %g0, [%g1]ASI_DTLB_DEMAP stxa %g0, [%g1]ASI_ITLB_DEMAP + flush %g5 ! flush required by immu + deccc %g3 /* decr pgcnt */ bnz,pt %icc,4b add %g1, %g2, %g1 /* next page */ retry 1: - /* We need to demap in a user context */ - sethi %hi(ctx_pgsz_array), %g4 - ldn [%g4 + %lo(ctx_pgsz_array)], %g4 + /* + * We need to demap in a user context + * + * g2 = sfmmup + * g3 = pgcnt + */ + SFMMU_CPU_CNUM(%g2, %g5, %g6) ! %g5 = sfmmu cnum on this CPU + or DEMAP_PRIMARY | DEMAP_PAGE_TYPE, %g1, %g1 - ldub [%g4 + %g2], %g4 + + ldub [%g2 + SFMMU_CEXT], %g4 ! %g4 = sfmmup->cext sll %g4, CTXREG_EXT_SHIFT, %g4 - or %g2, %g4, %g2 + or %g5, %g4, %g5 set MMU_PCONTEXT, %g4 - ldxa [%g4]ASI_DMMU, %g5 /* rd old ctxnum */ - stxa %g2, [%g4]ASI_DMMU /* wr new ctxum */ + ldxa [%g4]ASI_DMMU, %g6 /* rd old ctxnum */ + stxa %g5, [%g4]ASI_DMMU /* wr new ctxum */ set MMU_PAGESIZE, %g2 /* g2 = pgsize */ + sethi %hi(FLUSH_ADDR), %g5 3: stxa %g0, [%g1]ASI_DTLB_DEMAP stxa %g0, [%g1]ASI_ITLB_DEMAP + flush %g5 ! flush required by immu + deccc %g3 /* decr pgcnt */ bnz,pt %icc,3b add %g1, %g2, %g1 /* next page */ - stxa %g5, [%g4]ASI_DMMU /* restore old ctxnum */ + stxa %g6, [%g4]ASI_DMMU /* restore old ctxnum */ retry SET_SIZE(vtag_flush_pgcnt_tl1) @@ -349,39 +313,6 @@ vtag_flush_pgcnt_tl1(uint64_t vaddr, uint64_t ctx_pgcnt) #if defined(lint) -/* ARGSUSED */ -void -vtag_flushctx_tl1(uint64_t ctxnum, uint64_t dummy) -{} - -#else /* lint */ - - ENTRY_NP(vtag_flushctx_tl1) - /* - * x-trap to flush context from tlb - * - * %g1 = ctxnum - */ - sethi %hi(ctx_pgsz_array), %g4 - ldn [%g4 + %lo(ctx_pgsz_array)], %g4 - ldub [%g4 + %g1], %g4 - sll %g4, CTXREG_EXT_SHIFT, %g4 - or %g1, %g4, %g1 - set DEMAP_CTX_TYPE | DEMAP_PRIMARY, %g4 - set MMU_PCONTEXT, %g3 - ldxa [%g3]ASI_DMMU, %g5 /* rd old ctxnum */ - stxa %g1, [%g3]ASI_DMMU /* wr new ctxum */ - stxa %g0, [%g4]ASI_DTLB_DEMAP - stxa %g0, [%g4]ASI_ITLB_DEMAP - stxa %g5, [%g3]ASI_DMMU /* restore old ctxnum */ - retry - SET_SIZE(vtag_flushctx_tl1) - -#endif /* lint */ - - -#if defined(lint) - /*ARGSUSED*/ void vtag_flushall_tl1(uint64_t dummy1, uint64_t dummy2) diff --git a/usr/src/uts/sun4u/cpu/spitfire.c b/usr/src/uts/sun4u/cpu/spitfire.c index 7b2c100006..8399c21b75 100644 --- a/usr/src/uts/sun4u/cpu/spitfire.c +++ b/usr/src/uts/sun4u/cpu/spitfire.c @@ -2,9 +2,8 @@ * CDDL HEADER START * * The contents of this file are subject to the terms of the - * Common Development and Distribution License, Version 1.0 only - * (the "License"). You may not use this file except in compliance - * with the License. + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. @@ -20,7 +19,7 @@ * CDDL HEADER END */ /* - * Copyright 2005 Sun Microsystems, Inc. All rights reserved. + * Copyright 2006 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ @@ -63,7 +62,7 @@ #include <sys/dtrace.h> #include <sys/errclassify.h> -uchar_t *ctx_pgsz_array = NULL; +uint_t cpu_impl_dual_pgsz = 0; /* * Structure for the 8 byte ecache data dump and the associated AFSR state. @@ -407,11 +406,6 @@ ecc_psynd_score(ushort_t p_synd) #define UDB_FMTSTR "\020\012UE\011CE" /* - * Maximum number of contexts for Spitfire. - */ -#define MAX_NCTXS (1 << 13) - -/* * Save the cache bootup state for use when internal * caches are to be re-enabled after an error occurs. */ @@ -457,16 +451,6 @@ cpu_setup(void) */ cache_boot_state = get_lsu() & (LSU_IC | LSU_DC); - /* - * Use the maximum number of contexts available for Spitfire unless - * it has been tuned for debugging. - * We are checking against 0 here since this value can be patched - * while booting. It can not be patched via /etc/system since it - * will be patched too late and thus cause the system to panic. - */ - if (nctxs == 0) - nctxs = MAX_NCTXS; - if (use_page_coloring) { do_pg_coloring = 1; if (use_virtual_coloring) diff --git a/usr/src/uts/sun4u/cpu/spitfire_asm.s b/usr/src/uts/sun4u/cpu/spitfire_asm.s index 62b8e9bd13..13182ab509 100644 --- a/usr/src/uts/sun4u/cpu/spitfire_asm.s +++ b/usr/src/uts/sun4u/cpu/spitfire_asm.s @@ -2,9 +2,8 @@ * CDDL HEADER START * * The contents of this file are subject to the terms of the - * Common Development and Distribution License, Version 1.0 only - * (the "License"). You may not use this file except in compliance - * with the License. + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. @@ -195,6 +194,119 @@ sub tmp, linesize, tmp; \ 1: +#ifdef SF_ERRATA_32 +#define SF_WORKAROUND(tmp1, tmp2) \ + sethi %hi(FLUSH_ADDR), tmp2 ;\ + set MMU_PCONTEXT, tmp1 ;\ + stxa %g0, [tmp1]ASI_DMMU ;\ + flush tmp2 ; +#else +#define SF_WORKAROUND(tmp1, tmp2) +#endif /* SF_ERRATA_32 */ + +/* + * arg1 = vaddr + * arg2 = ctxnum + * - disable interrupts and clear address mask + * to access 64 bit physaddr + * - Blow out the TLB, flush user page. + * . use secondary context. + */ +#define VTAG_FLUSHUPAGE(lbl, arg1, arg2, tmp1, tmp2, tmp3, tmp4) \ + rdpr %pstate, tmp1 ;\ + andn tmp1, PSTATE_IE, tmp2 ;\ + wrpr tmp2, 0, %pstate ;\ + sethi %hi(FLUSH_ADDR), tmp2 ;\ + set MMU_SCONTEXT, tmp3 ;\ + ldxa [tmp3]ASI_DMMU, tmp4 ;\ + or DEMAP_SECOND | DEMAP_PAGE_TYPE, arg1, arg1 ;\ + cmp tmp4, arg2 ;\ + be,a,pt %icc, lbl/**/4 ;\ + nop ;\ + stxa arg2, [tmp3]ASI_DMMU ;\ +lbl/**/4: ;\ + stxa %g0, [arg1]ASI_DTLB_DEMAP ;\ + stxa %g0, [arg1]ASI_ITLB_DEMAP ;\ + flush tmp2 ;\ + be,a,pt %icc, lbl/**/5 ;\ + nop ;\ + stxa tmp4, [tmp3]ASI_DMMU ;\ + flush tmp2 ;\ +lbl/**/5: ;\ + wrpr %g0, tmp1, %pstate + + +/* + * macro that flushes all the user entries in dtlb + * arg1 = dtlb entries + * - Before first compare: + * tmp4 = tte + * tmp5 = vaddr + * tmp6 = cntxnum + */ +#define DTLB_FLUSH_UNLOCKED_UCTXS(lbl, arg1, tmp1, tmp2, tmp3, \ + tmp4, tmp5, tmp6) \ +lbl/**/0: ;\ + sllx arg1, 3, tmp3 ;\ + SF_WORKAROUND(tmp1, tmp2) ;\ + ldxa [tmp3]ASI_DTLB_ACCESS, tmp4 ;\ + srlx tmp4, 6, tmp4 ;\ + andcc tmp4, 1, %g0 ;\ + bnz,pn %xcc, lbl/**/1 ;\ + srlx tmp4, 57, tmp4 ;\ + andcc tmp4, 1, %g0 ;\ + beq,pn %xcc, lbl/**/1 ;\ + nop ;\ + set TAGREAD_CTX_MASK, tmp1 ;\ + ldxa [tmp3]ASI_DTLB_TAGREAD, tmp2 ;\ + and tmp2, tmp1, tmp6 ;\ + andn tmp2, tmp1, tmp5 ;\ + set KCONTEXT, tmp4 ;\ + cmp tmp6, tmp4 ;\ + be lbl/**/1 ;\ + nop ;\ + VTAG_FLUSHUPAGE(VD/**/lbl, tmp5, tmp6, tmp1, tmp2, tmp3, tmp4) ;\ +lbl/**/1: ;\ + brgz,pt arg1, lbl/**/0 ;\ + sub arg1, 1, arg1 + + +/* + * macro that flushes all the user entries in itlb + * arg1 = itlb entries + * - Before first compare: + * tmp4 = tte + * tmp5 = vaddr + * tmp6 = cntxnum + */ +#define ITLB_FLUSH_UNLOCKED_UCTXS(lbl, arg1, tmp1, tmp2, tmp3, \ + tmp4, tmp5, tmp6) \ +lbl/**/0: ;\ + sllx arg1, 3, tmp3 ;\ + SF_WORKAROUND(tmp1, tmp2) ;\ + ldxa [tmp3]ASI_ITLB_ACCESS, tmp4 ;\ + srlx tmp4, 6, tmp4 ;\ + andcc tmp4, 1, %g0 ;\ + bnz,pn %xcc, lbl/**/1 ;\ + srlx tmp4, 57, tmp4 ;\ + andcc tmp4, 1, %g0 ;\ + beq,pn %xcc, lbl/**/1 ;\ + nop ;\ + set TAGREAD_CTX_MASK, tmp1 ;\ + ldxa [tmp3]ASI_ITLB_TAGREAD, tmp2 ;\ + and tmp2, tmp1, tmp6 ;\ + andn tmp2, tmp1, tmp5 ;\ + set KCONTEXT, tmp4 ;\ + cmp tmp6, tmp4 ;\ + be lbl/**/1 ;\ + nop ;\ + VTAG_FLUSHUPAGE(VI/**/lbl, tmp5, tmp6, tmp1, tmp2, tmp3, tmp4) ;\ +lbl/**/1: ;\ + brgz,pt arg1, lbl/**/0 ;\ + sub arg1, 1, arg1 + + + /* * Macro for getting to offset from 'cpu_private' ptr. The 'cpu_private' * ptr is in the machcpu structure. @@ -245,32 +357,27 @@ /*ARGSUSED*/ void -vtag_flushpage(caddr_t vaddr, uint_t ctxnum) -{} - -/*ARGSUSED*/ -void -vtag_flushctx(uint_t ctxnum) +vtag_flushpage(caddr_t vaddr, uint64_t sfmmup) {} /*ARGSUSED*/ void vtag_flushall(void) {} - + /*ARGSUSED*/ void -vtag_flushpage_tl1(uint64_t vaddr, uint64_t ctxnum) +vtag_flushall_uctxs(void) {} - + /*ARGSUSED*/ void -vtag_flush_pgcnt_tl1(uint64_t vaddr, uint64_t ctx_pgcnt) +vtag_flushpage_tl1(uint64_t vaddr, uint64_t sfmmup) {} /*ARGSUSED*/ void -vtag_flushctx_tl1(uint64_t ctxnum, uint64_t dummy) +vtag_flush_pgcnt_tl1(uint64_t vaddr, uint64_t sfmmup_pgcnt) {} /*ARGSUSED*/ @@ -341,20 +448,11 @@ kdi_flush_idcache(int dcache_size, int dcache_lsize, * flush page from the tlb * * %o0 = vaddr - * %o1 = ctxnum + * %o1 = sfmmup */ rdpr %pstate, %o5 #ifdef DEBUG - andcc %o5, PSTATE_IE, %g0 /* if interrupts already */ - bnz,a,pt %icc, 3f /* disabled, panic */ - nop - save %sp, -SA(MINFRAME), %sp - sethi %hi(sfmmu_panic1), %o0 - call panic - or %o0, %lo(sfmmu_panic1), %o0 - ret - restore -3: + PANIC_IF_INTR_DISABLED_PSTR(%o5, sfdi_label1, %g1) #endif /* DEBUG */ /* * disable ints @@ -367,34 +465,40 @@ kdi_flush_idcache(int dcache_size, int dcache_lsize, * Interrupts are disabled to prevent the secondary ctx register * from changing underneath us. */ - brnz,pt %o1, 1f /* KCONTEXT? */ - sethi %hi(FLUSH_ADDR), %o3 + sethi %hi(ksfmmup), %o3 + ldx [%o3 + %lo(ksfmmup)], %o3 + cmp %o3, %o1 + bne,pt %xcc, 1f ! if not kernel as, go to 1 + sethi %hi(FLUSH_ADDR), %o3 /* * For KCONTEXT demaps use primary. type = page implicitly */ stxa %g0, [%o0]ASI_DTLB_DEMAP /* dmmu flush for KCONTEXT */ stxa %g0, [%o0]ASI_ITLB_DEMAP /* immu flush for KCONTEXT */ + flush %o3 b 5f - flush %o3 + nop 1: /* * User demap. We need to set the secondary context properly. * %o0 = vaddr - * %o1 = ctxnum + * %o1 = sfmmup * %o3 = FLUSH_ADDR */ + SFMMU_CPU_CNUM(%o1, %g1, %g2) /* %g1 = sfmmu cnum on this CPU */ + set MMU_SCONTEXT, %o4 ldxa [%o4]ASI_DMMU, %o2 /* rd old ctxnum */ or DEMAP_SECOND | DEMAP_PAGE_TYPE, %o0, %o0 - cmp %o2, %o1 - be,a,pt %icc, 4f + cmp %o2, %g1 + be,pt %icc, 4f nop - stxa %o1, [%o4]ASI_DMMU /* wr new ctxum */ + stxa %g1, [%o4]ASI_DMMU /* wr new ctxum */ 4: stxa %g0, [%o0]ASI_DTLB_DEMAP stxa %g0, [%o0]ASI_ITLB_DEMAP flush %o3 - be,a,pt %icc, 5f + be,pt %icc, 5f nop stxa %o2, [%o4]ASI_DMMU /* restore old ctxnum */ flush %o3 @@ -402,75 +506,65 @@ kdi_flush_idcache(int dcache_size, int dcache_lsize, retl wrpr %g0, %o5, %pstate /* enable interrupts */ SET_SIZE(vtag_flushpage) + + .seg ".text" +.flushallmsg: + .asciz "sfmmu_asm: unimplemented flush operation" + + ENTRY_NP(vtag_flushall) + sethi %hi(.flushallmsg), %o0 + call panic + or %o0, %lo(.flushallmsg), %o0 + SET_SIZE(vtag_flushall) - ENTRY_NP(vtag_flushctx) + ENTRY_NP(vtag_flushall_uctxs) /* - * flush context from the tlb - * - * %o0 = ctxnum - * We disable interrupts to prevent the secondary ctx register changing - * underneath us. + * flush entire DTLB/ITLB. */ - sethi %hi(FLUSH_ADDR), %o3 - set DEMAP_CTX_TYPE | DEMAP_SECOND, %g1 - rdpr %pstate, %o2 + CPU_INDEX(%g1, %g2) + mulx %g1, CPU_NODE_SIZE, %g1 + set cpunodes, %g2 + add %g1, %g2, %g1 + lduh [%g1 + ITLB_SIZE], %g2 ! %g2 = # entries in ITLB + lduh [%g1 + DTLB_SIZE], %g1 ! %g1 = # entries in DTLB + sub %g2, 1, %g2 ! %g2 = # entries in ITLB - 1 + sub %g1, 1, %g1 ! %g1 = # entries in DTLB - 1 + + ! + ! Flush itlb's + ! + ITLB_FLUSH_UNLOCKED_UCTXS(I, %g2, %g3, %g4, %o2, %o3, %o4, %o5) -#ifdef DEBUG - andcc %o2, PSTATE_IE, %g0 /* if interrupts already */ - bnz,a,pt %icc, 1f /* disabled, panic */ - nop - sethi %hi(sfmmu_panic1), %o0 - call panic - or %o0, %lo(sfmmu_panic1), %o0 -1: -#endif /* DEBUG */ + ! + ! Flush dtlb's + ! + DTLB_FLUSH_UNLOCKED_UCTXS(D, %g1, %g3, %g4, %o2, %o3, %o4, %o5) - wrpr %o2, PSTATE_IE, %pstate /* disable interrupts */ - set MMU_SCONTEXT, %o4 - ldxa [%o4]ASI_DMMU, %o5 /* rd old ctxnum */ - cmp %o5, %o0 - be,a,pt %icc, 4f - nop - stxa %o0, [%o4]ASI_DMMU /* wr new ctxum */ -4: - stxa %g0, [%g1]ASI_DTLB_DEMAP - stxa %g0, [%g1]ASI_ITLB_DEMAP - flush %o3 - be,a,pt %icc, 5f - nop - stxa %o5, [%o4]ASI_DMMU /* restore old ctxnum */ - flush %o3 -5: + membar #Sync retl - wrpr %g0, %o2, %pstate /* enable interrupts */ - SET_SIZE(vtag_flushctx) - - .seg ".text" -.flushallmsg: - .asciz "sfmmu_asm: unimplemented flush operation" - - ENTRY_NP(vtag_flushall) - sethi %hi(.flushallmsg), %o0 - call panic - or %o0, %lo(.flushallmsg), %o0 - SET_SIZE(vtag_flushall) + nop + + SET_SIZE(vtag_flushall_uctxs) ENTRY_NP(vtag_flushpage_tl1) /* * x-trap to flush page from tlb and tsb * * %g1 = vaddr, zero-extended on 32-bit kernel - * %g2 = ctxnum + * %g2 = sfmmup * * assumes TSBE_TAG = 0 */ srln %g1, MMU_PAGESHIFT, %g1 slln %g1, MMU_PAGESHIFT, %g1 /* g1 = vaddr */ + + SFMMU_CPU_CNUM(%g2, %g3, %g4) /* %g3 = sfmmu cnum on this CPU */ + /* We need to set the secondary context properly. */ set MMU_SCONTEXT, %g4 ldxa [%g4]ASI_DMMU, %g5 /* rd old ctxnum */ or DEMAP_SECOND | DEMAP_PAGE_TYPE, %g1, %g1 - stxa %g2, [%g4]ASI_DMMU /* wr new ctxum */ + stxa %g3, [%g4]ASI_DMMU /* wr new ctxum */ stxa %g0, [%g1]ASI_DTLB_DEMAP stxa %g0, [%g1]ASI_ITLB_DEMAP stxa %g5, [%g4]ASI_DMMU /* restore old ctxnum */ @@ -483,7 +577,7 @@ kdi_flush_idcache(int dcache_size, int dcache_lsize, * x-trap to flush pgcnt MMU_PAGESIZE pages from tlb * * %g1 = vaddr, zero-extended on 32-bit kernel - * %g2 = <zero32|ctx16|pgcnt16> + * %g2 = <sfmmup58 | pgcnt6> * * NOTE: this handler relies on the fact that no * interrupts or traps can occur during the loop @@ -496,44 +590,35 @@ kdi_flush_idcache(int dcache_size, int dcache_lsize, srln %g1, MMU_PAGESHIFT, %g1 slln %g1, MMU_PAGESHIFT, %g1 /* g1 = vaddr */ or DEMAP_SECOND | DEMAP_PAGE_TYPE, %g1, %g1 - set 0xffff, %g4 - and %g4, %g2, %g3 /* g3 = pgcnt */ - srln %g2, 16, %g2 /* g2 = ctxnum */ + + set SFMMU_PGCNT_MASK, %g4 + and %g4, %g2, %g3 /* g3 = pgcnt - 1 */ + add %g3, 1, %g3 /* g3 = pgcnt */ + + andn %g2, SFMMU_PGCNT_MASK, %g2 /* g2 = sfmmup */ + + SFMMU_CPU_CNUM(%g2, %g5, %g6) ! %g5 = sfmmu cnum on this CPU + /* We need to set the secondary context properly. */ set MMU_SCONTEXT, %g4 - ldxa [%g4]ASI_DMMU, %g5 /* read old ctxnum */ - stxa %g2, [%g4]ASI_DMMU /* write new ctxum */ + ldxa [%g4]ASI_DMMU, %g6 /* read old ctxnum */ + stxa %g5, [%g4]ASI_DMMU /* write new ctxum */ set MMU_PAGESIZE, %g2 /* g2 = pgsize */ + sethi %hi(FLUSH_ADDR), %g5 1: stxa %g0, [%g1]ASI_DTLB_DEMAP stxa %g0, [%g1]ASI_ITLB_DEMAP + flush %g5 deccc %g3 /* decr pgcnt */ bnz,pt %icc,1b - add %g1, %g2, %g1 /* go to nextpage */ + add %g1, %g2, %g1 /* go to nextpage */ - stxa %g5, [%g4]ASI_DMMU /* restore old ctxnum */ + stxa %g6, [%g4]ASI_DMMU /* restore old ctxnum */ membar #Sync retry SET_SIZE(vtag_flush_pgcnt_tl1) - ENTRY_NP(vtag_flushctx_tl1) - /* - * x-trap to flush context from tlb - * - * %g1 = ctxnum - */ - set DEMAP_CTX_TYPE | DEMAP_SECOND, %g4 - set MMU_SCONTEXT, %g3 - ldxa [%g3]ASI_DMMU, %g5 /* rd old ctxnum */ - stxa %g1, [%g3]ASI_DMMU /* wr new ctxum */ - stxa %g0, [%g4]ASI_DTLB_DEMAP - stxa %g0, [%g4]ASI_ITLB_DEMAP - stxa %g5, [%g3]ASI_DMMU /* restore old ctxnum */ - membar #Sync - retry - SET_SIZE(vtag_flushctx_tl1) - ! Not implemented on US1/US2 ENTRY_NP(vtag_flushall_tl1) retry diff --git a/usr/src/uts/sun4u/cpu/us3_common.c b/usr/src/uts/sun4u/cpu/us3_common.c index 6d1a99fa7b..c5ad6fe249 100644 --- a/usr/src/uts/sun4u/cpu/us3_common.c +++ b/usr/src/uts/sun4u/cpu/us3_common.c @@ -2,9 +2,8 @@ * CDDL HEADER START * * The contents of this file are subject to the terms of the - * Common Development and Distribution License, Version 1.0 only - * (the "License"). You may not use this file except in compliance - * with the License. + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. @@ -373,15 +372,8 @@ typedef struct { ss_t stick_sync_stats[NCPU]; #endif /* DEBUG */ -/* - * Maximum number of contexts for Cheetah. - */ -#define MAX_NCTXS (1 << 13) - -/* Will be set !NULL for Cheetah+ and derivatives. */ -uchar_t *ctx_pgsz_array = NULL; +uint_t cpu_impl_dual_pgsz = 0; #if defined(CPU_IMP_DUAL_PAGESIZE) -static uchar_t ctx_pgsz_arr[MAX_NCTXS]; uint_t disable_dual_pgsz = 0; #endif /* CPU_IMP_DUAL_PAGESIZE */ @@ -487,16 +479,6 @@ cpu_setup(void) cache_boot_state = get_dcu() & DCU_CACHE; /* - * Use the maximum number of contexts available for Cheetah - * unless it has been tuned for debugging. - * We are checking against 0 here since this value can be patched - * while booting. It can not be patched via /etc/system since it - * will be patched too late and thus cause the system to panic. - */ - if (nctxs == 0) - nctxs = MAX_NCTXS; - - /* * Due to the number of entries in the fully-associative tlb * this may have to be tuned lower than in spitfire. */ @@ -568,7 +550,7 @@ cpu_setup(void) * Use Cheetah+ and later dual page size support. */ if (!disable_dual_pgsz) { - ctx_pgsz_array = ctx_pgsz_arr; + cpu_impl_dual_pgsz = 1; } #endif /* CPU_IMP_DUAL_PAGESIZE */ @@ -699,7 +681,7 @@ mondo_recover_proc(uint16_t cpuid, int bn) proc_t *p; struct as *as; struct hat *hat; - short cnum; + uint_t cnum; struct tsb_info *tsbinfop; struct tsbe *tsbep; caddr_t tsbp; @@ -710,6 +692,7 @@ mondo_recover_proc(uint16_t cpuid, int bn) int pages_claimed = 0; tte_t tsbe_tte; int tried_kernel_tsb = 0; + mmu_ctx_t *mmu_ctxp; CHEETAH_LIVELOCK_STAT(proc_entry); @@ -769,10 +752,13 @@ mondo_recover_proc(uint16_t cpuid, int bn) goto badstruct; } - cnum = hat->sfmmu_cnum; + mmu_ctxp = CPU_MMU_CTXP(cp); + ASSERT(mmu_ctxp); + cnum = hat->sfmmu_ctxs[mmu_ctxp->mmu_idx].cnum; CHEETAH_LIVELOCK_STATSET(proc_cnum, cnum); - if ((cnum < 0) || (cnum == INVALID_CONTEXT) || (cnum >= nctxs)) { + if ((cnum < 0) || (cnum == INVALID_CONTEXT) || + (cnum >= mmu_ctxp->mmu_nctxs)) { CHEETAH_LIVELOCK_STAT(proc_cnum_bad); goto badstruct; } diff --git a/usr/src/uts/sun4u/cpu/us3_common_asm.s b/usr/src/uts/sun4u/cpu/us3_common_asm.s index 9fcbb21d34..d6b2aad404 100644 --- a/usr/src/uts/sun4u/cpu/us3_common_asm.s +++ b/usr/src/uts/sun4u/cpu/us3_common_asm.s @@ -2,9 +2,8 @@ * CDDL HEADER START * * The contents of this file are subject to the terms of the - * Common Development and Distribution License, Version 1.0 only - * (the "License"). You may not use this file except in compliance - * with the License. + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. @@ -146,7 +145,7 @@ /* ARGSUSED */ void -vtag_flushpage(caddr_t vaddr, u_int ctxnum) +vtag_flushpage(caddr_t vaddr, uint64_t sfmmup) {} #else /* lint */ @@ -156,20 +155,11 @@ vtag_flushpage(caddr_t vaddr, u_int ctxnum) * flush page from the tlb * * %o0 = vaddr - * %o1 = ctxnum + * %o1 = sfmmup */ rdpr %pstate, %o5 #ifdef DEBUG - andcc %o5, PSTATE_IE, %g0 /* if interrupts already */ - bnz,a,pt %icc, 3f /* disabled, panic */ - nop - save %sp, -SA(MINFRAME), %sp - sethi %hi(sfmmu_panic1), %o0 - call panic - or %o0, %lo(sfmmu_panic1), %o0 - ret - restore -3: + PANIC_IF_INTR_DISABLED_PSTR(%o5, u3_di_label0, %g1) #endif /* DEBUG */ /* * disable ints @@ -182,109 +172,51 @@ vtag_flushpage(caddr_t vaddr, u_int ctxnum) * Interrupts are disabled to prevent the primary ctx register * from changing underneath us. */ - brnz,pt %o1, 1f /* KCONTEXT */ - sethi %hi(FLUSH_ADDR), %o3 + sethi %hi(ksfmmup), %o3 + ldx [%o3 + %lo(ksfmmup)], %o3 + cmp %o3, %o1 + bne,pt %xcc, 1f ! if not kernel as, go to 1 + sethi %hi(FLUSH_ADDR), %o3 /* - * For KCONTEXT demaps use primary. type = page implicitly + * For Kernel demaps use primary. type = page implicitly */ stxa %g0, [%o0]ASI_DTLB_DEMAP /* dmmu flush for KCONTEXT */ stxa %g0, [%o0]ASI_ITLB_DEMAP /* immu flush for KCONTEXT */ flush %o3 - b 5f - nop + retl + wrpr %g0, %o5, %pstate /* enable interrupts */ 1: /* * User demap. We need to set the primary context properly. * Secondary context cannot be used for Cheetah IMMU. * %o0 = vaddr - * %o1 = ctxnum + * %o1 = sfmmup * %o3 = FLUSH_ADDR */ - sethi %hi(ctx_pgsz_array), %o4 - ldn [%o4 + %lo(ctx_pgsz_array)], %o4 - brz %o4, 2f - nop - ldub [%o4 + %o1], %o4 + SFMMU_CPU_CNUM(%o1, %g1, %g2) ! %g1 = sfmmu cnum on this CPU + + ldub [%o1 + SFMMU_CEXT], %o4 ! %o4 = sfmmup->sfmmu_cext sll %o4, CTXREG_EXT_SHIFT, %o4 - or %o1, %o4, %o1 -2: + or %g1, %o4, %g1 ! %g1 = pgsz | cnum + wrpr %g0, 1, %tl set MMU_PCONTEXT, %o4 or DEMAP_PRIMARY | DEMAP_PAGE_TYPE, %o0, %o0 - ldxa [%o4]ASI_DMMU, %o2 /* rd old ctxnum */ - stxa %o1, [%o4]ASI_DMMU /* wr new ctxum */ -4: + ldxa [%o4]ASI_DMMU, %o2 ! %o2 = save old ctxnum + stxa %g1, [%o4]ASI_DMMU ! wr new ctxum + stxa %g0, [%o0]ASI_DTLB_DEMAP stxa %g0, [%o0]ASI_ITLB_DEMAP stxa %o2, [%o4]ASI_DMMU /* restore old ctxnum */ flush %o3 wrpr %g0, 0, %tl -5: + retl wrpr %g0, %o5, %pstate /* enable interrupts */ SET_SIZE(vtag_flushpage) #endif /* lint */ - -#if defined(lint) - -/* ARGSUSED */ -void -vtag_flushctx(u_int ctxnum) -{} - -#else /* lint */ - - ENTRY_NP(vtag_flushctx) - /* - * flush context from the tlb - * - * %o0 = ctxnum - * We disable interrupts to prevent the primary ctx register changing - * underneath us. - */ - sethi %hi(FLUSH_ADDR), %o3 - rdpr %pstate, %o2 - -#ifdef DEBUG - andcc %o2, PSTATE_IE, %g0 /* if interrupts already */ - bnz,a,pt %icc, 1f /* disabled, panic */ - nop - sethi %hi(sfmmu_panic1), %o0 - call panic - or %o0, %lo(sfmmu_panic1), %o0 -1: -#endif /* DEBUG */ - - sethi %hi(ctx_pgsz_array), %o4 - ldn [%o4 + %lo(ctx_pgsz_array)], %o4 - brz %o4, 2f - nop - ldub [%o4 + %o0], %o4 - sll %o4, CTXREG_EXT_SHIFT, %o4 - or %o0, %o4, %o0 -2: - wrpr %o2, PSTATE_IE, %pstate /* disable interrupts */ - set MMU_PCONTEXT, %o4 - set DEMAP_CTX_TYPE | DEMAP_PRIMARY, %g1 - wrpr %g0, 1, %tl - ldxa [%o4]ASI_DMMU, %o5 /* rd old ctxnum */ - stxa %o0, [%o4]ASI_DMMU /* wr new ctxum */ -4: - stxa %g0, [%g1]ASI_DTLB_DEMAP - stxa %g0, [%g1]ASI_ITLB_DEMAP - stxa %o5, [%o4]ASI_DMMU /* restore old ctxnum */ - flush %o3 - wrpr %g0, 0, %tl -5: - retl - wrpr %g0, %o2, %pstate /* enable interrupts */ - SET_SIZE(vtag_flushctx) - -#endif /* lint */ - - #if defined(lint) void @@ -314,7 +246,7 @@ vtag_flushall(void) /* ARGSUSED */ void -vtag_flushpage_tl1(uint64_t vaddr, uint64_t ctxnum) +vtag_flushpage_tl1(uint64_t vaddr, uint64_t sfmmup) {} #else /* lint */ @@ -324,12 +256,16 @@ vtag_flushpage_tl1(uint64_t vaddr, uint64_t ctxnum) * x-trap to flush page from tlb and tsb * * %g1 = vaddr, zero-extended on 32-bit kernel - * %g2 = ctxnum + * %g2 = sfmmup * * assumes TSBE_TAG = 0 */ srln %g1, MMU_PAGESHIFT, %g1 - brnz,pt %g2, 1f /* KCONTEXT */ + + sethi %hi(ksfmmup), %g3 + ldx [%g3 + %lo(ksfmmup)], %g3 + cmp %g3, %g2 + bne,pt %xcc, 1f ! if not kernel as, go to 1 slln %g1, MMU_PAGESHIFT, %g1 /* g1 = vaddr */ /* We need to demap in the kernel context */ @@ -340,17 +276,16 @@ vtag_flushpage_tl1(uint64_t vaddr, uint64_t ctxnum) 1: /* We need to demap in a user context */ or DEMAP_PRIMARY | DEMAP_PAGE_TYPE, %g1, %g1 - sethi %hi(ctx_pgsz_array), %g4 - ldn [%g4 + %lo(ctx_pgsz_array)], %g4 - brz %g4, 2f - nop - ldub [%g4 + %g2], %g4 + + SFMMU_CPU_CNUM(%g2, %g6, %g3) ! %g6 = sfmmu cnum on this CPU + + ldub [%g2 + SFMMU_CEXT], %g4 ! %g4 = sfmmup->cext sll %g4, CTXREG_EXT_SHIFT, %g4 - or %g2, %g4, %g2 -2: + or %g6, %g4, %g6 ! %g6 = pgsz | cnum + set MMU_PCONTEXT, %g4 ldxa [%g4]ASI_DMMU, %g5 /* rd old ctxnum */ - stxa %g2, [%g4]ASI_DMMU /* wr new ctxum */ + stxa %g6, [%g4]ASI_DMMU /* wr new ctxum */ stxa %g0, [%g1]ASI_DTLB_DEMAP stxa %g0, [%g1]ASI_ITLB_DEMAP stxa %g5, [%g4]ASI_DMMU /* restore old ctxnum */ @@ -364,7 +299,7 @@ vtag_flushpage_tl1(uint64_t vaddr, uint64_t ctxnum) /* ARGSUSED */ void -vtag_flush_pgcnt_tl1(uint64_t vaddr, uint64_t ctx_pgcnt) +vtag_flush_pgcnt_tl1(uint64_t vaddr, uint64_t sfmmup_pgcnt) {} #else /* lint */ @@ -374,7 +309,7 @@ vtag_flush_pgcnt_tl1(uint64_t vaddr, uint64_t ctx_pgcnt) * x-trap to flush pgcnt MMU_PAGESIZE pages from tlb * * %g1 = vaddr, zero-extended on 32-bit kernel - * %g2 = <zero32|ctx16|pgcnt16> + * %g2 = <sfmmup58|pgcnt6>, (pgcnt - 1) is pass'ed in via pgcnt6 bits. * * NOTE: this handler relies on the fact that no * interrupts or traps can occur during the loop @@ -384,88 +319,68 @@ vtag_flush_pgcnt_tl1(uint64_t vaddr, uint64_t ctx_pgcnt) * * assumes TSBE_TAG = 0 */ - set 0xffff, %g4 - and %g4, %g2, %g3 /* g3 = pgcnt */ - srln %g2, 16, %g2 /* g2 = ctxnum */ + set SFMMU_PGCNT_MASK, %g4 + and %g4, %g2, %g3 /* g3 = pgcnt - 1 */ + add %g3, 1, %g3 /* g3 = pgcnt */ + + andn %g2, SFMMU_PGCNT_MASK, %g2 /* g2 = sfmmup */ srln %g1, MMU_PAGESHIFT, %g1 - brnz,pt %g2, 1f /* KCONTEXT? */ + + sethi %hi(ksfmmup), %g4 + ldx [%g4 + %lo(ksfmmup)], %g4 + cmp %g4, %g2 + bne,pn %xcc, 1f /* if not kernel as, go to 1 */ slln %g1, MMU_PAGESHIFT, %g1 /* g1 = vaddr */ /* We need to demap in the kernel context */ or DEMAP_NUCLEUS | DEMAP_PAGE_TYPE, %g1, %g1 set MMU_PAGESIZE, %g2 /* g2 = pgsize */ + sethi %hi(FLUSH_ADDR), %g5 4: stxa %g0, [%g1]ASI_DTLB_DEMAP stxa %g0, [%g1]ASI_ITLB_DEMAP + flush %g5 ! flush required by immu + deccc %g3 /* decr pgcnt */ bnz,pt %icc,4b add %g1, %g2, %g1 /* next page */ retry 1: - /* We need to demap in a user context */ - sethi %hi(ctx_pgsz_array), %g4 - ldn [%g4 + %lo(ctx_pgsz_array)], %g4 - brz %g4, 2f - or DEMAP_PRIMARY | DEMAP_PAGE_TYPE, %g1, %g1 - ldub [%g4 + %g2], %g4 + /* + * We need to demap in a user context + * + * g2 = sfmmup + * g3 = pgcnt + */ + SFMMU_CPU_CNUM(%g2, %g5, %g6) ! %g5 = sfmmu cnum on this CPU + + or DEMAP_PRIMARY | DEMAP_PAGE_TYPE, %g1, %g1 + + ldub [%g2 + SFMMU_CEXT], %g4 ! %g4 = sfmmup->cext sll %g4, CTXREG_EXT_SHIFT, %g4 - or %g2, %g4, %g2 -2: + or %g5, %g4, %g5 + set MMU_PCONTEXT, %g4 - ldxa [%g4]ASI_DMMU, %g5 /* rd old ctxnum */ - stxa %g2, [%g4]ASI_DMMU /* wr new ctxum */ + ldxa [%g4]ASI_DMMU, %g6 /* rd old ctxnum */ + stxa %g5, [%g4]ASI_DMMU /* wr new ctxum */ set MMU_PAGESIZE, %g2 /* g2 = pgsize */ + sethi %hi(FLUSH_ADDR), %g5 3: stxa %g0, [%g1]ASI_DTLB_DEMAP stxa %g0, [%g1]ASI_ITLB_DEMAP + flush %g5 ! flush required by immu + deccc %g3 /* decr pgcnt */ bnz,pt %icc,3b add %g1, %g2, %g1 /* next page */ - stxa %g5, [%g4]ASI_DMMU /* restore old ctxnum */ + stxa %g6, [%g4]ASI_DMMU /* restore old ctxnum */ retry SET_SIZE(vtag_flush_pgcnt_tl1) #endif /* lint */ - -#if defined(lint) - -/* ARGSUSED */ -void -vtag_flushctx_tl1(uint64_t ctxnum, uint64_t dummy) -{} - -#else /* lint */ - - ENTRY_NP(vtag_flushctx_tl1) - /* - * x-trap to flush context from tlb - * - * %g1 = ctxnum - */ - sethi %hi(ctx_pgsz_array), %g4 - ldn [%g4 + %lo(ctx_pgsz_array)], %g4 - brz %g4, 2f - nop - ldub [%g4 + %g1], %g4 - sll %g4, CTXREG_EXT_SHIFT, %g4 - or %g1, %g4, %g1 -2: - set DEMAP_CTX_TYPE | DEMAP_PRIMARY, %g4 - set MMU_PCONTEXT, %g3 - ldxa [%g3]ASI_DMMU, %g5 /* rd old ctxnum */ - stxa %g1, [%g3]ASI_DMMU /* wr new ctxum */ - stxa %g0, [%g4]ASI_DTLB_DEMAP - stxa %g0, [%g4]ASI_ITLB_DEMAP - stxa %g5, [%g3]ASI_DMMU /* restore old ctxnum */ - retry - SET_SIZE(vtag_flushctx_tl1) - -#endif /* lint */ - - #if defined(lint) /*ARGSUSED*/ @@ -3180,13 +3095,7 @@ itlb_erratum34_fixup(void) ENTRY_NP(itlb_erratum34_fixup) rdpr %pstate, %o3 #ifdef DEBUG - andcc %o3, PSTATE_IE, %g0 ! If interrupts already - bnz,pt %icc, 0f ! disabled, panic - nop - sethi %hi(sfmmu_panic1), %o0 - call panic - or %o0, %lo(sfmmu_panic1), %o0 -0: + PANIC_IF_INTR_DISABLED_PSTR(%o3, u3_di_label1, %g1) #endif /* DEBUG */ wrpr %o3, PSTATE_IE, %pstate ! Disable interrupts ldxa [%g0]ASI_ITLB_ACCESS, %o1 ! %o1 = entry 0 data diff --git a/usr/src/uts/sun4u/cpu/us3_common_mmu.c b/usr/src/uts/sun4u/cpu/us3_common_mmu.c index db279eeff8..ebaaae2cc1 100644 --- a/usr/src/uts/sun4u/cpu/us3_common_mmu.c +++ b/usr/src/uts/sun4u/cpu/us3_common_mmu.c @@ -2,9 +2,8 @@ * CDDL HEADER START * * The contents of this file are subject to the terms of the - * Common Development and Distribution License, Version 1.0 only - * (the "License"). You may not use this file except in compliance - * with the License. + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. @@ -20,7 +19,7 @@ * CDDL HEADER END */ /* - * Copyright 2005 Sun Microsystems, Inc. All rights reserved. + * Copyright 2006 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ @@ -47,9 +46,6 @@ * SPARC V9 JPS1 Implementation Supplement: Sun UltraSPARC-III */ -/* Will be set !NULL for Cheetah+ and derivatives. */ -extern uchar_t *ctx_pgsz_array; - /* * pan_disable_ism_large_pages and pan_disable_large_pages are the Panther- * specific versions of disable_ism_large_pages and disable_large_pages, @@ -115,7 +111,7 @@ mmu_large_pages_disabled(uint_t flag) * * The effect of these restrictions is to limit the allowable values in * sfmmu_pgsz[0] and sfmmu_pgsz[1], since these hat variables are used in - * mmu_set_ctx_page_sizes to set up the values in the ctx_pgsz_array that + * mmu_set_ctx_page_sizes to set up the values in the sfmmu_cext that * are used at context switch time. The value in sfmmu_pgsz[0] is used in * P_pgsz0 and sfmmu_pgsz[1] is used in P_pgsz1, as per Figure F-1-1 * IMMU and DMMU Primary Context Register in the Panther Implementation @@ -152,7 +148,7 @@ int init_mmu_page_sizes = 0; void mmu_init_large_pages(size_t ism_pagesize) { - if (ctx_pgsz_array == NULL) { /* disable_dual_pgsz flag */ + if (cpu_impl_dual_pgsz == 0) { /* disable_dual_pgsz flag */ pan_disable_ism_large_pages = ((1 << TTE64K) | (1 << TTE512K) | (1 << TTE32M) | (1 << TTE256M)); pan_disable_large_pages = ((1 << TTE32M) | (1 << TTE256M)); @@ -310,7 +306,7 @@ mmu_fixup_large_pages(struct hat *hat, uint64_t *ttecnt, uint8_t *tmp_pgsz) */ ASSERT(hat->sfmmu_ismhat == NULL); ASSERT(hat != ksfmmup); - ASSERT(ctx_pgsz_array != NULL); + ASSERT(cpu_impl_dual_pgsz == 1); ASSERT((!SFMMU_FLAGS_ISSET(hat, HAT_32M_FLAG)) || (!SFMMU_FLAGS_ISSET(hat, HAT_256M_FLAG))); @@ -362,7 +358,7 @@ mmu_setup_page_sizes(struct hat *hat, uint64_t *ttecnt, uint8_t *tmp_pgsz) ASSERT(hat->sfmmu_ismhat == NULL); ASSERT(hat != ksfmmup); - if (ctx_pgsz_array == NULL) /* disable_dual_pgsz flag */ + if (cpu_impl_dual_pgsz == 0) /* disable_dual_pgsz flag */ return; /* @@ -451,7 +447,7 @@ mmu_set_ctx_page_sizes(struct hat *hat) ASSERT(sfmmu_hat_lock_held(hat)); ASSERT(hat != ksfmmup); - if (ctx_pgsz_array == NULL) /* disable_dual_pgsz flag */ + if (cpu_impl_dual_pgsz == 0) /* disable_dual_pgsz flag */ return; /* @@ -473,12 +469,22 @@ mmu_set_ctx_page_sizes(struct hat *hat) #endif /* DEBUG */ new_cext = TAGACCEXT_MKSZPAIR(pgsz1, pgsz0); if (hat->sfmmu_cext != new_cext) { +#ifdef DEBUG + int i; + /* + * assert cnum should be invalid, this is because pagesize + * can only be changed after a proc's ctxs are invalidated. + */ + for (i = 0; i < max_mmu_ctxdoms; i++) { + ASSERT(hat->sfmmu_ctxs[i].cnum == INVALID_CONTEXT); + } +#endif /* DEBUG */ hat->sfmmu_cext = new_cext; } - ctx_pgsz_array[hat->sfmmu_cnum] = hat->sfmmu_cext; + /* * sfmmu_setctx_sec() will take care of the - * rest of the chores reprogramming the ctx_pgsz_array + * rest of the chores reprogramming the hat->sfmmu_cext * page size values into the DTLBs. */ } @@ -537,7 +543,7 @@ mmu_check_page_sizes(sfmmu_t *sfmmup, uint64_t *ttecnt) } newval = tmp_pgsz[0] << 8 | tmp_pgsz[1]; if (newval != oldval) { - sfmmu_steal_context(sfmmup, tmp_pgsz); + sfmmu_reprog_pgsz_arr(sfmmup, tmp_pgsz); } } } @@ -603,8 +609,6 @@ mmu_init_kernel_pgsz(struct hat *hat) new_cext_nucleus = TAGACCEXT_MKSZPAIR(tte, TTE8K); new_cext_primary = TAGACCEXT_MKSZPAIR(TTE8K, tte); - if (ctx_pgsz_array) - ctx_pgsz_array[KCONTEXT] = new_cext_primary; hat->sfmmu_cext = new_cext_primary; kcontextreg = ((uint64_t)new_cext_nucleus << CTXREG_NEXT_SHIFT) | ((uint64_t)new_cext_primary << CTXREG_EXT_SHIFT); @@ -618,6 +622,11 @@ mmu_get_kernel_lpsize(size_t lpsize) int impl = cpunodes[getprocessorid()].implementation; uint_t tte = TTE8K; + if (cpu_impl_dual_pgsz == 0) { + heaplp_use_dt512 = 0; + return (MMU_PAGESIZE); + } + pend_lpgsz = (struct heap_lp_page_size *) ((char *)heap_lp_pgsz + sizeof (heap_lp_pgsz)); diff --git a/usr/src/uts/sun4u/lw8/os/lw8_platmod.c b/usr/src/uts/sun4u/lw8/os/lw8_platmod.c index 0b6c8c4a0c..c21b03f0af 100644 --- a/usr/src/uts/sun4u/lw8/os/lw8_platmod.c +++ b/usr/src/uts/sun4u/lw8/os/lw8_platmod.c @@ -126,13 +126,12 @@ int serengeti_tsb_spares = (SG_MAX_IO_BDS * SG_SCHIZO_PER_IO_BD * IOMMU_PER_SCHIZO); /* - * sg_max_ncpus is the maximum number of CPUs supported on Serengeti - * and Wildcat at GA. We assume that the maximum number of SSM nodes - * supported at GA is 4. sg_max_ncpus is set to be smaller than NCPU - * to reduce the amount of memory the logs take up until we have a - * dynamic log memory allocation solution. + * sg_max_ncpus is the maximum number of CPUs supported on lw8. + * sg_max_ncpus is set to be smaller than NCPU to reduce the amount of + * memory the logs take up until we have a dynamic log memory allocation + * solution. */ -int sg_max_ncpus = (24 * 4); /* (CPUs per node * max number of nodes) */ +int sg_max_ncpus = (12 * 2); /* (max # of processors * # of cores/proc) */ /* * variables to control mailbox message timeouts. diff --git a/usr/src/uts/sun4u/ml/mach_offsets.in b/usr/src/uts/sun4u/ml/mach_offsets.in index 560127697e..6f1f317f67 100644 --- a/usr/src/uts/sun4u/ml/mach_offsets.in +++ b/usr/src/uts/sun4u/ml/mach_offsets.in @@ -83,8 +83,8 @@ #include <sys/traptrace.h> machcpu - intrstat MCPU_INTRSTAT - pil_high_start MCPU_PIL_HIGH_START + intrstat MCPU_INTRSTAT + pil_high_start MCPU_PIL_HIGH_START trap_trace_record TRAP_ENT_SIZE tt_tl TRAP_ENT_TL @@ -100,11 +100,13 @@ trap_trace_record TRAP_ENT_SIZE tt_f4 TRAP_ENT_F4 hat HAT_SIZE - sfmmu_cnum sfmmu_cpusran sfmmu_tsb sfmmu_ismblkpa sfmmu_flags + sfmmu_cext + sfmmu_ctx_lock + sfmmu_ctxs sfmmu_global_stat HATSTAT_SIZE sf_pagefaults HATSTAT_PAGEFAULT @@ -113,9 +115,6 @@ sfmmu_global_stat HATSTAT_SIZE sf_khash_searches HATSTAT_KHASH_SEARCH sf_khash_links HATSTAT_KHASH_LINKS -ctx CTX_SIZE CTX_SZ_SHIFT - ctx_un.ctx_sfmmup CTX_SFMMUP - sf_hment SFHME_SIZE SFHME_SHIFT hme_tte SFHME_TTE @@ -212,6 +211,8 @@ cpu_node CPU_NODE_SIZE ecache_size ECACHE_SIZE ecache_linesize ECACHE_LINESIZE device_id DEVICE_ID + itlb_size ITLB_SIZE + dtlb_size DTLB_SIZE spitfire_scrub_misc_t ec_scrub_outstanding diff --git a/usr/src/uts/sun4u/opl/io/drmach.c b/usr/src/uts/sun4u/opl/io/drmach.c index 3b6f520447..61609b0d97 100644 --- a/usr/src/uts/sun4u/opl/io/drmach.c +++ b/usr/src/uts/sun4u/opl/io/drmach.c @@ -3681,12 +3681,12 @@ drmach_lock_critical(caddr_t va) int i; for (i = 0; i < DRMACH_FMEM_LOCKED_PAGES; i++) { - vtag_flushpage(va, KCONTEXT); + vtag_flushpage(va, (uint64_t)ksfmmup); sfmmu_memtte(&tte, va_to_pfn(va), PROC_DATA|HAT_NOSYNC, TTE8K); tte.tte_intlo |= TTE_LCK_INT; - sfmmu_dtlb_ld(va, KCONTEXT, &tte); - sfmmu_itlb_ld(va, KCONTEXT, &tte); + sfmmu_dtlb_ld_kva(va, &tte); + sfmmu_itlb_ld_kva(va, &tte); va += PAGESIZE; } } @@ -3697,7 +3697,7 @@ drmach_unlock_critical(caddr_t va) int i; for (i = 0; i < DRMACH_FMEM_LOCKED_PAGES; i++) { - vtag_flushpage(va, KCONTEXT); + vtag_flushpage(va, (uint64_t)ksfmmup); va += PAGESIZE; } } diff --git a/usr/src/uts/sun4u/opl/io/mc-opl.c b/usr/src/uts/sun4u/opl/io/mc-opl.c index 9a5415dcd9..c6b6ee51b0 100644 --- a/usr/src/uts/sun4u/opl/io/mc-opl.c +++ b/usr/src/uts/sun4u/opl/io/mc-opl.c @@ -19,6 +19,10 @@ * CDDL HEADER END */ /* + * Copyright 2006 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ +/* * All Rights Reserved, Copyright (c) FUJITSU LIMITED 2006 */ @@ -30,6 +34,7 @@ #include <sys/modctl.h> #include <sys/stat.h> #include <sys/async.h> +#include <sys/machcpuvar.h> #include <sys/machsystm.h> #include <sys/promif.h> #include <sys/ksynch.h> diff --git a/usr/src/uts/sun4u/opl/os/opl.c b/usr/src/uts/sun4u/opl/os/opl.c index 3efec3ee6b..36dc3da282 100644 --- a/usr/src/uts/sun4u/opl/os/opl.c +++ b/usr/src/uts/sun4u/opl/os/opl.c @@ -72,6 +72,17 @@ int opl_tsb_spares = (OPL_MAX_BOARDS) * (OPL_MAX_PCICH_UNITS_PER_BOARD) * pgcnt_t opl_startup_cage_size = 0; +static opl_model_info_t opl_models[] = { + { "FF1", OPL_MAX_BOARDS_FF1 }, + { "FF2", OPL_MAX_BOARDS_FF2 }, + { "DC1", OPL_MAX_BOARDS_DC1 }, + { "DC2", OPL_MAX_BOARDS_DC2 }, + { "DC3", OPL_MAX_BOARDS_DC3 }, +}; +static int opl_num_models = sizeof (opl_models)/sizeof (opl_model_info_t); + +static opl_model_info_t *opl_cur_model = NULL; + static struct memlist *opl_memlist_per_board(struct memlist *ml); static enum { @@ -92,6 +103,53 @@ set_platform_tsb_spares(void) return (MIN(opl_tsb_spares, MAX_UPA)); } +static void +set_model_info() +{ + char name[MAXSYSNAME]; + int i; + + /* + * Get model name from the root node. + * + * We are using the prom device tree since, at this point, + * the Solaris device tree is not yet setup. + */ + (void) prom_getprop(prom_rootnode(), "model", (caddr_t)name); + + for (i = 0; i < opl_num_models; i++) { + if (strncmp(name, opl_models[i].model_name, MAXSYSNAME) == 0) { + opl_cur_model = &opl_models[i]; + break; + } + } + if (i == opl_num_models) + cmn_err(CE_WARN, "No valid OPL model is found!" + "Set max_mmu_ctxdoms to the default."); +} + +static void +set_max_mmu_ctxdoms() +{ + extern uint_t max_mmu_ctxdoms; + int max_boards; + + /* + * From the model, get the maximum number of boards + * supported and set the value accordingly. If the model + * could not be determined or recognized, we assume the max value. + */ + if (opl_cur_model == NULL) + max_boards = OPL_MAX_BOARDS; + else + max_boards = opl_cur_model->model_max_boards; + + /* + * On OPL, cores and MMUs are one-to-one. + */ + max_mmu_ctxdoms = OPL_MAX_CORE_UNITS_PER_BOARD * max_boards; +} + #pragma weak mmu_init_large_pages void @@ -123,6 +181,9 @@ set_platform_defaults(void) } tsb_lgrp_affinity = 1; + + set_model_info(); + set_max_mmu_ctxdoms(); } /* @@ -906,6 +967,23 @@ startup_platform(void) { } +void +plat_cpuid_to_mmu_ctx_info(processorid_t cpuid, mmu_ctx_info_t *info) +{ + int impl; + + impl = cpunodes[cpuid].implementation; + if (IS_OLYMPUS_C(impl)) { + /* + * Olympus-C processor supports 2 strands per core. + */ + info->mmu_idx = cpuid >> 1; + info->mmu_nctxs = 8192; + } else { + cmn_err(CE_PANIC, "Unknown processor %d", impl); + } +} + int plat_get_mem_sid(char *unum, char *buf, int buflen, int *lenp) { diff --git a/usr/src/uts/sun4u/os/cpr_impl.c b/usr/src/uts/sun4u/os/cpr_impl.c index b66b18d098..b2744bab98 100644 --- a/usr/src/uts/sun4u/os/cpr_impl.c +++ b/usr/src/uts/sun4u/os/cpr_impl.c @@ -707,7 +707,7 @@ i_cpr_mapin(caddr_t vaddr, uint_t pages, pfn_t ppn) tte.tte_inthi = TTE_VALID_INT | TTE_PFN_INTHI(ppn); tte.tte_intlo = TTE_PFN_INTLO(ppn) | TTE_LCK_INT | TTE_CP_INT | TTE_PRIV_INT | TTE_HWWR_INT; - sfmmu_dtlb_ld(vaddr, KCONTEXT, &tte); + sfmmu_dtlb_ld_kva(vaddr, &tte); } } @@ -721,7 +721,7 @@ i_cpr_mapout(caddr_t vaddr, uint_t pages) curthreadremapped = 0; for (; pages--; vaddr += MMU_PAGESIZE) - vtag_flushpage(vaddr, KCONTEXT); + vtag_flushpage(vaddr, (uint64_t)ksfmmup); } /* diff --git a/usr/src/uts/sun4u/os/ppage.c b/usr/src/uts/sun4u/os/ppage.c index 83283feda2..1a456e4ca4 100644 --- a/usr/src/uts/sun4u/os/ppage.c +++ b/usr/src/uts/sun4u/os/ppage.c @@ -2,9 +2,8 @@ * CDDL HEADER START * * The contents of this file are subject to the terms of the - * Common Development and Distribution License, Version 1.0 only - * (the "License"). You may not use this file except in compliance - * with the License. + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. @@ -20,7 +19,7 @@ * CDDL HEADER END */ /* - * Copyright 2005 Sun Microsystems, Inc. All rights reserved. + * Copyright 2006 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ @@ -340,7 +339,7 @@ pp_load_tlb(processorid_t cpu, caddr_t **pslot, page_t *pp, uint_t prot) TTE_CV_INT | TTE_PRIV_INT | TTE_LCK_INT | prot; ASSERT(CPU->cpu_id == cpu); - sfmmu_dtlb_ld(va, KCONTEXT, &tte); + sfmmu_dtlb_ld_kva(va, &tte); *pslot = myslot; /* Return ptr to the slot we used. */ @@ -352,7 +351,7 @@ pp_unload_tlb(caddr_t *pslot, caddr_t va) { ASSERT(*pslot == va); - vtag_flushpage(va, KCONTEXT); + vtag_flushpage(va, (uint64_t)ksfmmup); *pslot = NULL; /* release the slot */ } diff --git a/usr/src/uts/sun4u/serengeti/io/sbdp_cpu.c b/usr/src/uts/sun4u/serengeti/io/sbdp_cpu.c index c294f2035d..0fd6bd12cb 100644 --- a/usr/src/uts/sun4u/serengeti/io/sbdp_cpu.c +++ b/usr/src/uts/sun4u/serengeti/io/sbdp_cpu.c @@ -20,7 +20,7 @@ */ /* - * Copyright 2005 Sun Microsystems, Inc. All rights reserved. + * Copyright 2006 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ @@ -668,8 +668,8 @@ sbdp_cpu_stop_self(uint64_t pa) TTE_PFN_INTHI(bbsram_pfn); tte.tte_intlo = TTE_PFN_INTLO(bbsram_pfn) | TTE_HWWR_INT | TTE_PRIV_INT | TTE_LCK_INT; - sfmmu_dtlb_ld(sbdp_shutdown_va, KCONTEXT, &tte); /* load dtlb */ - sfmmu_itlb_ld(sbdp_shutdown_va, KCONTEXT, &tte); /* load itlb */ + sfmmu_dtlb_ld_kva(sbdp_shutdown_va, &tte); /* load dtlb */ + sfmmu_itlb_ld_kva(sbdp_shutdown_va, &tte); /* load itlb */ for (src = (uint_t *)sbdp_shutdown_asm, dst = (uint_t *)bbsram_addr; src < (uint_t *)sbdp_shutdown_asm_end; src++, dst++) diff --git a/usr/src/uts/sun4u/serengeti/os/serengeti.c b/usr/src/uts/sun4u/serengeti/os/serengeti.c index 5a30846f35..f6a055181f 100644 --- a/usr/src/uts/sun4u/serengeti/os/serengeti.c +++ b/usr/src/uts/sun4u/serengeti/os/serengeti.c @@ -126,13 +126,12 @@ int serengeti_tsb_spares = (SG_MAX_IO_BDS * SG_SCHIZO_PER_IO_BD * IOMMU_PER_SCHIZO); /* - * sg_max_ncpus is the maximum number of CPUs supported on Serengeti - * and Wildcat at GA. We assume that the maximum number of SSM nodes - * supported at GA is 4. sg_max_ncpus is set to be smaller than NCPU - * to reduce the amount of memory the logs take up until we have a - * dynamic log memory allocation solution. + * sg_max_ncpus is the maximum number of CPUs supported on Serengeti. + * sg_max_ncpus is set to be smaller than NCPU to reduce the amount of + * memory the logs take up until we have a dynamic log memory allocation + * solution. */ -int sg_max_ncpus = (24 * 4); /* (CPUs per node * max number of nodes) */ +int sg_max_ncpus = (24 * 2); /* (max # of processors * # of cores/proc) */ /* * variables to control mailbox message timeouts. diff --git a/usr/src/uts/sun4u/starcat/io/drmach.c b/usr/src/uts/sun4u/starcat/io/drmach.c index 417991781b..63d3b9f855 100644 --- a/usr/src/uts/sun4u/starcat/io/drmach.c +++ b/usr/src/uts/sun4u/starcat/io/drmach.c @@ -3350,8 +3350,8 @@ drmach_copy_rename_init(drmachid_t t_id, uint64_t t_slice_offset, tte = &drmach_cpu_sram_tte[CPU->cpu_id]; ASSERT(TTE_IS_VALID(tte) && TTE_IS_8K(tte) && TTE_IS_PRIVILEGED(tte) && TTE_IS_LOCKED(tte)); - sfmmu_dtlb_ld(drmach_cpu_sram_va, KCONTEXT, tte); - sfmmu_itlb_ld(drmach_cpu_sram_va, KCONTEXT, tte); + sfmmu_dtlb_ld_kva(drmach_cpu_sram_va, tte); + sfmmu_itlb_ld_kva(drmach_cpu_sram_va, tte); bp = wp = drmach_cpu_sram_va; @@ -3377,7 +3377,7 @@ drmach_copy_rename_init(drmachid_t t_id, uint64_t t_slice_offset, if (err) { cleanup: xt_one(CPU->cpu_id, vtag_flushpage_tl1, - (uint64_t)drmach_cpu_sram_va, (uint64_t)KCONTEXT); + (uint64_t)drmach_cpu_sram_va, (uint64_t)ksfmmup); return (err); } @@ -3436,7 +3436,7 @@ drmach_copy_rename_fini(drmachid_t id) axq_cdc_enable_all(); xt_one(CPU->cpu_id, vtag_flushpage_tl1, - (uint64_t)drmach_cpu_sram_va, (uint64_t)KCONTEXT); + (uint64_t)drmach_cpu_sram_va, (uint64_t)ksfmmup); switch (cr->ecode) { case DRMACH_CR_OK: @@ -5447,7 +5447,7 @@ drmach_cpu_start(struct cpu *cp) } xt_one(cpuid, vtag_flushpage_tl1, - (uint64_t)drmach_cpu_sram_va, (uint64_t)KCONTEXT); + (uint64_t)drmach_cpu_sram_va, (uint64_t)ksfmmup); return (0); } @@ -5499,8 +5499,8 @@ drmach_cpu_stop_self(void) tte = &drmach_cpu_sram_tte[CPU->cpu_id]; ASSERT(TTE_IS_VALID(tte) && TTE_IS_8K(tte) && TTE_IS_PRIVILEGED(tte) && TTE_IS_LOCKED(tte)); - sfmmu_dtlb_ld(drmach_cpu_sram_va, KCONTEXT, tte); - sfmmu_itlb_ld(drmach_cpu_sram_va, KCONTEXT, tte); + sfmmu_dtlb_ld_kva(drmach_cpu_sram_va, tte); + sfmmu_itlb_ld_kva(drmach_cpu_sram_va, tte); /* copy text. standard bcopy not designed to work in nc space */ p = (uint_t *)drmach_cpu_sram_va; diff --git a/usr/src/uts/sun4u/starcat/ml/drmach_asm.s b/usr/src/uts/sun4u/starcat/ml/drmach_asm.s index 21083aaf85..c062e6596b 100644 --- a/usr/src/uts/sun4u/starcat/ml/drmach_asm.s +++ b/usr/src/uts/sun4u/starcat/ml/drmach_asm.s @@ -2,9 +2,8 @@ * CDDL HEADER START * * The contents of this file are subject to the terms of the - * Common Development and Distribution License, Version 1.0 only - * (the "License"). You may not use this file except in compliance - * with the License. + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. @@ -20,7 +19,7 @@ * CDDL HEADER END */ /* - * Copyright 2005 Sun Microsystems, Inc. All rights reserved. + * Copyright 2006 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ @@ -602,12 +601,16 @@ drmach_rename_end: ldx [%g1], %g1 or %g1, KCONTEXT, %g2 ! preserve %g1 set MMU_TAG_ACCESS, %g4 - sethi %hi(ctx_pgsz_array), %g6 - ldn [%g6 + %lo(ctx_pgsz_array)], %g6 + set cpu_impl_dual_pgsz, %g6 + ld [%g6], %g6 brz %g6, 1f nop - ldub [%g6 + KCONTEXT], %g6 - sll %g6, TAGACCEXT_SHIFT, %g6 + + sethi %hi(ksfmmup), %g6 + ldx [%g6 + %lo(ksfmmup)], %g6 + ldub [%g6 + SFMMU_CEXT], %g6 + sll %g6, TAGACCEXT_SHIFT, %g6 + set MMU_TAG_ACCESS_EXT, %g7 stxa %g6, [%g7]ASI_DMMU 1: diff --git a/usr/src/uts/sun4u/starcat/sys/starcat.h b/usr/src/uts/sun4u/starcat/sys/starcat.h index af02d56e93..b5af261bf2 100644 --- a/usr/src/uts/sun4u/starcat/sys/starcat.h +++ b/usr/src/uts/sun4u/starcat/sys/starcat.h @@ -2,9 +2,8 @@ * CDDL HEADER START * * The contents of this file are subject to the terms of the - * Common Development and Distribution License, Version 1.0 only - * (the "License"). You may not use this file except in compliance - * with the License. + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. @@ -20,7 +19,7 @@ * CDDL HEADER END */ /* - * Copyright 2004 Sun Microsystems, Inc. All rights reserved. + * Copyright 2006 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ @@ -33,7 +32,6 @@ extern "C" { #endif - /* * Manifest constants of Starcat configuration */ @@ -80,6 +78,30 @@ extern "C" { #define STARCAT_DMV_IDN_BASE (MAX_UPA) /* + * The CPU ID on starcat looks like this: + * + * 9 5 4 3 2 1 0 + * -------------------------------------- + * | Expander | | Slot | Core | LPORT | + * -------------------------------------- + * + * Expander Starcat has STARCAT_BDSET_MAX (18) expanders. + * Slot Starcat has STARCAT_BDSET_SLOT_MAX (2) slots per expander. + * Slot 0 carries a CPU-MEM board which has 4 processor chips. + * Slot 1 carries an I/O board typically. But it can be + * configured to carry a MAXCAT board which has 2 processor + * chips on board. + * LPORT Port number within the slot for a chip. This is also the + * chip number within the slot. Note that Slot 1 can have only + * 2 chips, but this representation allows for 4. This is just + * the theoretical max. + * Core Core number within the chip. + * + * Currently, the maximum number of cores supported is 2 per chip (on + * Panther and Jaguar). + * + */ +/* * Macros for manipulating CPU IDs */ #define STARCAT_CPUID_TO_EXPANDER(p) (((p) >> 5) & 0x1f) diff --git a/usr/src/uts/sun4u/starfire/io/drmach.c b/usr/src/uts/sun4u/starfire/io/drmach.c index ed322c68a7..ee70d97eb2 100644 --- a/usr/src/uts/sun4u/starfire/io/drmach.c +++ b/usr/src/uts/sun4u/starfire/io/drmach.c @@ -2452,7 +2452,7 @@ drmach_cpu_start(struct cpu *cp) drmach_cpu_ntries - ntries, drmach_cpu_ntries, cpuid); xt_one(cpuid, vtag_flushpage_tl1, - (uint64_t)drmach_shutdown_va, (uint64_t)KCONTEXT); + (uint64_t)drmach_shutdown_va, (uint64_t)ksfmmup); return (0); } @@ -2516,8 +2516,8 @@ drmach_cpu_stop_self(void) TTE_PFN_INTHI(bbsram_pfn); tte.tte_intlo = TTE_PFN_INTLO(bbsram_pfn) | TTE_HWWR_INT | TTE_PRIV_INT | TTE_LCK_INT; - sfmmu_dtlb_ld(drmach_shutdown_va, KCONTEXT, &tte); /* load dtlb */ - sfmmu_itlb_ld(drmach_shutdown_va, KCONTEXT, &tte); /* load itlb */ + sfmmu_dtlb_ld_kva(drmach_shutdown_va, &tte); /* load dtlb */ + sfmmu_itlb_ld_kva(drmach_shutdown_va, &tte); /* load itlb */ for (src = (uint_t *)drmach_shutdown_asm, dst = (uint_t *)bbsram_addr; src < (uint_t *)drmach_shutdown_asm_end; src++, dst++) diff --git a/usr/src/uts/sun4u/sunfire/io/ac_test.c b/usr/src/uts/sun4u/sunfire/io/ac_test.c index 4a1a1e9396..729282db9c 100644 --- a/usr/src/uts/sun4u/sunfire/io/ac_test.c +++ b/usr/src/uts/sun4u/sunfire/io/ac_test.c @@ -83,14 +83,14 @@ ac_mapin(uint64_t pa, caddr_t va) TTE_PFN_INTHI(pfn); tte.tte_intlo = TTE_PFN_INTLO(pfn) | TTE_CP_INT | TTE_PRIV_INT | TTE_LCK_INT | TTE_HWWR_INT; - sfmmu_dtlb_ld(va, KCONTEXT, &tte); + sfmmu_dtlb_ld_kva(va, &tte); } void ac_unmap(caddr_t va) { - vtag_flushpage(va, KCONTEXT); + vtag_flushpage(va, (uint64_t)ksfmmup); } int diff --git a/usr/src/uts/sun4u/sunfire/io/fhc.c b/usr/src/uts/sun4u/sunfire/io/fhc.c index 7b648c692e..e310fa8b6a 100644 --- a/usr/src/uts/sun4u/sunfire/io/fhc.c +++ b/usr/src/uts/sun4u/sunfire/io/fhc.c @@ -20,7 +20,7 @@ */ /* - * Copyright 2005 Sun Microsystems, Inc. All rights reserved. + * Copyright 2006 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ @@ -3402,8 +3402,8 @@ os_completes_shutdown(void) TTE_PFN_INTHI(pfn); tte.tte_intlo = TTE_PFN_INTLO(pfn) | TTE_HWWR_INT | TTE_PRIV_INT | TTE_LCK_INT; /* un$ */ - sfmmu_dtlb_ld(shutdown_va, KCONTEXT, &tte); /* load dtlb */ - sfmmu_itlb_ld(shutdown_va, KCONTEXT, &tte); /* load itlb */ + sfmmu_dtlb_ld_kva(shutdown_va, &tte); /* load dtlb */ + sfmmu_itlb_ld_kva(shutdown_va, &tte); /* load itlb */ /* * copy the special shutdown function to sram diff --git a/usr/src/uts/sun4u/sys/cpu_module.h b/usr/src/uts/sun4u/sys/cpu_module.h index 0df1b445c7..a743d4264d 100644 --- a/usr/src/uts/sun4u/sys/cpu_module.h +++ b/usr/src/uts/sun4u/sys/cpu_module.h @@ -74,12 +74,12 @@ int mmu_init_mmu_page_sizes(int cinfo); /* * virtual demap flushes (tlbs & virtual tag caches) */ -void vtag_flushpage(caddr_t addr, uint_t ctx); -void vtag_flushctx(uint_t ctx); +void vtag_flushpage(caddr_t addr, uint64_t sfmmup); void vtag_flushall(void); -void vtag_flushpage_tl1(uint64_t addr, uint64_t ctx); -void vtag_flush_pgcnt_tl1(uint64_t addr, uint64_t ctx_pgcnt); -void vtag_flushctx_tl1(uint64_t ctx, uint64_t dummy); +#pragma weak vtag_flushall_uctxs +void vtag_flushall_uctxs(void); +void vtag_flushpage_tl1(uint64_t addr, uint64_t sfmmup); +void vtag_flush_pgcnt_tl1(uint64_t addr, uint64_t sfmmup_pgcnt); void vtag_flushall_tl1(uint64_t dummy1, uint64_t dummy2); /* @@ -100,10 +100,10 @@ void send_mondo_set(cpuset_t set); #endif /* - * Calculate, set optimal dtlb pagesize, for ISM and mpss, to support - * cpus with non-fully-associative dtlbs. + * flag to support optimal dtlb pagesize setting, for ISM and mpss, to support + * cpus with non-fully-associative dtlbs. Page size is stored in hat sfmmu_cext */ -extern uchar_t *ctx_pgsz_array; +extern uint_t cpu_impl_dual_pgsz; /* * flush instruction cache if needed diff --git a/usr/src/uts/sun4u/sys/machcpuvar.h b/usr/src/uts/sun4u/sys/machcpuvar.h index 641154b543..29e9cc19b9 100644 --- a/usr/src/uts/sun4u/sys/machcpuvar.h +++ b/usr/src/uts/sun4u/sys/machcpuvar.h @@ -115,6 +115,11 @@ struct machcpu { * E$ data, which is needed for the specific cpu type. */ void *cpu_private; /* ptr to cpu private data */ + /* + * per-MMU ctxdom CPU data. + */ + uint_t cpu_mmu_idx; + struct mmu_ctx *cpu_mmu_ctxp; ptl1_state_t ptl1_state; @@ -134,6 +139,8 @@ struct machcpu { typedef struct machcpu machcpu_t; #define cpu_startup_thread cpu_m.startup_thread +#define CPU_MMU_IDX(cp) ((cp)->cpu_m.cpu_mmu_idx) +#define CPU_MMU_CTXP(cp) ((cp)->cpu_m.cpu_mmu_ctxp) #define NINTR_THREADS (LOCK_LEVEL) /* number of interrupt threads */ /* diff --git a/usr/src/uts/sun4u/sys/machparam.h b/usr/src/uts/sun4u/sys/machparam.h index de12ef0ec1..209b651472 100644 --- a/usr/src/uts/sun4u/sys/machparam.h +++ b/usr/src/uts/sun4u/sys/machparam.h @@ -320,6 +320,7 @@ extern "C" { #define PTL1_BAD_CTX_STEAL 12 #define PTL1_BAD_ECC 13 #define PTL1_BAD_CTX 14 +#define PTL1_BAD_RAISE_TSBEXCP 20 /* * Defines used for ptl1 related data structs. diff --git a/usr/src/uts/sun4u/sys/opl.h b/usr/src/uts/sun4u/sys/opl.h index 69fef17266..a7355249cd 100644 --- a/usr/src/uts/sun4u/sys/opl.h +++ b/usr/src/uts/sun4u/sys/opl.h @@ -67,6 +67,23 @@ extern "C" { (OPL_MAX_COREID_PER_CMP - 1)) #define STRAND_ID(x) ((uint_t)(x) & (OPL_MAX_STRANDID_PER_CORE - 1)) +/* + * Max. boards supported in a domain per model. + */ +#define OPL_MAX_BOARDS_FF1 1 +#define OPL_MAX_BOARDS_FF2 2 +#define OPL_MAX_BOARDS_DC1 4 +#define OPL_MAX_BOARDS_DC2 8 +#define OPL_MAX_BOARDS_DC3 16 + +/* + * Structure to gather model-specific information at boot. + */ +typedef struct opl_model_info { + char model_name[MAXSYSNAME]; + int model_max_boards; +} opl_model_info_t; + extern int plat_max_boards(void); extern int plat_max_cpu_units_per_board(void); extern int plat_max_mem_units_per_board(void); diff --git a/usr/src/uts/sun4u/vm/mach_sfmmu.c b/usr/src/uts/sun4u/vm/mach_sfmmu.c index b9cf3266f0..fdd75209f6 100644 --- a/usr/src/uts/sun4u/vm/mach_sfmmu.c +++ b/usr/src/uts/sun4u/vm/mach_sfmmu.c @@ -204,7 +204,7 @@ sfmmu_clear_user_tsbs() va = utsb_vabase; end_va = va + tsb_slab_size; while (va < end_va) { - vtag_flushpage(va, KCONTEXT); + vtag_flushpage(va, (uint64_t)ksfmmup); va += MMU_PAGESIZE; } @@ -212,7 +212,7 @@ sfmmu_clear_user_tsbs() va = utsb4m_vabase; end_va = va + tsb_slab_size; while (va < end_va) { - vtag_flushpage(va, KCONTEXT); + vtag_flushpage(va, (uint64_t)ksfmmup); va += MMU_PAGESIZE; } } @@ -294,18 +294,18 @@ kdi_tlb_page_lock(caddr_t va, int do_dtlb) tte.tte_intlo = TTE_PFN_INTLO(pfn) | TTE_LCK_INT | TTE_CP_INT | TTE_PRIV_INT | TTE_HWWR_INT; - vtag_flushpage(va, KCONTEXT); + vtag_flushpage(va, (uint64_t)ksfmmup); - sfmmu_itlb_ld(va, KCONTEXT, &tte); + sfmmu_itlb_ld_kva(va, &tte); if (do_dtlb) - sfmmu_dtlb_ld(va, KCONTEXT, &tte); + sfmmu_dtlb_ld_kva(va, &tte); } /*ARGSUSED*/ void kdi_tlb_page_unlock(caddr_t va, int do_dtlb) { - vtag_flushpage(va, KCONTEXT); + vtag_flushpage(va, (uint64_t)ksfmmup); } /* clear user TSB information (applicable to hardware TSB walkers) */ diff --git a/usr/src/uts/sun4u/vm/mach_sfmmu_asm.s b/usr/src/uts/sun4u/vm/mach_sfmmu_asm.s index 0a68ce88f7..2bd7ee732f 100644 --- a/usr/src/uts/sun4u/vm/mach_sfmmu_asm.s +++ b/usr/src/uts/sun4u/vm/mach_sfmmu_asm.s @@ -61,25 +61,19 @@ /* * sfmmu related subroutines */ - -/* ARGSUSED */ -void -sfmmu_ctx_steal_tl1(uint64_t sctx, uint64_t rctx) -{} - /* ARGSUSED */ void -sfmmu_raise_tsb_exception(uint64_t sctx, uint64_t rctx) +sfmmu_raise_tsb_exception(uint64_t sfmmup, uint64_t rctx) {} /* ARGSUSED */ void -sfmmu_itlb_ld(caddr_t vaddr, int ctxnum, tte_t *tte) +sfmmu_itlb_ld_kva(caddr_t vaddr, tte_t *tte) {} /* ARGSUSED */ void -sfmmu_dtlb_ld(caddr_t vaddr, int ctxnum, tte_t *tte) +sfmmu_dtlb_ld_kva(caddr_t vaddr, tte_t *tte) {} int @@ -102,103 +96,113 @@ sfmmu_load_mmustate(sfmmu_t *sfmmup) } #else /* lint */ - + /* - * 1. If stealing ctx, flush all TLB entries whose ctx is ctx-being-stolen. - * 2. If processor is running in the ctx-being-stolen, set the - * context to the resv context. That is - * If processor in User-mode - pri/sec-ctx both set to ctx-being-stolen, - * change both pri/sec-ctx registers to resv ctx. - * If processor in Kernel-mode - pri-ctx is 0, sec-ctx is ctx-being-stolen, - * just change sec-ctx register to resv ctx. When it returns to - * kernel-mode, user_rtt will change pri-ctx. - * - * Note: For multiple page size TLB, no need to set page sizes for - * DEMAP context. + * Invalidate either the context of a specific victim or any process + * currently running on this CPU. * - * %g1 = ctx being stolen (victim) - * %g2 = invalid ctx to replace victim with + * %g1 = sfmmup whose ctx is being invalidated + * when called from sfmmu_wrap_around, %g1 == INVALID_CONTEXT + * Note %g1 is the only input argument used by this xcall handler. */ - ENTRY(sfmmu_ctx_steal_tl1) - /* - * Flush TLBs. - */ - set MMU_PCONTEXT, %g3 - set DEMAP_CTX_TYPE | DEMAP_PRIMARY, %g4 - ldxa [%g3]ASI_MMU_CTX, %g5 /* get pri-ctx */ - sethi %hi(FLUSH_ADDR), %g6 - stxa %g1, [%g3]ASI_MMU_CTX /* temporarily set our */ - /* pri-ctx to victim */ - stxa %g0, [%g4]ASI_DTLB_DEMAP /* flush DTLB */ - stxa %g0, [%g4]ASI_ITLB_DEMAP /* flush ITLB */ - stxa %g5, [%g3]ASI_MMU_CTX /* restore original pri-ctx */ - flush %g6 /* ensure stxa's committed */ - /* fall through to the code below */ - - /* - * We enter here if we're just raising a TSB miss - * exception, without switching MMU contexts. In - * this case, there is no need to flush the TLB. - */ - ALTENTRY(sfmmu_raise_tsb_exception) + ENTRY(sfmmu_raise_tsb_exception) ! - ! if (sec-ctx != victim) { + ! if (victim == INVALID_CONTEXT) { + ! if (sec-ctx > INVALID_CONTEXT) { + ! write INVALID_CONTEXT to sec-ctx + ! } + ! if (pri-ctx > INVALID_CONTEXT) { + ! write INVALID_CONTEXT to pri-ctx + ! } + ! } else if (current CPU tsbmiss->usfmmup != victim sfmmup) { ! return ! } else { - ! if (pri-ctx == victim) { + ! if (sec-ctx > INVALID_CONTEXT) ! write INVALID_CONTEXT to sec-ctx + ! + ! if (pri-ctx > INVALID_CONTEXT) ! write INVALID_CONTEXT to pri-ctx - ! } else { - ! write INVALID_CONTEXT to sec-ctx - ! } ! } ! - cmp %g1, NUM_LOCKED_CTXS - blt,a,pn %icc, ptl1_panic /* can't steal locked ctx */ - mov PTL1_BAD_CTX_STEAL, %g1 - set CTXREG_CTX_MASK, %g6 - set MMU_SCONTEXT, %g3 - ldxa [%g3]ASI_MMU_CTX, %g5 /* get sec-ctx */ - and %g5, %g6, %g5 - cmp %g5, %g1 /* is it the victim? */ - bne,pn %icc, 2f /* was our sec-ctx a victim? */ + + sethi %hi(ksfmmup), %g3 + ldx [%g3 + %lo(ksfmmup)], %g3 + cmp %g1, %g3 + be,a,pn %xcc, ptl1_panic /* can't invalidate kernel ctx */ + mov PTL1_BAD_RAISE_TSBEXCP, %g1 + + set INVALID_CONTEXT, %g2 + + cmp %g1, INVALID_CONTEXT + bne,pt %xcc, 1f /* called from wrap_around? */ + mov MMU_SCONTEXT, %g3 + + ldxa [%g3]ASI_MMU_CTX, %g5 /* %g5 = pgsz | sec-ctx */ + set CTXREG_CTX_MASK, %g4 + and %g5, %g4, %g5 /* %g5 = sec-ctx */ + cmp %g5, INVALID_CONTEXT /* kernel ctx or invald ctx? */ + ble,pn %xcc, 0f /* yes, no need to change */ + mov MMU_PCONTEXT, %g7 + + stxa %g2, [%g3]ASI_MMU_CTX /* set invalid ctx */ + membar #Sync + +0: + ldxa [%g7]ASI_MMU_CTX, %g5 /* get pgz | pri-ctx */ + and %g5, %g4, %g5 /* %g5 = pri-ctx */ + cmp %g5, INVALID_CONTEXT /* kernel ctx or invald ctx? */ + ble,pn %xcc, 2f /* yes, no need to change */ + nop + + stxa %g2, [%g7]ASI_MMU_CTX /* set pri-ctx to invalid */ + retry + +1: + /* %g3 = MMU_SCONTEXT */ + CPU_TSBMISS_AREA(%g5, %g6) /* load cpu tsbmiss area */ + ldx [%g5 + TSBMISS_UHATID], %g5 /* load usfmmup */ + + cmp %g5, %g1 /* hat toBe-invalid running? */ + bne,pt %xcc, 2f + nop + + ldxa [%g3]ASI_MMU_CTX, %g5 /* %g5 = pgsz | sec-ctx */ + set CTXREG_CTX_MASK, %g4 + and %g5, %g4, %g5 /* %g5 = sec-ctx */ + cmp %g5, INVALID_CONTEXT /* kernel or invalid ctx ? */ + ble,pn %xcc, 0f /* yes, no need to change */ mov MMU_PCONTEXT, %g7 - ldxa [%g7]ASI_MMU_CTX, %g4 /* get pri-ctx */ - and %g4, %g6, %g4 - stxa %g2, [%g3]ASI_MMU_CTX /* set sec-ctx to invalid ctx */ + + stxa %g2, [%g3]ASI_MMU_CTX /* set sec-ctx to invalid */ membar #Sync - cmp %g1, %g4 /* is it the victim? */ - bne %icc, 2f /* nope, no need to change it */ + +0: + ldxa [%g7]ASI_MMU_CTX, %g4 /* %g4 = pgsz | pri-ctx */ + set CTXREG_CTX_MASK, %g6 + and %g4, %g6, %g4 /* %g4 = pri-ctx */ + cmp %g4, INVALID_CONTEXT /* is pri-ctx the victim? */ + ble %icc, 2f /* no, no need to change it */ nop - stxa %g2, [%g7]ASI_MMU_CTX /* set pri-ctx to invalid ctx */ + stxa %g2, [%g7]ASI_MMU_CTX /* set pri-ctx to invalid */ /* next instruction is retry so no membar sync */ 2: retry - SET_SIZE(sfmmu_ctx_steal_tl1) + SET_SIZE(sfmmu_raise_tsb_exception) - ENTRY_NP(sfmmu_itlb_ld) + /* + * %o0 = virtual address + * %o1 = address of TTE to be loaded + */ + ENTRY_NP(sfmmu_itlb_ld_kva) rdpr %pstate, %o3 #ifdef DEBUG - andcc %o3, PSTATE_IE, %g0 ! If interrupts already - bnz,pt %icc, 1f ! disabled, panic - nop - - sethi %hi(panicstr), %g1 - ldx [%g1 + %lo(panicstr)], %g1 - tst %g1 - bnz,pt %icc, 1f - nop - - sethi %hi(sfmmu_panic1), %o0 - call panic - or %o0, %lo(sfmmu_panic1), %o0 -1: + PANIC_IF_INTR_DISABLED_PSTR(%o3, msfmmu_di_l1, %g1) #endif /* DEBUG */ wrpr %o3, PSTATE_IE, %pstate ! Disable interrupts srln %o0, MMU_PAGESHIFT, %o0 slln %o0, MMU_PAGESHIFT, %o0 ! Clear page offset - or %o0, %o1, %o0 - ldx [%o2], %g1 + + ldx [%o1], %g1 set MMU_TAG_ACCESS, %o5 #ifdef CHEETAHPLUS_ERRATUM_34 ! @@ -241,7 +245,7 @@ sfmmu_load_mmustate(sfmmu_t *sfmmup) call panic ! found no unlocked TTE so or %o0, %lo(sfmmu_panic5), %o0 ! give up. - + 2: ! ! We have found an unlocked or non-valid entry; we'll explicitly load @@ -252,7 +256,7 @@ sfmmu_load_mmustate(sfmmu_t *sfmmup) stxa %g1, [%g3]ASI_ITLB_ACCESS flush %o1 ! Flush required for I-MMU ba 3f ! Delay slot of ba is empty - nop ! per Erratum 64 + nop ! per Erratum 64 0: #endif /* CHEETAHPLUS_ERRATUM_34 */ @@ -263,7 +267,7 @@ sfmmu_load_mmustate(sfmmu_t *sfmmup) 3: retl wrpr %g0, %o3, %pstate ! Enable interrupts - SET_SIZE(sfmmu_itlb_ld) + SET_SIZE(sfmmu_itlb_ld_kva) /* * Load an entry into the DTLB. @@ -272,36 +276,33 @@ sfmmu_load_mmustate(sfmmu_t *sfmmup) * are some TLB slots that are reserved for the kernel but not * always held locked. We want to avoid loading locked TTEs * into those slots since they could be displaced. + * + * %o0 = virtual address + * %o1 = address of TTE to be loaded */ - ENTRY_NP(sfmmu_dtlb_ld) + ENTRY_NP(sfmmu_dtlb_ld_kva) rdpr %pstate, %o3 #ifdef DEBUG - andcc %o3, PSTATE_IE, %g0 ! if interrupts already - bnz,pt %icc, 1f ! disabled, panic - nop - - sethi %hi(panicstr), %g1 - ldx [%g1 + %lo(panicstr)], %g1 - tst %g1 - bnz,pt %icc, 1f - nop - - sethi %hi(sfmmu_panic1), %o0 - call panic - or %o0, %lo(sfmmu_panic1), %o0 -1: + PANIC_IF_INTR_DISABLED_PSTR(%o3, msfmmu_di_l2, %g1) #endif /* DEBUG */ wrpr %o3, PSTATE_IE, %pstate ! disable interrupts srln %o0, MMU_PAGESHIFT, %o0 slln %o0, MMU_PAGESHIFT, %o0 ! clear page offset - or %o0, %o1, %o0 ! or in ctx to form tagacc - ldx [%o2], %g1 - sethi %hi(ctx_pgsz_array), %o2 ! Check for T8s - ldn [%o2 + %lo(ctx_pgsz_array)], %o2 - brz %o2, 1f + + ldx [%o1], %g1 + set MMU_TAG_ACCESS, %o5 - ldub [%o2 + %o1], %o2 ! Cheetah+: set up tag access - sll %o2, TAGACCEXT_SHIFT, %o2 ! extension register so entry + + set cpu_impl_dual_pgsz, %o2 + ld [%o2], %o2 + brz %o2, 1f + nop + + sethi %hi(ksfmmup), %o2 + ldx [%o2 + %lo(ksfmmup)], %o2 + ldub [%o2 + SFMMU_CEXT], %o2 + sll %o2, TAGACCEXT_SHIFT, %o2 + set MMU_TAG_ACCESS_EXT, %o4 ! can go into T8 if unlocked stxa %o2,[%o4]ASI_DMMU membar #Sync @@ -330,7 +331,7 @@ sfmmu_load_mmustate(sfmmu_t *sfmmup) bz,pn %icc, 4f ! If unlocked, go displace nop sub %g3, (1 << 3), %g3 ! Decrement idx - brgez %g3, 3b + brgez %g3, 3b nop sethi %hi(sfmmu_panic5), %o0 ! We searched all entries and call panic ! found no unlocked TTE so @@ -345,7 +346,7 @@ sfmmu_load_mmustate(sfmmu_t *sfmmup) membar #Sync retl wrpr %g0, %o3, %pstate ! enable interrupts - SET_SIZE(sfmmu_dtlb_ld) + SET_SIZE(sfmmu_dtlb_ld_kva) ENTRY_NP(sfmmu_getctx_pri) set MMU_PCONTEXT, %o0 @@ -358,45 +359,43 @@ sfmmu_load_mmustate(sfmmu_t *sfmmup) set CTXREG_CTX_MASK, %o1 ldxa [%o0]ASI_MMU_CTX, %o0 retl - and %o0, %o1, %o0 + and %o0, %o1, %o0 SET_SIZE(sfmmu_getctx_sec) /* * Set the secondary context register for this process. - * %o0 = context number for this process. + * %o0 = page_size | context number for this process. */ ENTRY_NP(sfmmu_setctx_sec) /* * From resume we call sfmmu_setctx_sec with interrupts disabled. * But we can also get called from C with interrupts enabled. So, - * we need to check first. Also, resume saves state in %o3 and %o5 - * so we can't use those registers here. + * we need to check first. */ /* If interrupts are not disabled, then disable them */ rdpr %pstate, %g1 btst PSTATE_IE, %g1 bnz,a,pt %icc, 1f - wrpr %g1, PSTATE_IE, %pstate /* disable interrupts */ + wrpr %g1, PSTATE_IE, %pstate /* disable interrupts */ + 1: mov MMU_SCONTEXT, %o1 - sethi %hi(ctx_pgsz_array), %g2 - ldn [%g2 + %lo(ctx_pgsz_array)], %g2 - brz %g2, 2f - nop - ldub [%g2 + %o0], %g2 - sll %g2, CTXREG_EXT_SHIFT, %g2 - or %g2, %o0, %o0 -2: + sethi %hi(FLUSH_ADDR), %o4 stxa %o0, [%o1]ASI_MMU_CTX /* set 2nd context reg. */ flush %o4 + /* + * if the routine was entered with intr enabled, then enable intr now. + * otherwise, keep intr disabled, return without enabing intr. + * %g1 - old intr state + */ btst PSTATE_IE, %g1 - bnz,a,pt %icc, 1f - wrpr %g0, %g1, %pstate /* enable interrupts */ -1: retl - nop + bnz,a,pt %icc, 2f + wrpr %g0, %g1, %pstate /* enable interrupts */ +2: retl + nop SET_SIZE(sfmmu_setctx_sec) /* @@ -432,25 +431,17 @@ sfmmu_load_mmustate(sfmmu_t *sfmmup) * %o0 - hat pointer */ ENTRY_NP(sfmmu_load_mmustate) - /* - * From resume we call sfmmu_load_mmustate with interrupts disabled. - * But we can also get called from C with interrupts enabled. So, - * we need to check first. Also, resume saves state in %o5 and we - * can't use this register here. - */ - sethi %hi(ksfmmup), %o3 +#ifdef DEBUG + PANIC_IF_INTR_ENABLED_PSTR(msfmmu_ei_l3, %g1) +#endif /* DEBUG */ + + sethi %hi(ksfmmup), %o3 ldx [%o3 + %lo(ksfmmup)], %o3 cmp %o3, %o0 be,pn %xcc, 3f ! if kernel as, do nothing nop - /* If interrupts are not disabled, then disable them */ - rdpr %pstate, %g1 - btst PSTATE_IE, %g1 - bnz,a,pt %icc, 1f - wrpr %g1, PSTATE_IE, %pstate ! disable interrupts -1: /* * We need to set up the TSB base register, tsbmiss * area, and load locked TTE(s) for the TSB. @@ -477,7 +468,7 @@ sfmmu_load_mmustate(sfmmu_t *sfmmup) brz,pt %g2, 4f nop /* - * We have a second TSB for this process, so we need to + * We have a second TSB for this process, so we need to * encode data for both the first and second TSB in our single * TSB base register. See hat_sfmmu.h for details on what bits * correspond to which TSB. @@ -522,11 +513,8 @@ sfmmu_load_mmustate(sfmmu_t *sfmmup) stx %o0, [%o2 + TSBMISS_UHATID] stuh %o3, [%o2 + TSBMISS_HATFLAGS] - btst PSTATE_IE, %g1 - bnz,a,pt %icc, 3f - wrpr %g0, %g1, %pstate ! enable interrupts 3: retl - nop + nop SET_SIZE(sfmmu_load_mmustate) #endif /* lint */ @@ -624,7 +612,7 @@ sfmmu_inv_tsb_fast(caddr_t tsb_base, uint_t tsb_bytes) dec %l3 stb %l3, [THREAD_REG + T_PREEMPT] ret - restore + restore SET_SIZE(sfmmu_inv_tsb_fast) #endif /* lint */ @@ -656,12 +644,12 @@ prefetch_tsbe_write(struct tsbe *tsbep) ENTRY(prefetch_tsbe_read) retl - prefetch [%o0+448], #n_reads + prefetch [%o0+448], #n_reads SET_SIZE(prefetch_tsbe_read) ENTRY(prefetch_tsbe_write) retl - prefetch [%o0], #n_writes + prefetch [%o0], #n_writes SET_SIZE(prefetch_tsbe_write) #endif /* lint */ diff --git a/usr/src/uts/sun4v/cpu/common_asm.s b/usr/src/uts/sun4v/cpu/common_asm.s index 8de96b3bed..ee5b3784f7 100644 --- a/usr/src/uts/sun4v/cpu/common_asm.s +++ b/usr/src/uts/sun4v/cpu/common_asm.s @@ -77,6 +77,7 @@ #include <sys/asm_linkage.h> #include <sys/privregs.h> +#include <vm/hat_sfmmu.h> #include <sys/machparam.h> /* To get SYSBASE and PAGESIZE */ #include <sys/machthread.h> #include <sys/clock.h> @@ -118,12 +119,12 @@ tickcmpr_set(uint64_t clock_cycles) cmp %o2, %o0 ! If the value we wrote was in the bg,pt %xcc, 2f ! future, then blow out of here. - sllx %o3, 1, %o3 ! If not, then double our step size, + sllx %o3, 1, %o3 ! If not, then double our step size, ba,pt %xcc, 1b ! and take another lap. - add %o0, %o3, %o2 ! + add %o0, %o3, %o2 ! 2: retl - nop + nop SET_SIZE(tickcmpr_set) #endif /* lint */ @@ -141,7 +142,7 @@ tickcmpr_disable(void) sllx %g1, TICKINT_DIS_SHFT, %o0 WR_TICKCMPR(%o0,%o4,%o5,__LINE__) ! Write to TICK_CMPR retl - nop + nop SET_SIZE(tickcmpr_disable) #endif @@ -174,10 +175,10 @@ tick_write_delta_panic: sethi %hi(tick_write_delta_panic), %o1 save %sp, -SA(MINFRAME), %sp ! get a new window to preserve caller call panic - or %i1, %lo(tick_write_delta_panic), %o0 + or %i1, %lo(tick_write_delta_panic), %o0 /*NOTREACHED*/ retl - nop + nop #endif #if defined(lint) @@ -194,7 +195,7 @@ tickcmpr_disabled(void) ENTRY_NP(tickcmpr_disabled) RD_TICKCMPR(%g1, %o0) retl - srlx %g1, TICKINT_DIS_SHFT, %o0 + srlx %g1, TICKINT_DIS_SHFT, %o0 SET_SIZE(tickcmpr_disabled) #endif /* lint */ @@ -213,7 +214,7 @@ gettick(void) ENTRY(gettick) GET_NATIVE_TIME(%o0, %o2, %o3) retl - nop + nop SET_SIZE(gettick) #endif /* lint */ @@ -235,7 +236,7 @@ gettick_counter(void) rdpr %tick, %o0 sllx %o0, 1, %o0 retl - srlx %o0, 1, %o0 ! shake off npt bit + srlx %o0, 1, %o0 ! shake off npt bit SET_SIZE(gettick_counter) #endif /* lint */ @@ -307,13 +308,13 @@ panic_hres_tick(void) GET_HRTIME(%g1, %o0, %o1, %o2, %o3, %o4, %o5, %g2) ! %g1 = hrtime retl - mov %g1, %o0 + mov %g1, %o0 SET_SIZE(gethrtime) ENTRY_NP(gethrtime_unscaled) GET_NATIVE_TIME(%g1, %o2, %o3) ! %g1 = native time retl - mov %g1, %o0 + mov %g1, %o0 SET_SIZE(gethrtime_unscaled) ENTRY_NP(gethrtime_waitfree) @@ -321,7 +322,7 @@ panic_hres_tick(void) GET_NATIVE_TIME(%g1, %o2, %o3) ! %g1 = native time NATIVE_TIME_TO_NSEC(%g1, %o2, %o3) retl - mov %g1, %o0 + mov %g1, %o0 SET_SIZE(dtrace_gethrtime) SET_SIZE(gethrtime_waitfree) @@ -332,17 +333,17 @@ panic_hres_tick(void) ! hrtime_t's are signed, max hrtime_t must be positive mov -1, %o2 brlz,a %g1, 1f - srlx %o2, 1, %g1 + srlx %o2, 1, %g1 1: retl - mov %g1, %o0 + mov %g1, %o0 SET_SIZE(gethrtime_max) ENTRY(scalehrtime) ldx [%o0], %o1 NATIVE_TIME_TO_NSEC(%o1, %o2, %o3) retl - stx %o1, [%o0] + stx %o1, [%o0] SET_SIZE(scalehrtime) /* @@ -369,18 +370,18 @@ panic_hres_tick(void) brz,pt adj, 3f; /* no adjustments, it's easy */ \ add hrestnsec, nslt, hrestnsec; /* hrest.tv_nsec += nslt */ \ brlz,pn adj, 2f; /* if hrestime_adj negative */ \ - srl nslt, ADJ_SHIFT, nslt; /* delay: nslt >>= 4 */ \ + srl nslt, ADJ_SHIFT, nslt; /* delay: nslt >>= 4 */ \ subcc adj, nslt, %g0; /* hrestime_adj - nslt/16 */ \ movg %xcc, nslt, adj; /* adj by min(adj, nslt/16) */ \ ba 3f; /* go convert to sec/nsec */ \ - add hrestnsec, adj, hrestnsec; /* delay: apply adjustment */ \ + add hrestnsec, adj, hrestnsec; /* delay: apply adjustment */ \ 2: addcc adj, nslt, %g0; /* hrestime_adj + nslt/16 */ \ bge,a,pt %xcc, 3f; /* is adj less negative? */ \ - add hrestnsec, adj, hrestnsec; /* yes: hrest.nsec += adj */ \ + add hrestnsec, adj, hrestnsec; /* yes: hrest.nsec += adj */ \ sub hrestnsec, nslt, hrestnsec; /* no: hrest.nsec -= nslt/16 */ \ 3: cmp hrestnsec, nano; /* more than a billion? */ \ bl,pt %xcc, 4f; /* if not, we're done */ \ - nop; /* delay: do nothing :( */ \ + nop; /* delay: do nothing :( */ \ add hrestsec, 1, hrestsec; /* hrest.tv_sec++; */ \ sub hrestnsec, nano, hrestnsec; /* hrest.tv_nsec -= NANOSEC; */ \ 4: @@ -390,7 +391,7 @@ panic_hres_tick(void) CONV_HRESTIME(%o1, %o2, %o3, %o4, %o5) stn %o1, [%o0] retl - stn %o2, [%o0 + CLONGSIZE] + stn %o2, [%o0 + CLONGSIZE] SET_SIZE(gethrestime) /* @@ -401,7 +402,7 @@ panic_hres_tick(void) GET_HRESTIME(%o0, %o2, %o3, %o4, %o5, %g1, %g2, %g3, %g4) CONV_HRESTIME(%o0, %o2, %o3, %o4, %o5) retl ! %o0 current hrestime seconds - nop + nop SET_SIZE(gethrestime_sec) /* @@ -425,9 +426,9 @@ panic_hres_tick(void) lduw [%o1 + %lo(hres_lock)], %o3 ! Reload lock value cmp %o3, %o2 ! If lock is locked or has bne 0b ! changed, retry. - stn %g1, [%o0] ! Delay: store seconds + stn %g1, [%o0] ! Delay: store seconds retl - stn %g2, [%o0 + CLONGSIZE] ! Delay: store nanoseconds + stn %g2, [%o0 + CLONGSIZE] ! Delay: store nanoseconds SET_SIZE(gethrestime_lasttick) /* @@ -496,13 +497,13 @@ hrtime_base_panic: ldstub [%l4 + %lo(hres_lock + HRES_LOCK_OFFSET)], %l5 ! try locking 7: tst %l5 bz,pt %xcc, 8f ! if we got it, drive on - ld [%l4 + %lo(nsec_scale)], %l5 ! delay: %l5 = scaling factor + ld [%l4 + %lo(nsec_scale)], %l5 ! delay: %l5 = scaling factor ldub [%l4 + %lo(hres_lock + HRES_LOCK_OFFSET)], %l5 9: tst %l5 bz,a,pn %xcc, 7b - ldstub [%l4 + %lo(hres_lock + HRES_LOCK_OFFSET)], %l5 + ldstub [%l4 + %lo(hres_lock + HRES_LOCK_OFFSET)], %l5 ba,pt %xcc, 9b - ldub [%l4 + %lo(hres_lock + HRES_LOCK_OFFSET)], %l5 + ldub [%l4 + %lo(hres_lock + HRES_LOCK_OFFSET)], %l5 8: membar #StoreLoad|#StoreStore @@ -520,7 +521,7 @@ hrtime_base_panic: ldx [%l4 + %lo(hrtime_base)], %l1 cmp %l1, %l0 bg,pn %xcc, 9f - nop + nop stx %l0, [%l4 + %lo(hrtime_base)] ! update hrtime_base @@ -531,20 +532,20 @@ hrtime_base_panic: brz %l0, 2f ! hrestime_adj == 0 ? ! yes, skip adjustments - clr %l5 ! delay: set adj to zero + clr %l5 ! delay: set adj to zero tst %l0 ! is hrestime_adj >= 0 ? bge,pt %xcc, 1f ! yes, go handle positive case - srl %i1, ADJ_SHIFT, %l5 ! delay: %l5 = adj + srl %i1, ADJ_SHIFT, %l5 ! delay: %l5 = adj addcc %l0, %l5, %g0 ! hrestime_adj < -adj ? bl,pt %xcc, 2f ! yes, use current adj - neg %l5 ! delay: %l5 = -adj + neg %l5 ! delay: %l5 = -adj ba,pt %xcc, 2f - mov %l0, %l5 ! no, so set adj = hrestime_adj + mov %l0, %l5 ! no, so set adj = hrestime_adj 1: subcc %l0, %l5, %g0 ! hrestime_adj < adj ? bl,a,pt %xcc, 2f ! yes, set adj = hrestime_adj - mov %l0, %l5 ! delay: adj = hrestime_adj + mov %l0, %l5 ! delay: adj = hrestime_adj 2: ldx [%l4 + %lo(timedelta)], %l0 ! %l0 = timedelta sub %l0, %l5, %l0 ! timedelta -= adj @@ -561,7 +562,7 @@ hrtime_base_panic: set NANOSEC, %l5 ! %l5 = NANOSEC cmp %i3, %l5 bl,pt %xcc, 5f ! if hrestime.tv_nsec < NANOSEC - sethi %hi(one_sec), %i1 ! delay + sethi %hi(one_sec), %i1 ! delay add %i2, 0x1, %i2 ! hrestime.tv_sec++ sub %i3, %l5, %i3 ! hrestime.tv_nsec - NANOSEC mov 0x1, %l5 @@ -589,7 +590,7 @@ hrtime_base_panic: sethi %hi(hrtime_base_panic), %o0 call panic - or %o0, %lo(hrtime_base_panic), %o0 + or %o0, %lo(hrtime_base_panic), %o0 SET_SIZE(hres_tick) @@ -605,7 +606,7 @@ kstat_q_panic_msg: save %sp, -SA(MINFRAME), %sp sethi %hi(kstat_q_panic_msg), %o0 call panic - or %o0, %lo(kstat_q_panic_msg), %o0 + or %o0, %lo(kstat_q_panic_msg), %o0 /*NOTREACHED*/ SET_SIZE(kstat_q_panic) @@ -750,7 +751,7 @@ usec_delay(int n) ENTRY(drv_usecwait) ALTENTRY(usec_delay) brlez,a,pn %o0, 0f - mov 1, %o0 + mov 1, %o0 0: sethi %hi(sticks_per_usec), %o1 lduw [%o1 + %lo(sticks_per_usec)], %o1 @@ -762,9 +763,9 @@ usec_delay(int n) 1: cmp %o1, %o2 GET_NATIVE_TIME(%o2, %o3, %o4) bgeu,pt %xcc, 1b - nop + nop retl - nop + nop SET_SIZE(usec_delay) SET_SIZE(drv_usecwait) #endif /* lint */ @@ -789,12 +790,12 @@ pil14_interrupt(int level) rdpr %tpc, %g5 btst TSTATE_PRIV, %g6 ! trap from supervisor mode? bnz,a,pt %xcc, 1f - stn %g5, [%g1 + CPU_PROFILE_PC] ! if so, record kernel PC + stn %g5, [%g1 + CPU_PROFILE_PC] ! if so, record kernel PC stn %g5, [%g1 + CPU_PROFILE_UPC] ! if not, record user PC ba pil_interrupt_common ! must be large-disp branch - stn %g0, [%g1 + CPU_PROFILE_PC] ! zero kernel PC + stn %g0, [%g1 + CPU_PROFILE_PC] ! zero kernel PC 1: ba pil_interrupt_common ! must be large-disp branch - stn %g0, [%g1 + CPU_PROFILE_UPC] ! zero user PC + stn %g0, [%g1 + CPU_PROFILE_UPC] ! zero user PC SET_SIZE(pil14_interrupt) ENTRY_NP(tick_rtt) @@ -813,7 +814,7 @@ pil14_interrupt(int level) RD_TICKCMPR(%o5, %g1) srlx %o5, TICKINT_DIS_SHFT, %g1 brnz,pt %g1, 2f - nop + nop rdpr %pstate, %g5 andn %g5, PSTATE_IE, %g1 @@ -822,14 +823,14 @@ pil14_interrupt(int level) sethi %hi(cbe_level14_inum), %o1 ld [%o1 + %lo(cbe_level14_inum)], %o1 call intr_enqueue_req ! preserves %o5 and %g5 - mov PIL_14, %o0 + mov PIL_14, %o0 ! Check SOFTINT for TICKINT/STICKINT rd SOFTINT, %o4 set (TICK_INT_MASK | STICK_INT_MASK), %o0 andcc %o4, %o0, %g0 bz,a,pn %icc, 2f - wrpr %g0, %g5, %pstate ! Enable vec interrupts + wrpr %g0, %g5, %pstate ! Enable vec interrupts ! clear TICKINT/STICKINT wr %o0, CLEAR_SOFTINT @@ -844,7 +845,7 @@ pil14_interrupt(int level) srlx %o0, 1, %o0 cmp %o5, %o0 ! In the future? bg,a,pt %xcc, 2f ! Yes, drive on. - wrpr %g0, %g5, %pstate ! delay: enable vec intr + wrpr %g0, %g5, %pstate ! delay: enable vec intr ! ! If we're here, then we have programmed TICK_COMPARE with a %tick @@ -859,12 +860,12 @@ pil14_interrupt(int level) srlx %o0, 1, %o0 cmp %o5, %o0 ! In the future? bg,a,pt %xcc, 2f ! Yes, drive on. - wrpr %g0, %g5, %pstate ! delay: enable vec intr + wrpr %g0, %g5, %pstate ! delay: enable vec intr ba 1b ! No, try again. - sllx %o4, 1, %o4 ! delay: double step size + sllx %o4, 1, %o4 ! delay: double step size 2: ba current_thread_complete - nop + nop SET_SIZE(tick_rtt) #endif /* lint */ @@ -888,12 +889,12 @@ prefetch_page_r(void *pp) /* XXXQ These should be inline templates, not functions */ ENTRY(prefetch_page_w) retl - nop + nop SET_SIZE(prefetch_page_w) ENTRY(prefetch_page_r) retl - nop + nop SET_SIZE(prefetch_page_r) #endif /* lint */ @@ -911,7 +912,7 @@ prefetch_smap_w(void *smp) /* XXXQ These should be inline templates, not functions */ ENTRY(prefetch_smap_w) retl - nop + nop SET_SIZE(prefetch_smap_w) #endif /* lint */ @@ -924,12 +925,7 @@ prefetch_smap_w(void *smp) /*ARGSUSED*/ void -vtag_flushpage(caddr_t vaddr, uint_t ctxnum) -{} - -/*ARGSUSED*/ -void -vtag_flushctx(uint_t ctxnum) +vtag_flushpage(caddr_t vaddr, uint64_t sfmmup) {} /*ARGSUSED*/ @@ -944,17 +940,12 @@ vtag_unmap_perm_tl1(uint64_t vaddr, uint64_t ctxnum) /*ARGSUSED*/ void -vtag_flushpage_tl1(uint64_t vaddr, uint64_t ctxnum) -{} - -/*ARGSUSED*/ -void -vtag_flush_pgcnt_tl1(uint64_t vaddr, uint64_t ctx_pgcnt) +vtag_flushpage_tl1(uint64_t vaddr, uint64_t sfmmup) {} /*ARGSUSED*/ void -vtag_flushctx_tl1(uint64_t ctxnum, uint64_t dummy) +vtag_flush_pgcnt_tl1(uint64_t vaddr, uint64_t sfmmup_pgcnt) {} /*ARGSUSED*/ @@ -984,39 +975,21 @@ flush_instr_mem(caddr_t vaddr, size_t len) * flush page from the tlb * * %o0 = vaddr - * %o1 = ctxnum + * %o1 = sfmmup */ + SFMMU_CPU_CNUM(%o1, %g1, %g2) /* %g1 = sfmmu cnum on this CPU */ + + mov %g1, %o1 mov MAP_ITLB | MAP_DTLB, %o2 ta MMU_UNMAP_ADDR brz,pt %o0, 1f - nop + nop ba panic_bad_hcall - mov MMU_UNMAP_ADDR, %o1 + mov MMU_UNMAP_ADDR, %o1 1: retl - nop - SET_SIZE(vtag_flushpage) - - ENTRY_NP(vtag_flushctx) - /* - * flush context from the tlb - * - * %o0 = ctxnum - */ - mov %o0, %o2 - mov %g0, %o0 ! XXXQ no cpu list yet - mov %g0, %o1 ! XXXQ no cpu list yet - mov MAP_ITLB | MAP_DTLB, %o3 - mov MMU_DEMAP_CTX, %o5 - ta FAST_TRAP - brz,pt %o0, 1f - nop - ba panic_bad_hcall - mov MMU_DEMAP_CTX, %o1 -1: - retl nop - SET_SIZE(vtag_flushctx) + SET_SIZE(vtag_flushpage) ENTRY_NP(vtag_flushall) mov %g0, %o0 ! XXX no cpu list yet @@ -1025,19 +998,19 @@ flush_instr_mem(caddr_t vaddr, size_t len) mov MMU_DEMAP_ALL, %o5 ta FAST_TRAP brz,pt %o0, 1f - nop + nop ba panic_bad_hcall - mov MMU_DEMAP_ALL, %o1 + mov MMU_DEMAP_ALL, %o1 1: retl - nop + nop SET_SIZE(vtag_flushall) ENTRY_NP(vtag_unmap_perm_tl1) /* * x-trap to unmap perm map entry * %g1 = vaddr - * %g2 = ctxnum + * %g2 = ctxnum (KCONTEXT only) */ mov %o0, %g3 mov %o1, %g4 @@ -1073,7 +1046,7 @@ flush_instr_mem(caddr_t vaddr, size_t len) * x-trap to flush page from tlb and tsb * * %g1 = vaddr, zero-extended on 32-bit kernel - * %g2 = ctxnum + * %g2 = sfmmup * * assumes TSBE_TAG = 0 */ @@ -1082,13 +1055,15 @@ flush_instr_mem(caddr_t vaddr, size_t len) mov %o0, %g3 mov %o1, %g4 mov %o2, %g5 - mov %g1, %o0 ! vaddr - mov %g2, %o1 ! ctx + mov %g1, %o0 /* vaddr */ + + SFMMU_CPU_CNUM(%g2, %o1, %g6) /* %o1 = sfmmu cnum on this CPU */ + mov MAP_ITLB | MAP_DTLB, %o2 ta MMU_UNMAP_ADDR brz,pt %o0, 1f nop - ba ptl1_panic + ba ptl1_panic mov PTL1_BAD_HCALL, %g1 1: mov %g5, %o2 @@ -1103,7 +1078,7 @@ flush_instr_mem(caddr_t vaddr, size_t len) * x-trap to flush pgcnt MMU_PAGESIZE pages from tlb * * %g1 = vaddr, zero-extended on 32-bit kernel - * %g2 = <zero32|ctx16|pgcnt16> + * %g2 = <sfmmup58|pgcnt6>, (pgcnt - 1) is pass'ed in via pgcnt6 bits. * * NOTE: this handler relies on the fact that no * interrupts or traps can occur during the loop @@ -1119,24 +1094,28 @@ flush_instr_mem(caddr_t vaddr, size_t len) mov %o1, %g4 mov %o2, %g5 - set 0xffff, %g6 - and %g6, %g2, %g7 /* g7 = pgcnt */ - srln %g2, 16, %g2 /* g2 = ctxnum */ + and %g2, SFMMU_PGCNT_MASK, %g7 /* g7 = pgcnt - 1 */ + add %g7, 1, %g7 /* g7 = pgcnt */ - set MMU_PAGESIZE, %g6 /* g2 = pgsize */ -1: - mov %g1, %o0 ! vaddr - mov %g2, %o1 ! ctx + andn %g2, SFMMU_PGCNT_MASK, %o0 /* %o0 = sfmmup */ + + SFMMU_CPU_CNUM(%o0, %g2, %g6) /* %g2 = sfmmu cnum on this CPU */ + + set MMU_PAGESIZE, %g6 /* g6 = pgsize */ + +1: + mov %g1, %o0 /* vaddr */ + mov %g2, %o1 /* cnum */ mov MAP_ITLB | MAP_DTLB, %o2 ta MMU_UNMAP_ADDR brz,pt %o0, 2f - nop + nop ba ptl1_panic - mov PTL1_BAD_HCALL, %g1 + mov PTL1_BAD_HCALL, %g1 2: deccc %g7 /* decr pgcnt */ bnz,pt %icc,1b - add %g1, %g6, %g1 /* go to nextpage */ + add %g1, %g6, %g1 /* go to nextpage */ mov %g5, %o2 mov %g4, %o1 @@ -1145,37 +1124,6 @@ flush_instr_mem(caddr_t vaddr, size_t len) retry SET_SIZE(vtag_flush_pgcnt_tl1) - ENTRY_NP(vtag_flushctx_tl1) - /* - * x-trap to flush context from tlb - * - * %g1 = ctxnum - */ - mov %o0, %g3 - mov %o1, %g4 - mov %o2, %g5 - mov %o3, %g6 - mov %o5, %g7 - mov %g1, %o2 - mov %g0, %o0 ! XXXQ no cpu list yet - mov %g0, %o1 ! XXXQ no cpu list yet - mov MAP_ITLB | MAP_DTLB, %o3 - mov MMU_DEMAP_CTX, %o5 - ta FAST_TRAP - brz,pt %o0, 1f - nop - ba ptl1_panic - mov PTL1_BAD_HCALL, %g1 -1: - mov %g7, %o5 - mov %g6, %o3 - mov %g5, %o2 - mov %g4, %o1 - mov %g3, %o0 - membar #Sync - retry - SET_SIZE(vtag_flushctx_tl1) - ! Not implemented on US1/US2 ENTRY_NP(vtag_flushall_tl1) mov %o0, %g3 @@ -1189,9 +1137,9 @@ flush_instr_mem(caddr_t vaddr, size_t len) mov MMU_DEMAP_ALL, %o5 ta FAST_TRAP brz,pt %o0, 1f - nop + nop ba ptl1_panic - mov PTL1_BAD_HCALL, %g1 + mov PTL1_BAD_HCALL, %g1 1: mov %g7, %o5 mov %g6, %o3 ! XXXQ not used? @@ -1220,7 +1168,7 @@ flush_instr_mem(caddr_t vaddr, size_t len) */ ! XXXQ retl - nop + nop SET_SIZE(vac_flushpage) ENTRY_NP(vac_flushpage_tl1) @@ -1264,10 +1212,10 @@ flush_instr_mem(caddr_t vaddr, size_t len) flush %o0 subcc %o1, ICACHE_FLUSHSZ, %o1 ! bytes = bytes-0x20 bgu,pt %ncc, 1b - add %o0, ICACHE_FLUSHSZ, %o0 ! vaddr = vaddr+0x20 + add %o0, ICACHE_FLUSHSZ, %o0 ! vaddr = vaddr+0x20 retl - nop + nop SET_SIZE(flush_instr_mem) #endif /* !lint */ diff --git a/usr/src/uts/sun4v/ml/mach_offsets.in b/usr/src/uts/sun4v/ml/mach_offsets.in index 1770b73bb2..b58ed79949 100644 --- a/usr/src/uts/sun4v/ml/mach_offsets.in +++ b/usr/src/uts/sun4v/ml/mach_offsets.in @@ -134,12 +134,14 @@ trap_trace_record TRAP_ENT_SIZE htrap_trace_record HTRAP_ENT_SIZE hat HAT_SIZE - sfmmu_cnum sfmmu_cpusran sfmmu_tsb sfmmu_ismblkpa sfmmu_flags sfmmu_hvblock + sfmmu_cext + sfmmu_ctx_lock + sfmmu_ctxs sfmmu_global_stat HATSTAT_SIZE sf_pagefaults HATSTAT_PAGEFAULT @@ -148,9 +150,6 @@ sfmmu_global_stat HATSTAT_SIZE sf_khash_searches HATSTAT_KHASH_SEARCH sf_khash_links HATSTAT_KHASH_LINKS -ctx CTX_SIZE CTX_SZ_SHIFT - ctx_un.ctx_sfmmup CTX_SFMMUP - sf_hment SFHME_SIZE SFHME_SHIFT hme_tte SFHME_TTE diff --git a/usr/src/uts/sun4v/os/fillsysinfo.c b/usr/src/uts/sun4v/os/fillsysinfo.c index 173019f902..1a3e63b056 100644 --- a/usr/src/uts/sun4v/os/fillsysinfo.c +++ b/usr/src/uts/sun4v/os/fillsysinfo.c @@ -339,17 +339,11 @@ cpu_setup_common(char **cpu_module_isa_set) } /* - * Get the valid contexts, mmu page sizes mask, Q sizes and isalist/r + * Get the valid mmu page sizes mask, Q sizes and isalist/r * from the MD for the first available CPU in cpulist. + * + * Do not expect the MMU page sizes mask to be more than 32-bit. */ - - if (nctxs == 0) - nctxs = (uint_t)(1 << get_mmu_ctx_bits(mdp, cpulist[0])); - - if (nctxs > MAX_NCTXS) - nctxs = MAX_NCTXS; - - /* Do not expect the MMU page sizes mask to be more than 32-bit. */ mmu_exported_pagesize_mask = (int)get_cpu_pagesizes(mdp, cpulist[0]); for (i = 0; i < nocpus; i++) diff --git a/usr/src/uts/sun4v/os/ppage.c b/usr/src/uts/sun4v/os/ppage.c index 80e4d5cdbe..fb0ee5b746 100644 --- a/usr/src/uts/sun4v/os/ppage.c +++ b/usr/src/uts/sun4v/os/ppage.c @@ -261,7 +261,7 @@ pp_unload_tlb(caddr_t *pslot, caddr_t va) { ASSERT(*pslot == va); - vtag_flushpage(va, KCONTEXT); + vtag_flushpage(va, (uint64_t)ksfmmup); *pslot = NULL; /* release the slot */ } diff --git a/usr/src/uts/sun4v/sys/cpu_module.h b/usr/src/uts/sun4v/sys/cpu_module.h index 902c088411..9ad7975309 100644 --- a/usr/src/uts/sun4v/sys/cpu_module.h +++ b/usr/src/uts/sun4v/sys/cpu_module.h @@ -69,12 +69,12 @@ int mmu_init_mmu_page_sizes(int cinfo); /* * virtual demap flushes (tlbs & virtual tag caches) */ -void vtag_flushpage(caddr_t addr, uint_t ctx); -void vtag_flushctx(uint_t ctx); +void vtag_flushpage(caddr_t addr, uint64_t sfmmup); void vtag_flushall(void); -void vtag_flushpage_tl1(uint64_t addr, uint64_t ctx); -void vtag_flush_pgcnt_tl1(uint64_t addr, uint64_t ctx_pgcnt); -void vtag_flushctx_tl1(uint64_t ctx, uint64_t dummy); +#pragma weak vtag_flushall_uctxs +void vtag_flushall_uctxs(void); +void vtag_flushpage_tl1(uint64_t addr, uint64_t sfmmup); +void vtag_flush_pgcnt_tl1(uint64_t addr, uint64_t sfmmup_pgcnt); void vtag_flushall_tl1(uint64_t dummy1, uint64_t dummy2); void vtag_unmap_perm_tl1(uint64_t addr, uint64_t ctx); @@ -87,12 +87,6 @@ void vac_flushcolor(int color, pfn_t pf); void vac_flushcolor_tl1(uint64_t color, uint64_t dummy); /* - * Calculate, set optimal dtlb pagesize, for ISM and mpss, to support - * cpus with non-fully-associative dtlbs. - */ -extern uchar_t *ctx_pgsz_array; - -/* * flush instruction cache if needed */ void flush_instr_mem(caddr_t addr, size_t len); diff --git a/usr/src/uts/sun4v/sys/machcpuvar.h b/usr/src/uts/sun4v/sys/machcpuvar.h index 8d9ac241aa..20d452be65 100644 --- a/usr/src/uts/sun4v/sys/machcpuvar.h +++ b/usr/src/uts/sun4v/sys/machcpuvar.h @@ -122,6 +122,11 @@ struct machcpu { * E$ data, which is needed for the specific cpu type. */ void *cpu_private; /* ptr to cpu private data */ + /* + * per-MMU ctxdom CPU data. + */ + uint_t cpu_mmu_idx; + struct mmu_ctx *cpu_mmu_ctxp; ptl1_state_t ptl1_state; @@ -163,6 +168,8 @@ struct machcpu { typedef struct machcpu machcpu_t; #define cpu_startup_thread cpu_m.startup_thread +#define CPU_MMU_IDX(cp) ((cp)->cpu_m.cpu_mmu_idx) +#define CPU_MMU_CTXP(cp) ((cp)->cpu_m.cpu_mmu_ctxp) #define NINTR_THREADS (LOCK_LEVEL) /* number of interrupt threads */ /* diff --git a/usr/src/uts/sun4v/sys/machparam.h b/usr/src/uts/sun4v/sys/machparam.h index 130e8e662f..53f6c2b2c8 100644 --- a/usr/src/uts/sun4v/sys/machparam.h +++ b/usr/src/uts/sun4v/sys/machparam.h @@ -300,6 +300,7 @@ extern "C" { #define PTL1_BAD_RED 17 #define PTL1_BAD_HCALL_UNMAP_PERM_EINVAL 18 #define PTL1_BAD_HCALL_UNMAP_PERM_ENOMAP 19 +#define PTL1_BAD_RAISE_TSBEXCP 20 /* * Defines the max trap level allowed diff --git a/usr/src/uts/sun4v/vm/mach_sfmmu_asm.s b/usr/src/uts/sun4v/vm/mach_sfmmu_asm.s index e330692603..cf21f174bb 100644 --- a/usr/src/uts/sun4v/vm/mach_sfmmu_asm.s +++ b/usr/src/uts/sun4v/vm/mach_sfmmu_asm.s @@ -2,9 +2,8 @@ * CDDL HEADER START * * The contents of this file are subject to the terms of the - * Common Development and Distribution License, Version 1.0 only - * (the "License"). You may not use this file except in compliance - * with the License. + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. @@ -20,7 +19,7 @@ * CDDL HEADER END */ /* - * Copyright 2005 Sun Microsystems, Inc. All rights reserved. + * Copyright 2006 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ @@ -61,12 +60,7 @@ /* ARGSUSED */ void -sfmmu_ctx_steal_tl1(uint64_t sctx, uint64_t rctx) -{} - -/* ARGSUSED */ -void -sfmmu_raise_tsb_exception(uint64_t sctx, uint64_t rctx) +sfmmu_raise_tsb_exception(uint64_t sfmmup, uint64_t rctx) {} int @@ -91,92 +85,115 @@ sfmmu_load_mmustate(sfmmu_t *sfmmup) #else /* lint */ /* - * 1. If stealing ctx, flush all TLB entries whose ctx is ctx-being-stolen. - * 2. If processor is running in the ctx-being-stolen, set the - * context to the resv context. That is - * If processor in User-mode - pri/sec-ctx both set to ctx-being-stolen, - * change both pri/sec-ctx registers to resv ctx. - * If processor in Kernel-mode - pri-ctx is 0, sec-ctx is ctx-being-stolen, - * just change sec-ctx register to resv ctx. When it returns to - * kernel-mode, user_rtt will change pri-ctx. - * - * Note: For multiple page size TLB, no need to set page sizes for - * DEMAP context. + * Invalidate either the context of a specific victim or any process + * currently running on this CPU. * - * %g1 = ctx being stolen (victim) - * %g2 = invalid ctx to replace victim with + * %g1 = sfmmup whose ctx is being stolen (victim) + * when called from sfmmu_wrap_around, %g1 == INVALID_CONTEXT. + * Note %g1 is the only input argument used by this xcall handler. */ - ENTRY(sfmmu_ctx_steal_tl1) - /* - * Flush TLBs. - */ - /* flush context from the tlb via HV call */ + ENTRY(sfmmu_raise_tsb_exception) + ! + ! if (victim == INVALID_CONTEXT) { + ! if (sec-ctx > INVALID_CONTEXT) + ! write INVALID_CONTEXT to sec-ctx + ! if (pri-ctx > INVALID_CONTEXT) + ! write INVALID_CONTEXT to pri-ctx + ! + ! } else if (current CPU tsbmiss->usfmmup != victim sfmmup) { + ! return + ! } else { + ! if (sec-ctx > INVALID_CONTEXT) + ! write INVALID_CONTEXT to sec-ctx + ! + ! if (pri-ctx > INVALID_CONTEXT) + ! write INVALID_CONTEXT to pri-ctx + ! } + ! + + sethi %hi(ksfmmup), %g3 + ldx [%g3 + %lo(ksfmmup)], %g3 + cmp %g1, %g3 + be,a,pn %xcc, ptl1_panic /* can't invalidate kernel ctx */ + mov PTL1_BAD_RAISE_TSBEXCP, %g1 + + set INVALID_CONTEXT, %g2 + + cmp %g1, INVALID_CONTEXT + bne,pt %xcc, 1f /* called from wrap_around? */ + mov MMU_SCONTEXT, %g3 + + ldxa [%g3]ASI_MMU_CTX, %g5 /* %g5 = sec-ctx */ + cmp %g5, INVALID_CONTEXT /* kernel or invalid ctx ? */ + ble,pn %xcc, 0f /* yes, no need to change */ + mov MMU_PCONTEXT, %g7 + + stxa %g2, [%g3]ASI_MMU_CTX /* set invalid ctx */ + membar #Sync + +0: + ldxa [%g7]ASI_MMU_CTX, %g5 /* %g5 = pri-ctx */ + cmp %g5, INVALID_CONTEXT /* kernel or invalid ctx? */ + ble,pn %xcc, 6f /* yes, no need to change */ + nop + + stxa %g2, [%g7]ASI_MMU_CTX /* set pri-ctx to invalid */ + membar #Sync + +6: /* flushall tlb */ mov %o0, %g3 mov %o1, %g4 - mov %o2, %g5 - mov %o3, %g6 + mov %o2, %g6 mov %o5, %g7 - mov %g1, %o2 ! ctx# - mov %g0, %o0 ! Current CPU only (use NULL) - mov %g0, %o1 ! Current CPU only (use NULL) - mov MAP_ITLB | MAP_DTLB, %o3 - mov MMU_DEMAP_CTX, %o5 - ta FAST_TRAP - brnz,a,pn %o0, ptl1_panic - mov PTL1_BAD_HCALL, %g1 - + mov %g0, %o0 ! XXX no cpu list yet + mov %g0, %o1 ! XXX no cpu list yet + mov MAP_ITLB | MAP_DTLB, %o2 + mov MMU_DEMAP_ALL, %o5 + ta FAST_TRAP + brz,pt %o0, 5f + nop + ba panic_bad_hcall + mov MMU_DEMAP_ALL, %o1 +5: mov %g3, %o0 mov %g4, %o1 - mov %g5, %o2 - mov %g6, %o3 + mov %g6, %o2 mov %g7, %o5 - - /* fall through to the code below */ - + + ba 3f + nop +1: /* - * We enter here if we're just raising a TSB miss - * exception, without switching MMU contexts. In - * this case, there is no need to flush the TLB. + * %g1 = sfmmup + * %g2 = INVALID_CONTEXT + * %g3 = MMU_SCONTEXT */ - ALTENTRY(sfmmu_raise_tsb_exception) - ! - ! %g1 = ctx being stolen (victim) - ! %g2 = invalid ctx to replace victim with - ! - ! if (sec-ctx != victim) { - ! return - ! } else { - ! if (pri-ctx == victim) { - ! write INVALID_CONTEXT to sec-ctx - ! write INVALID_CONTEXT to pri-ctx - ! } else { - ! write INVALID_CONTEXT to sec-ctx - ! } - ! } - ! - cmp %g1, NUM_LOCKED_CTXS - blt,a,pn %icc, ptl1_panic /* can't steal locked ctx */ - mov PTL1_BAD_CTX_STEAL, %g1 - set CTXREG_CTX_MASK, %g6 - set MMU_SCONTEXT, %g3 - ldxa [%g3]ASI_MMU_CTX, %g5 /* get sec-ctx */ - and %g5, %g6, %g5 + CPU_TSBMISS_AREA(%g5, %g6) /* load cpu tsbmiss area */ + ldx [%g5 + TSBMISS_UHATID], %g5 /* load usfmmup */ + cmp %g5, %g1 /* is it the victim? */ - bne,pn %icc, 2f /* was our sec-ctx a victim? */ + bne,pt %xcc, 2f /* is our sec-ctx a victim? */ + nop + + ldxa [%g3]ASI_MMU_CTX, %g5 /* %g5 = sec-ctx */ + cmp %g5, INVALID_CONTEXT /* kernel or invalid ctx ? */ + ble,pn %xcc, 0f /* yes, no need to change */ mov MMU_PCONTEXT, %g7 - ldxa [%g7]ASI_MMU_CTX, %g4 /* get pri-ctx */ - and %g4, %g6, %g4 - stxa %g2, [%g3]ASI_MMU_CTX /* set sec-ctx to invalid ctx */ + + stxa %g2, [%g3]ASI_MMU_CTX /* set sec-ctx to invalid */ membar #Sync - cmp %g1, %g4 /* is it the victim? */ - bne %icc, 3f /* nope, no need to change it */ + +0: + ldxa [%g7]ASI_MMU_CTX, %g4 /* %g4 = pri-ctx */ + cmp %g4, INVALID_CONTEXT /* is pri-ctx the victim? */ + ble %icc, 3f /* no need to change pri-ctx */ nop - stxa %g2, [%g7]ASI_MMU_CTX /* set pri-ctx to invalid ctx */ - /* next instruction is retry so no membar sync */ -3: + stxa %g2, [%g7]ASI_MMU_CTX /* set pri-ctx to invalid */ membar #Sync + +3: /* TSB program must be cleared - walkers do not check a context. */ mov %o0, %g3 mov %o1, %g4 @@ -192,7 +209,7 @@ sfmmu_load_mmustate(sfmmu_t *sfmmup) mov %g7, %o5 2: retry - SET_SIZE(sfmmu_ctx_steal_tl1) + SET_SIZE(sfmmu_raise_tsb_exception) ENTRY_NP(sfmmu_getctx_pri) set MMU_PCONTEXT, %o0 @@ -212,14 +229,13 @@ sfmmu_load_mmustate(sfmmu_t *sfmmup) /* * Set the secondary context register for this process. - * %o0 = context number for this process. + * %o0 = context number */ ENTRY_NP(sfmmu_setctx_sec) /* * From resume we call sfmmu_setctx_sec with interrupts disabled. * But we can also get called from C with interrupts enabled. So, - * we need to check first. Also, resume saves state in %o3 and %o5 - * so we can't use those registers here. + * we need to check first. */ /* If interrupts are not disabled, then disable them */ @@ -233,10 +249,15 @@ sfmmu_load_mmustate(sfmmu_t *sfmmup) stxa %o0, [%o1]ASI_MMU_CTX /* set 2nd context reg. */ flush %o4 + /* + * if the routine is entered with intr enabled, then enable intr now. + * otherwise, keep intr disabled, return without enabing intr. + * %g1 - old intr state + */ btst PSTATE_IE, %g1 - bnz,a,pt %icc, 1f + bnz,a,pt %icc, 2f wrpr %g0, %g1, %pstate /* enable interrupts */ -1: retl +2: retl nop SET_SIZE(sfmmu_setctx_sec) @@ -260,12 +281,10 @@ sfmmu_load_mmustate(sfmmu_t *sfmmup) * %o0 - hat pointer */ ENTRY_NP(sfmmu_load_mmustate) - /* - * From resume we call sfmmu_load_mmustate with interrupts disabled. - * But we can also get called from C with interrupts enabled. So, - * we need to check first. Also, resume saves state in %o5 and we - * can't use this register here. - */ + +#ifdef DEBUG + PANIC_IF_INTR_ENABLED_PSTR(msfmmu_ei_l1, %g1) +#endif /* DEBUG */ sethi %hi(ksfmmup), %o3 ldx [%o3 + %lo(ksfmmup)], %o3 @@ -273,12 +292,6 @@ sfmmu_load_mmustate(sfmmu_t *sfmmup) be,pn %xcc, 3f ! if kernel as, do nothing nop - /* If interrupts are not disabled, then disable them */ - rdpr %pstate, %g1 - btst PSTATE_IE, %g1 - bnz,a,pt %icc, 1f - wrpr %g1, PSTATE_IE, %pstate ! disable interrupts -1: /* * We need to set up the TSB base register, tsbmiss * area, and pass the TSB information into the hypervisor @@ -307,7 +320,14 @@ sfmmu_load_mmustate(sfmmu_t *sfmmup) #endif /* DEBUG */ CPU_ADDR(%o2, %o4) ! load CPU struct addr to %o2 using %o4 ldub [%o2 + CPU_TSTAT_FLAGS], %o1 ! load cpu_tstat_flag to %o1 - lduh [%o0 + SFMMU_CNUM], %o2 + + /* + * %o0 = sfmmup + * %o2 = returned sfmmu cnum on this CPU + * %o4 = scratch + */ + SFMMU_CPU_CNUM(%o0, %o2, %o4) + mov %o5, %o4 ! preserve %o5 for resume mov %o0, %o3 ! preserve %o0 btst TSTAT_TLB_STATS, %o1 @@ -333,9 +353,6 @@ sfmmu_load_mmustate(sfmmu_t *sfmmup) stx %o0, [%o2 + TSBMISS_UHATID] stuh %o3, [%o2 + TSBMISS_HATFLAGS] - btst PSTATE_IE, %g1 - bnz,a,pt %icc, 3f - wrpr %g0, %g1, %pstate ! enable interrupts 3: retl nop SET_SIZE(sfmmu_load_mmustate) |