/* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ /* * General assembly language routines. * It is the intent of this file to contain routines that are * independent of the specific kernel architecture, and those that are * common across kernel architectures. * As architectures diverge, and implementations of specific * architecture-dependent routines change, the routines should be moved * from this file into the respective ../`arch -k`/subr.s file. * Or, if you want to be really nice, move them to a file whose * name has something to do with the routine you are moving. */ #include <sys/asm_linkage.h> #include <sys/privregs.h> #include <sys/machparam.h> /* To get SYSBASE and PAGESIZE */ #include <sys/machthread.h> #include <sys/clock.h> #include <sys/psr_compat.h> #include <sys/isa_defs.h> #include <sys/dditypes.h> #include <sys/panic.h> #include <sys/machlock.h> #include <sys/ontrap.h> #include "assym.h" .seg ".text" .align 4 /* * Macro to raise processor priority level. * Avoid dropping processor priority if already at high level. * Also avoid going below CPU->cpu_base_spl, which could've just been set by * a higher-level interrupt thread that just blocked. * * level can be %o0 (not other regs used here) or a constant. */ #define RAISE(level) \ rdpr %pil, %o1; /* get current PIL */ \ cmp %o1, level; /* is PIL high enough? */ \ bge 1f; /* yes, return */ \ nop; \ wrpr %g0, PIL_MAX, %pil; /* freeze CPU_BASE_SPL */ \ ldn [THREAD_REG + T_CPU], %o2; \ ld [%o2 + CPU_BASE_SPL], %o2; \ cmp %o2, level; /* compare new to base */ \ movl %xcc, level, %o2; /* use new if base lower */ \ wrpr %g0, %o2, %pil; \ 1: \ retl; \ mov %o1, %o0 /* return old PIL */ /* * Macro to raise processor priority level to level >= DISP_LEVEL. * Doesn't require comparison to CPU->cpu_base_spl. * * newpil can be %o0 (not other regs used here) or a constant. */ #define RAISE_HIGH(level) \ rdpr %pil, %o1; /* get current PIL */ \ cmp %o1, level; /* is PIL high enough? */ \ bge 1f; /* yes, return */ \ nop; \ wrpr %g0, level, %pil; /* use chose value */ \ 1: \ retl; \ mov %o1, %o0 /* return old PIL */ /* * Macro to set the priority to a specified level. * Avoid dropping the priority below CPU->cpu_base_spl. * * newpil can be %o0 (not other regs used here) or a constant with * the new PIL in the PSR_PIL field of the level arg. */ #define SETPRI(level) \ rdpr %pil, %o1; /* get current PIL */ \ wrpr %g0, PIL_MAX, %pil; /* freeze CPU_BASE_SPL */ \ ldn [THREAD_REG + T_CPU], %o2; \ ld [%o2 + CPU_BASE_SPL], %o2; \ cmp %o2, level; /* compare new to base */ \ movl %xcc, level, %o2; /* use new if base lower */ \ wrpr %g0, %o2, %pil; \ retl; \ mov %o1, %o0 /* return old PIL */ /* * Macro to set the priority to a specified level at or above LOCK_LEVEL. * Doesn't require comparison to CPU->cpu_base_spl. * * newpil can be %o0 (not other regs used here) or a constant with * the new PIL in the PSR_PIL field of the level arg. */ #define SETPRI_HIGH(level) \ rdpr %pil, %o1; /* get current PIL */ \ wrpr %g0, level, %pil; \ retl; \ mov %o1, %o0 /* return old PIL */ /* * Berkley 4.3 introduced symbolically named interrupt levels * as a way deal with priority in a machine independent fashion. * Numbered priorities are machine specific, and should be * discouraged where possible. * * Note, for the machine specific priorities there are * examples listed for devices that use a particular priority. * It should not be construed that all devices of that * type should be at that priority. It is currently were * the current devices fit into the priority scheme based * upon time criticalness. * * The underlying assumption of these assignments is that * SPARC9 IPL 10 is the highest level from which a device * routine can call wakeup. Devices that interrupt from higher * levels are restricted in what they can do. If they need * kernels services they should schedule a routine at a lower * level (via software interrupt) to do the required * processing. * * Examples of this higher usage: * Level Usage * 15 Asynchronous memory exceptions * 14 Profiling clock (and PROM uart polling clock) * 13 Audio device * 12 Serial ports * 11 Floppy controller * * The serial ports request lower level processing on level 6. * Audio and floppy request lower level processing on level 4. * * Also, almost all splN routines (where N is a number or a * mnemonic) will do a RAISE(), on the assumption that they are * never used to lower our priority. * The exceptions are: * spl8() Because you can't be above 15 to begin with! * splzs() Because this is used at boot time to lower our * priority, to allow the PROM to poll the uart. * spl0() Used to lower priority to 0. */ /* locks out all interrupts, including memory errors */ ENTRY(spl8) SETPRI_HIGH(15) SET_SIZE(spl8) /* just below the level that profiling runs */ ENTRY(spl7) RAISE_HIGH(13) SET_SIZE(spl7) /* sun specific - highest priority onboard serial i/o zs ports */ ENTRY(splzs) SETPRI_HIGH(12) /* Can't be a RAISE, as it's used to lower us */ SET_SIZE(splzs) /* * should lock out clocks and all interrupts, * as you can see, there are exceptions */ ENTRY(splhi) ALTENTRY(splhigh) ALTENTRY(spl6) ALTENTRY(i_ddi_splhigh) RAISE_HIGH(DISP_LEVEL) SET_SIZE(i_ddi_splhigh) SET_SIZE(spl6) SET_SIZE(splhigh) SET_SIZE(splhi) /* allow all interrupts */ ENTRY(spl0) SETPRI(0) SET_SIZE(spl0) /* * splx - set PIL back to that indicated by the old %pil passed as an argument, * or to the CPU's base priority, whichever is higher. */ ENTRY(splx) ALTENTRY(i_ddi_splx) SETPRI(%o0) /* set PIL */ SET_SIZE(i_ddi_splx) SET_SIZE(splx) /* * splr() * * splr is like splx but will only raise the priority and never drop it * Be careful not to set priority lower than CPU->cpu_base_pri, * even though it seems we're raising the priority, it could be set higher * at any time by an interrupt routine, so we must block interrupts and * look at CPU->cpu_base_pri. */ ENTRY(splr) RAISE(%o0) SET_SIZE(splr) /* * on_fault() * Catch lofault faults. Like setjmp except it returns one * if code following causes uncorrectable fault. Turned off * by calling no_fault(). */ ENTRY(on_fault) membar #Sync ! sync error barrier (see copy.s) stn %o0, [THREAD_REG + T_ONFAULT] set catch_fault, %o1 b setjmp ! let setjmp do the rest stn %o1, [THREAD_REG + T_LOFAULT] ! put catch_fault in t_lofault catch_fault: save %sp, -SA(WINDOWSIZE), %sp ! goto next window so that we can rtn ldn [THREAD_REG + T_ONFAULT], %o0 membar #Sync ! sync error barrier stn %g0, [THREAD_REG + T_ONFAULT] ! turn off onfault b longjmp ! let longjmp do the rest stn %g0, [THREAD_REG + T_LOFAULT] ! turn off lofault SET_SIZE(on_fault) /* * no_fault() * turn off fault catching. */ ENTRY(no_fault) membar #Sync ! sync error barrier stn %g0, [THREAD_REG + T_ONFAULT] retl stn %g0, [THREAD_REG + T_LOFAULT] ! turn off lofault SET_SIZE(no_fault) /* * Default trampoline code for on_trap() (see <sys/ontrap.h>). On sparcv9, * the trap code will complete trap processing but reset the return %pc to * ot_trampoline, which will by default be set to the address of this code. * We longjmp(&curthread->t_ontrap->ot_jmpbuf) to return back to on_trap(). */ ENTRY(on_trap_trampoline) ldn [THREAD_REG + T_ONTRAP], %o0 b longjmp add %o0, OT_JMPBUF, %o0 SET_SIZE(on_trap_trampoline) /* * Push a new element on to the t_ontrap stack. Refer to <sys/ontrap.h> for * more information about the on_trap() mechanism. If the on_trap_data is the * same as the topmost stack element, we just modify that element. * On UltraSPARC, we need to issue a membar #Sync before modifying t_ontrap. * The issue barrier is defined to force all deferred errors to complete before * we go any further. We want these errors to be processed before we modify * our current error protection. */ ENTRY(on_trap) membar #Sync ! force error barrier sth %o1, [%o0 + OT_PROT] ! ot_prot = prot sth %g0, [%o0 + OT_TRAP] ! ot_trap = 0 set on_trap_trampoline, %o2 ! %o2 = &on_trap_trampoline stn %o2, [%o0 + OT_TRAMPOLINE] ! ot_trampoline = %o2 stn %g0, [%o0 + OT_HANDLE] ! ot_handle = NULL ldn [THREAD_REG + T_ONTRAP], %o2 ! %o2 = curthread->t_ontrap cmp %o0, %o2 ! if (otp == %o2) be 0f ! don't modify t_ontrap stn %g0, [%o0 + OT_PAD1] ! delay - ot_pad1 = NULL stn %o2, [%o0 + OT_PREV] ! ot_prev = t_ontrap membar #Sync ! force error barrier stn %o0, [THREAD_REG + T_ONTRAP] ! t_ontrap = otp 0: b setjmp ! let setjmp do the rest add %o0, OT_JMPBUF, %o0 ! %o0 = &ot_jmpbuf SET_SIZE(on_trap) /* * Setjmp and longjmp implement non-local gotos using state vectors * type label_t. */ ENTRY(setjmp) stn %o7, [%o0 + L_PC] ! save return address stn %sp, [%o0 + L_SP] ! save stack ptr retl clr %o0 ! return 0 SET_SIZE(setjmp) ENTRY(longjmp) ! ! The following save is required so that an extra register ! window is flushed. Flushw flushes nwindows-2 ! register windows. If setjmp and longjmp are called from ! within the same window, that window will not get pushed ! out onto the stack without the extra save below. Tail call ! optimization can lead to callers of longjmp executing ! from a window that could be the same as the setjmp, ! thus the need for the following save. ! save %sp, -SA(MINFRAME), %sp flushw ! flush all but this window ldn [%i0 + L_PC], %i7 ! restore return addr ldn [%i0 + L_SP], %fp ! restore sp for dest on foreign stack ret ! return 1 restore %g0, 1, %o0 ! takes underflow, switches stacks SET_SIZE(longjmp) /* * movtuc(length, from, to, table) * * VAX movtuc instruction (sort of). */ ENTRY(movtuc) tst %o0 ble,pn %ncc, 2f ! check length clr %o4 ldub [%o1 + %o4], %g1 ! get next byte in string 0: ldub [%o3 + %g1], %g1 ! get corresponding table entry tst %g1 ! escape char? bnz 1f stb %g1, [%o2 + %o4] ! delay slot, store it retl ! return (bytes moved) mov %o4, %o0 1: inc %o4 ! increment index cmp %o4, %o0 ! index < length ? bl,a,pt %ncc, 0b ldub [%o1 + %o4], %g1 ! delay slot, get next byte in string 2: retl ! return (bytes moved) mov %o4, %o0 SET_SIZE(movtuc) /* * scanc(length, string, table, mask) * * VAX scanc instruction. */ ENTRY(scanc) tst %o0 ble,pn %ncc, 1f ! check length clr %o4 0: ldub [%o1 + %o4], %g1 ! get next byte in string cmp %o4, %o0 ! interlock slot, index < length ? ldub [%o2 + %g1], %g1 ! get corresponding table entry bge,pn %ncc, 1f ! interlock slot btst %o3, %g1 ! apply the mask bz,a 0b inc %o4 ! delay slot, increment index 1: retl ! return(length - index) sub %o0, %o4, %o0 SET_SIZE(scanc) /* * if a() calls b() calls caller(), * caller() returns return address in a(). */ ENTRY(caller) retl mov %i7, %o0 SET_SIZE(caller) /* * if a() calls callee(), callee() returns the * return address in a(); */ ENTRY(callee) retl mov %o7, %o0 SET_SIZE(callee) /* * return the current frame pointer */ ENTRY(getfp) retl mov %fp, %o0 SET_SIZE(getfp) /* * Get vector base register */ ENTRY(gettbr) retl mov %tbr, %o0 SET_SIZE(gettbr) /* * Get processor state register, V9 faked to look like V8. * Note: does not provide ccr.xcc and provides FPRS.FEF instead of * PSTATE.PEF, because PSTATE.PEF is always on in order to allow the * libc_psr memcpy routines to run without hitting the fp_disabled trap. */ ENTRY(getpsr) rd %ccr, %o1 ! get ccr sll %o1, PSR_ICC_SHIFT, %o0 ! move icc to V8 psr.icc rd %fprs, %o1 ! get fprs and %o1, FPRS_FEF, %o1 ! mask out dirty upper/lower sllx %o1, PSR_FPRS_FEF_SHIFT, %o1 ! shift fef to V8 psr.ef or %o0, %o1, %o0 ! or into psr.ef set V9_PSR_IMPLVER, %o1 ! SI assigned impl/ver: 0xef retl or %o0, %o1, %o0 ! or into psr.impl/ver SET_SIZE(getpsr) /* * Get current processor interrupt level */ ENTRY(getpil) retl rdpr %pil, %o0 SET_SIZE(getpil) ENTRY(setpil) retl wrpr %g0, %o0, %pil SET_SIZE(setpil) /* * _insque(entryp, predp) * * Insert entryp after predp in a doubly linked list. */ ENTRY(_insque) ldn [%o1], %g1 ! predp->forw stn %o1, [%o0 + CPTRSIZE] ! entryp->back = predp stn %g1, [%o0] ! entryp->forw = predp->forw stn %o0, [%o1] ! predp->forw = entryp retl stn %o0, [%g1 + CPTRSIZE] ! predp->forw->back = entryp SET_SIZE(_insque) /* * _remque(entryp) * * Remove entryp from a doubly linked list */ ENTRY(_remque) ldn [%o0], %g1 ! entryp->forw ldn [%o0 + CPTRSIZE], %g2 ! entryp->back stn %g1, [%g2] ! entryp->back->forw = entryp->forw retl stn %g2, [%g1 + CPTRSIZE] ! entryp->forw->back = entryp->back SET_SIZE(_remque) /* * strlen(str) * * Returns the number of non-NULL bytes in string argument. * * XXX - why is this here, rather than the traditional file? * why does it have local labels which don't start with a `.'? */ ENTRY(strlen) mov %o0, %o1 andcc %o1, 3, %o3 ! is src word aligned bz $nowalgnd clr %o0 ! length of non-zero bytes cmp %o3, 2 ! is src half-word aligned be $s2algn cmp %o3, 3 ! src is byte aligned ldub [%o1], %o3 ! move 1 or 3 bytes to align it inc 1, %o1 ! in either case, safe to do a byte be $s3algn tst %o3 $s1algn: bnz,a $s2algn ! now go align dest inc 1, %o0 b,a $done $s2algn: lduh [%o1], %o3 ! know src is half-byte aligned inc 2, %o1 srl %o3, 8, %o4 tst %o4 ! is the first byte zero bnz,a 1f inc %o0 b,a $done 1: andcc %o3, 0xff, %o3 ! is the second byte zero bnz,a $nowalgnd inc %o0 b,a $done $s3algn: bnz,a $nowalgnd inc 1, %o0 b,a $done $nowalgnd: ! use trick to check if any read bytes of a word are zero ! the following two constants will generate "byte carries" ! and check if any bit in a byte is set, if all characters ! are 7bits (unsigned) this allways works, otherwise ! there is a specil case that rarely happens, see below set 0x7efefeff, %o3 set 0x81010100, %o4 3: ld [%o1], %o2 ! main loop inc 4, %o1 add %o2, %o3, %o5 ! generate byte-carries xor %o5, %o2, %o5 ! see if orignal bits set and %o5, %o4, %o5 cmp %o5, %o4 ! if ==, no zero bytes be,a 3b inc 4, %o0 ! check for the zero byte and increment the count appropriately ! some information (the carry bit) is lost if bit 31 ! was set (very rare), if this is the rare condition, ! return to the main loop again sethi %hi(0xff000000), %o5 ! mask used to test for terminator andcc %o2, %o5, %g0 ! check if first byte was zero bnz 1f srl %o5, 8, %o5 $done: retl nop 1: andcc %o2, %o5, %g0 ! check if second byte was zero bnz 1f srl %o5, 8, %o5 $done1: retl inc %o0 1: andcc %o2, %o5, %g0 ! check if third byte was zero bnz 1f andcc %o2, 0xff, %g0 ! check if last byte is zero $done2: retl inc 2, %o0 1: bnz,a 3b inc 4, %o0 ! count of bytes $done3: retl inc 3, %o0 SET_SIZE(strlen) /* * Provide a C callable interface to the membar instruction. */ ENTRY(membar_ldld) retl membar #LoadLoad SET_SIZE(membar_ldld) ENTRY(membar_stld) retl membar #StoreLoad SET_SIZE(membar_stld) ENTRY(membar_ldst) retl membar #LoadStore SET_SIZE(membar_ldst) ENTRY(membar_stst) retl membar #StoreStore SET_SIZE(membar_stst) ENTRY(membar_ldld_stld) ALTENTRY(membar_stld_ldld) retl membar #LoadLoad|#StoreLoad SET_SIZE(membar_stld_ldld) SET_SIZE(membar_ldld_stld) ENTRY(membar_ldld_ldst) ALTENTRY(membar_ldst_ldld) retl membar #LoadLoad|#LoadStore SET_SIZE(membar_ldst_ldld) SET_SIZE(membar_ldld_ldst) ENTRY(membar_ldld_stst) ALTENTRY(membar_stst_ldld) retl membar #LoadLoad|#StoreStore SET_SIZE(membar_stst_ldld) SET_SIZE(membar_ldld_stst) ENTRY(membar_stld_ldst) ALTENTRY(membar_ldst_stld) retl membar #StoreLoad|#LoadStore SET_SIZE(membar_ldst_stld) SET_SIZE(membar_stld_ldst) ENTRY(membar_stld_stst) ALTENTRY(membar_stst_stld) retl membar #StoreLoad|#StoreStore SET_SIZE(membar_stst_stld) SET_SIZE(membar_stld_stst) ENTRY(membar_ldst_stst) ALTENTRY(membar_stst_ldst) retl membar #LoadStore|#StoreStore SET_SIZE(membar_stst_ldst) SET_SIZE(membar_ldst_stst) ENTRY(membar_lookaside) retl membar #Lookaside SET_SIZE(membar_lookaside) ENTRY(membar_memissue) retl membar #MemIssue SET_SIZE(membar_memissue) ENTRY(membar_sync) retl membar #Sync SET_SIZE(membar_sync) /* * Since all of the fuword() variants are so similar, we have a macro to spit * them out. */ #define FUWORD(NAME, LOAD, STORE, COPYOP) \ ENTRY(NAME); \ sethi %hi(1f), %o5; \ ldn [THREAD_REG + T_LOFAULT], %o3; \ or %o5, %lo(1f), %o5; \ membar #Sync; \ stn %o5, [THREAD_REG + T_LOFAULT]; \ LOAD [%o0]ASI_USER, %o2; \ membar #Sync; \ stn %o3, [THREAD_REG + T_LOFAULT]; \ mov 0, %o0; \ retl; \ STORE %o2, [%o1]; \ 1: \ membar #Sync; \ stn %o3, [THREAD_REG + T_LOFAULT]; \ ldn [THREAD_REG + T_COPYOPS], %o2; \ brz %o2, 2f; \ nop; \ ldn [%o2 + COPYOP], %g1; \ jmp %g1; \ nop; \ 2: \ retl; \ mov -1, %o0; \ SET_SIZE(NAME) FUWORD(fuword64, ldxa, stx, CP_FUWORD64) FUWORD(fuword32, lda, st, CP_FUWORD32) FUWORD(fuword16, lduha, sth, CP_FUWORD16) FUWORD(fuword8, lduba, stb, CP_FUWORD8) /* * Since all of the suword() variants are so similar, we have a macro to spit * them out. */ #define SUWORD(NAME, STORE, COPYOP) \ ENTRY(NAME) \ sethi %hi(1f), %o5; \ ldn [THREAD_REG + T_LOFAULT], %o3; \ or %o5, %lo(1f), %o5; \ membar #Sync; \ stn %o5, [THREAD_REG + T_LOFAULT]; \ STORE %o1, [%o0]ASI_USER; \ membar #Sync; \ stn %o3, [THREAD_REG + T_LOFAULT]; \ retl; \ clr %o0; \ 1: \ membar #Sync; \ stn %o3, [THREAD_REG + T_LOFAULT]; \ ldn [THREAD_REG + T_COPYOPS], %o2; \ brz %o2, 2f; \ nop; \ ldn [%o2 + COPYOP], %g1; \ jmp %g1; \ nop; \ 2: \ retl; \ mov -1, %o0; \ SET_SIZE(NAME) SUWORD(suword64, stxa, CP_SUWORD64) SUWORD(suword32, sta, CP_SUWORD32) SUWORD(suword16, stha, CP_SUWORD16) SUWORD(suword8, stba, CP_SUWORD8) ENTRY(fuword8_noerr) lduba [%o0]ASI_USER, %o0 retl stb %o0, [%o1] SET_SIZE(fuword8_noerr) ENTRY(fuword16_noerr) lduha [%o0]ASI_USER, %o0 retl sth %o0, [%o1] SET_SIZE(fuword16_noerr) ENTRY(fuword32_noerr) lda [%o0]ASI_USER, %o0 retl st %o0, [%o1] SET_SIZE(fuword32_noerr) ENTRY(fuword64_noerr) ldxa [%o0]ASI_USER, %o0 retl stx %o0, [%o1] SET_SIZE(fuword64_noerr) ENTRY(suword8_noerr) retl stba %o1, [%o0]ASI_USER SET_SIZE(suword8_noerr) ENTRY(suword16_noerr) retl stha %o1, [%o0]ASI_USER SET_SIZE(suword16_noerr) ENTRY(suword32_noerr) retl sta %o1, [%o0]ASI_USER SET_SIZE(suword32_noerr) ENTRY(suword64_noerr) retl stxa %o1, [%o0]ASI_USER SET_SIZE(suword64_noerr) .weak subyte subyte=suword8 .weak subyte_noerr subyte_noerr=suword8_noerr #ifdef _LP64 .weak fulword fulword=fuword64 .weak fulword_noerr fulword_noerr=fuword64_noerr .weak sulword sulword=suword64 .weak sulword_noerr sulword_noerr=suword64_noerr #else .weak fulword fulword=fuword32 .weak fulword_noerr fulword_noerr=fuword32_noerr .weak sulword sulword=suword32 .weak sulword_noerr sulword_noerr=suword32_noerr #endif /* LP64 */ /* * We define rdtick here, but not for sun4v. On sun4v systems, the %tick * and %stick should not be read directly without considering the tick * and stick offset kernel variables introduced to support sun4v OS * suspension. */ #if !defined (sun4v) ENTRY(rdtick) retl rd %tick, %o0 SET_SIZE(rdtick) #endif /* !sun4v */ /* * Set tba to given address, no side effects. */ ENTRY(set_tba) mov %o0, %o1 rdpr %tba, %o0 wrpr %o1, %tba retl nop SET_SIZE(set_tba) ENTRY(get_tba) retl rdpr %tba, %o0 SET_SIZE(get_tba) ENTRY_NP(setpstate) retl wrpr %g0, %o0, %pstate SET_SIZE(setpstate) ENTRY_NP(getpstate) retl rdpr %pstate, %o0 SET_SIZE(getpstate) ENTRY_NP(dtrace_interrupt_disable) rdpr %pstate, %o0 andn %o0, PSTATE_IE, %o1 retl wrpr %g0, %o1, %pstate SET_SIZE(dtrace_interrupt_disable) ENTRY_NP(dtrace_interrupt_enable) retl wrpr %g0, %o0, %pstate SET_SIZE(dtrace_interrupt_enable) #ifdef SF_ERRATA_51 .align 32 ENTRY(dtrace_membar_return) retl nop SET_SIZE(dtrace_membar_return) #define DTRACE_MEMBAR_RETURN ba,pt %icc, dtrace_membar_return #else #define DTRACE_MEMBAR_RETURN retl #endif ENTRY(dtrace_membar_producer) DTRACE_MEMBAR_RETURN membar #StoreStore SET_SIZE(dtrace_membar_producer) ENTRY(dtrace_membar_consumer) DTRACE_MEMBAR_RETURN membar #LoadLoad SET_SIZE(dtrace_membar_consumer) ENTRY_NP(dtrace_flush_windows) retl flushw SET_SIZE(dtrace_flush_windows) /* * %g1 pcstack * %g2 iteration count * %g3 final %fp * %g4 final %i7 * %g5 saved %cwp (so we can get back to the original window) * * %o0 pcstack / return value (iteration count) * %o1 limit / saved %cansave * %o2 lastfp * %o3 lastpc * %o4 saved %canrestore * %o5 saved %pstate (to restore interrupts) * * Note: The frame pointer returned via lastfp is safe to use as * long as getpcstack_top() returns either (0) or a value less * than (limit). */ ENTRY_NP(getpcstack_top) rdpr %pstate, %o5 andn %o5, PSTATE_IE, %g1 wrpr %g0, %g1, %pstate ! disable interrupts mov %o0, %g1 ! we need the pcstack pointer while ! we're visiting other windows rdpr %canrestore, %g2 ! number of available windows sub %g2, 1, %g2 ! account for skipped frame cmp %g2, %o1 ! compare with limit movg %icc, %o1, %g2 ! %g2 = min(%canrestore-1, limit) brlez,a,pn %g2, 3f ! Use slow path if count <= 0 -- clr %o0 ! return zero. mov %g2, %o0 ! set up return value rdpr %cwp, %g5 ! remember the register window state rdpr %cansave, %o1 ! 'restore' changes, so we can undo rdpr %canrestore, %o4 ! its effects when we finish. restore ! skip caller's frame 1: st %i7, [%g1] ! stash return address in pcstack restore ! go to the next frame subcc %g2, 1, %g2 ! decrement the count bnz,pt %icc, 1b ! loop until count reaches 0 add %g1, 4, %g1 ! increment pcstack mov %i6, %g3 ! copy the final %fp and return PC mov %i7, %g4 ! aside so we can return them to our ! caller wrpr %g0, %g5, %cwp ! jump back to the original window wrpr %g0, %o1, %cansave ! and restore the original register wrpr %g0, %o4, %canrestore ! window state. 2: stn %g3, [%o2] ! store the frame pointer and pc st %g4, [%o3] ! so our caller can continue the trace retl ! return to caller wrpr %g0, %o5, %pstate ! restore interrupts 3: flushw ! flush register windows, then ldn [%fp + STACK_BIAS + 14*CLONGSIZE], %g3 ! load initial fp ba 2b ldn [%fp + STACK_BIAS + 15*CLONGSIZE], %g4 ! and pc SET_SIZE(getpcstack_top) ENTRY_NP(setwstate) retl wrpr %g0, %o0, %wstate SET_SIZE(setwstate) ENTRY_NP(getwstate) retl rdpr %wstate, %o0 SET_SIZE(getwstate) /* * int panic_trigger(int *tp) * * A panic trigger is a word which is updated atomically and can only be set * once. We atomically store 0xFF into the high byte and load the old value. * If the byte was 0xFF, the trigger has already been activated and we fail. * If the previous value was 0 or not 0xFF, we succeed. This allows a * partially corrupt trigger to still trigger correctly. DTrace has its own * version of this function to allow it to panic correctly from probe context. */ ENTRY_NP(panic_trigger) ldstub [%o0], %o0 ! store 0xFF, load byte into %o0 cmp %o0, 0xFF ! compare %o0 to 0xFF set 1, %o1 ! %o1 = 1 be,a 0f ! if (%o0 == 0xFF) goto 0f (else annul) set 0, %o1 ! delay - %o1 = 0 0: retl mov %o1, %o0 ! return (%o1); SET_SIZE(panic_trigger) ENTRY_NP(dtrace_panic_trigger) ldstub [%o0], %o0 ! store 0xFF, load byte into %o0 cmp %o0, 0xFF ! compare %o0 to 0xFF set 1, %o1 ! %o1 = 1 be,a 0f ! if (%o0 == 0xFF) goto 0f (else annul) set 0, %o1 ! delay - %o1 = 0 0: retl mov %o1, %o0 ! return (%o1); SET_SIZE(dtrace_panic_trigger) /* * void vpanic(const char *format, va_list alist) * * The panic() and cmn_err() functions invoke vpanic() as a common entry point * into the panic code implemented in panicsys(). vpanic() is responsible * for passing through the format string and arguments, and constructing a * regs structure on the stack into which it saves the current register * values. If we are not dying due to a fatal trap, these registers will * then be preserved in panicbuf as the current processor state. Before * invoking panicsys(), vpanic() activates the first panic trigger (see * common/os/panic.c) and switches to the panic_stack if successful. Note that * DTrace takes a slightly different panic path if it must panic from probe * context. Instead of calling panic, it calls into dtrace_vpanic(), which * sets up the initial stack as vpanic does, calls dtrace_panic_trigger(), and * branches back into vpanic(). */ ENTRY_NP(vpanic) save %sp, -SA(MINFRAME + REGSIZE), %sp ! save and allocate regs ! ! The v9 struct regs has a 64-bit r_tstate field, which we use here ! to store the %ccr, %asi, %pstate, and %cwp as they would appear ! in %tstate if a trap occurred. We leave it up to the debugger to ! realize what happened and extract the register values. ! rd %ccr, %l0 ! %l0 = %ccr sllx %l0, TSTATE_CCR_SHIFT, %l0 ! %l0 <<= CCR_SHIFT rd %asi, %l1 ! %l1 = %asi sllx %l1, TSTATE_ASI_SHIFT, %l1 ! %l1 <<= ASI_SHIFT or %l0, %l1, %l0 ! %l0 |= %l1 rdpr %pstate, %l1 ! %l1 = %pstate sllx %l1, TSTATE_PSTATE_SHIFT, %l1 ! %l1 <<= PSTATE_SHIFT or %l0, %l1, %l0 ! %l0 |= %l1 rdpr %cwp, %l1 ! %l1 = %cwp sllx %l1, TSTATE_CWP_SHIFT, %l1 ! %l1 <<= CWP_SHIFT or %l0, %l1, %l0 ! %l0 |= %l1 set vpanic, %l1 ! %l1 = %pc (vpanic) add %l1, 4, %l2 ! %l2 = %npc (vpanic+4) rd %y, %l3 ! %l3 = %y ! ! Flush register windows before panic_trigger() in order to avoid a ! problem that a dump hangs if flush_windows() causes another panic. ! call flush_windows nop sethi %hi(panic_quiesce), %o0 call panic_trigger or %o0, %lo(panic_quiesce), %o0 ! if (!panic_trigger( vpanic_common: tst %o0 ! &panic_quiesce)) be 0f ! goto 0f; mov %o0, %l4 ! delay - %l4 = %o0 ! ! If panic_trigger() was successful, we are the first to initiate a ! panic: switch to the panic_stack. ! set panic_stack, %o0 ! %o0 = panic_stack set PANICSTKSIZE, %o1 ! %o1 = size of stack add %o0, %o1, %o0 ! %o0 = top of stack sub %o0, SA(MINFRAME + REGSIZE) + STACK_BIAS, %sp ! ! Now that we've got everything set up, store each register to its ! designated location in the regs structure allocated on the stack. ! The register set we store is the equivalent of the registers at ! the time the %pc was pointing to vpanic, thus the %i's now contain ! what the %o's contained prior to the save instruction. ! 0: stx %l0, [%sp + STACK_BIAS + SA(MINFRAME) + TSTATE_OFF] stx %g1, [%sp + STACK_BIAS + SA(MINFRAME) + G1_OFF] stx %g2, [%sp + STACK_BIAS + SA(MINFRAME) + G2_OFF] stx %g3, [%sp + STACK_BIAS + SA(MINFRAME) + G3_OFF] stx %g4, [%sp + STACK_BIAS + SA(MINFRAME) + G4_OFF] stx %g5, [%sp + STACK_BIAS + SA(MINFRAME) + G5_OFF] stx %g6, [%sp + STACK_BIAS + SA(MINFRAME) + G6_OFF] stx %g7, [%sp + STACK_BIAS + SA(MINFRAME) + G7_OFF] stx %i0, [%sp + STACK_BIAS + SA(MINFRAME) + O0_OFF] stx %i1, [%sp + STACK_BIAS + SA(MINFRAME) + O1_OFF] stx %i2, [%sp + STACK_BIAS + SA(MINFRAME) + O2_OFF] stx %i3, [%sp + STACK_BIAS + SA(MINFRAME) + O3_OFF] stx %i4, [%sp + STACK_BIAS + SA(MINFRAME) + O4_OFF] stx %i5, [%sp + STACK_BIAS + SA(MINFRAME) + O5_OFF] stx %i6, [%sp + STACK_BIAS + SA(MINFRAME) + O6_OFF] stx %i7, [%sp + STACK_BIAS + SA(MINFRAME) + O7_OFF] stn %l1, [%sp + STACK_BIAS + SA(MINFRAME) + PC_OFF] stn %l2, [%sp + STACK_BIAS + SA(MINFRAME) + NPC_OFF] st %l3, [%sp + STACK_BIAS + SA(MINFRAME) + Y_OFF] mov %l4, %o3 ! %o3 = on_panic_stack add %sp, STACK_BIAS + SA(MINFRAME), %o2 ! %o2 = ®s mov %i1, %o1 ! %o1 = alist call panicsys ! panicsys(); mov %i0, %o0 ! %o0 = format ret restore SET_SIZE(vpanic) ENTRY_NP(dtrace_vpanic) save %sp, -SA(MINFRAME + REGSIZE), %sp ! save and allocate regs ! ! The v9 struct regs has a 64-bit r_tstate field, which we use here ! to store the %ccr, %asi, %pstate, and %cwp as they would appear ! in %tstate if a trap occurred. We leave it up to the debugger to ! realize what happened and extract the register values. ! rd %ccr, %l0 ! %l0 = %ccr sllx %l0, TSTATE_CCR_SHIFT, %l0 ! %l0 <<= CCR_SHIFT rd %asi, %l1 ! %l1 = %asi sllx %l1, TSTATE_ASI_SHIFT, %l1 ! %l1 <<= ASI_SHIFT or %l0, %l1, %l0 ! %l0 |= %l1 rdpr %pstate, %l1 ! %l1 = %pstate sllx %l1, TSTATE_PSTATE_SHIFT, %l1 ! %l1 <<= PSTATE_SHIFT or %l0, %l1, %l0 ! %l0 |= %l1 rdpr %cwp, %l1 ! %l1 = %cwp sllx %l1, TSTATE_CWP_SHIFT, %l1 ! %l1 <<= CWP_SHIFT or %l0, %l1, %l0 ! %l0 |= %l1 set dtrace_vpanic, %l1 ! %l1 = %pc (vpanic) add %l1, 4, %l2 ! %l2 = %npc (vpanic+4) rd %y, %l3 ! %l3 = %y ! ! Flush register windows before panic_trigger() in order to avoid a ! problem that a dump hangs if flush_windows() causes another panic. ! call dtrace_flush_windows nop sethi %hi(panic_quiesce), %o0 call dtrace_panic_trigger or %o0, %lo(panic_quiesce), %o0 ! if (!panic_trigger( ba,a vpanic_common SET_SIZE(dtrace_vpanic) ENTRY(get_subcc_ccr) wr %g0, %ccr ! clear condition codes subcc %o0, %o1, %g0 retl rd %ccr, %o0 ! return condition codes SET_SIZE(get_subcc_ccr) ENTRY_NP(ftrace_interrupt_disable) rdpr %pstate, %o0 andn %o0, PSTATE_IE, %o1 retl wrpr %g0, %o1, %pstate SET_SIZE(ftrace_interrupt_disable) ENTRY_NP(ftrace_interrupt_enable) retl wrpr %g0, %o0, %pstate SET_SIZE(ftrace_interrupt_enable)