path: root/usr/src/uts/sparc/v9/ml/sparcv9_subr.s

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * General assembly language routines.
 * It is the intent of this file to contain routines that are
 * independent of the specific kernel architecture, and those that are
 * common across kernel architectures.
 * As architectures diverge, and implementations of specific
 * architecture-dependent routines change, the routines should be moved
 * from this file into the respective ../`arch -k`/subr.s file.
 * Or, if you want to be really nice, move them to a file whose
 * name has something to do with the routine you are moving.
 */

#include <sys/asm_linkage.h>
#include <sys/privregs.h>
#include <sys/machparam.h>	/* To get SYSBASE and PAGESIZE */
#include <sys/machthread.h>
#include <sys/clock.h>
#include <sys/psr_compat.h>
#include <sys/isa_defs.h>
#include <sys/dditypes.h>
#include <sys/panic.h>
#include <sys/machlock.h>
#include <sys/ontrap.h>

#include "assym.h"

	.seg	".text"
	.align	4

/*
 * Macro to raise processor priority level.
 * Avoid dropping processor priority if already at high level.
 * Also avoid going below CPU->cpu_base_spl, which could've just been set by
 * a higher-level interrupt thread that just blocked.
 *
 * level can be %o0 (not other regs used here) or a constant.
 */
#define	RAISE(level) \
	rdpr	%pil, %o1;		/* get current PIL */		\
	cmp	%o1, level;		/* is PIL high enough? */	\
	bge	1f;			/* yes, return */		\
	nop;								\
	wrpr	%g0, PIL_MAX, %pil;	/* freeze CPU_BASE_SPL */	\
	ldn	[THREAD_REG + T_CPU], %o2;				\
	ld	[%o2 + CPU_BASE_SPL], %o2;				\
	cmp	%o2, level;		/* compare new to base */	\
	movl	%xcc, level, %o2;	/* use new if base lower */	\
	wrpr	%g0, %o2, %pil;						\
1:									\
	retl;								\
	mov	%o1, %o0		/* return old PIL */

/*
 * Macro to raise processor priority level to level >= DISP_LEVEL.
 * Doesn't require comparison to CPU->cpu_base_spl.
 *
 * newpil can be %o0 (not other regs used here) or a constant.
 */
#define	RAISE_HIGH(level) \
	rdpr	%pil, %o1;		/* get current PIL */		\
	cmp	%o1, level;		/* is PIL high enough? */	\
	bge	1f;			/* yes, return */		\
	nop;								\
	wrpr	%g0, level, %pil;	/* use chose value */		\
1:									\
	retl;								\
	mov	%o1, %o0		/* return old PIL */
	
/*
 * Macro to set the priority to a specified level.
 * Avoid dropping the priority below CPU->cpu_base_spl.
 *
 * newpil can be %o0 (not other regs used here) or a constant with
 * the new PIL in the PSR_PIL field of the level arg.
 */
#define SETPRI(level) \
	rdpr	%pil, %o1;		/* get current PIL */		\
	wrpr	%g0, PIL_MAX, %pil;	/* freeze CPU_BASE_SPL */	\
	ldn	[THREAD_REG + T_CPU], %o2;				\
	ld	[%o2 + CPU_BASE_SPL], %o2;				\
	cmp	%o2, level;		/* compare new to base */	\
	movl	%xcc, level, %o2;	/* use new if base lower */	\
	wrpr	%g0, %o2, %pil;						\
	retl;								\
	mov	%o1, %o0		/* return old PIL */

/*
 * Macro to set the priority to a specified level at or above LOCK_LEVEL.
 * Doesn't require comparison to CPU->cpu_base_spl.
 *
 * newpil can be %o0 (not other regs used here) or a constant with
 * the new PIL in the PSR_PIL field of the level arg.
 */
#define	SETPRI_HIGH(level) \
	rdpr	%pil, %o1;		/* get current PIL */		\
	wrpr	%g0, level, %pil;					\
	retl;								\
	mov	%o1, %o0		/* return old PIL */

	/*
	 * Berkley 4.3 introduced symbolically named interrupt levels
	 * as a way deal with priority in a machine independent fashion.
	 * Numbered priorities are machine specific, and should be
	 * discouraged where possible.
	 *
	 * Note, for the machine specific priorities there are
	 * examples listed for devices that use a particular priority.
	 * It should not be construed that all devices of that
	 * type should be at that priority.  It is currently were
	 * the current devices fit into the priority scheme based
	 * upon time criticalness.
	 *
	 * The underlying assumption of these assignments is that
	 * SPARC9 IPL 10 is the highest level from which a device
	 * routine can call wakeup.  Devices that interrupt from higher
	 * levels are restricted in what they can do.  If they need
	 * kernels services they should schedule a routine at a lower
	 * level (via software interrupt) to do the required
	 * processing.
	 *
	 * Examples of this higher usage:
	 *	Level	Usage
	 *	15	Asynchronous memory exceptions
	 *	14	Profiling clock (and PROM uart polling clock)
	 *	13	Audio device
	 *	12	Serial ports
	 *	11	Floppy controller
	 *
	 * The serial ports request lower level processing on level 6.
	 * Audio and floppy request lower level processing on level 4.
	 *
	 * Also, almost all splN routines (where N is a number or a
	 * mnemonic) will do a RAISE(), on the assumption that they are
	 * never used to lower our priority.
	 * The exceptions are:
	 *	spl8()		Because you can't be above 15 to begin with!
	 *	splzs()		Because this is used at boot time to lower our
	 *			priority, to allow the PROM to poll the uart.
	 *	spl0()		Used to lower priority to 0.
	 */

	/* locks out all interrupts, including memory errors */
	ENTRY(spl8)
	SETPRI_HIGH(15)
	SET_SIZE(spl8)

	/* just below the level that profiling runs */
	ENTRY(spl7)
	RAISE_HIGH(13)
	SET_SIZE(spl7)

	/* sun specific - highest priority onboard serial i/o zs ports */
	ENTRY(splzs)
	SETPRI_HIGH(12)	/* Can't be a RAISE, as it's used to lower us */
	SET_SIZE(splzs)

	/*
	 * should lock out clocks and all interrupts,
	 * as you can see, there are exceptions
	 */
	ENTRY(splhi)
	ALTENTRY(splhigh)
	ALTENTRY(spl6)
	ALTENTRY(i_ddi_splhigh)
	RAISE_HIGH(DISP_LEVEL)
	SET_SIZE(i_ddi_splhigh)
	SET_SIZE(spl6)
	SET_SIZE(splhigh)
	SET_SIZE(splhi)

	/* allow all interrupts */
	ENTRY(spl0)
	SETPRI(0)
	SET_SIZE(spl0)

/*
 * splx - set PIL back to that indicated by the old %pil passed as an argument,
 * or to the CPU's base priority, whichever is higher.
 */

	ENTRY(splx)
	ALTENTRY(i_ddi_splx)
	SETPRI(%o0)		/* set PIL */
	SET_SIZE(i_ddi_splx)
	SET_SIZE(splx)

/*
 * splr()
 *
 * splr is like splx but will only raise the priority and never drop it
 * Be careful not to set priority lower than CPU->cpu_base_pri,
 * even though it seems we're raising the priority, it could be set higher
 * at any time by an interrupt routine, so we must block interrupts and
 * look at CPU->cpu_base_pri.
 */

	ENTRY(splr)
	RAISE(%o0)
	SET_SIZE(splr)

/*
 * on_fault()
 * Catch lofault faults. Like setjmp except it returns one
 * if code following causes uncorrectable fault. Turned off
 * by calling no_fault().
 */

	ENTRY(on_fault)
	membar	#Sync			! sync error barrier (see copy.s)
	stn	%o0, [THREAD_REG + T_ONFAULT]
	set	catch_fault, %o1
	b	setjmp			! let setjmp do the rest
	stn	%o1, [THREAD_REG + T_LOFAULT]	! put catch_fault in t_lofault

catch_fault:
	save	%sp, -SA(WINDOWSIZE), %sp ! goto next window so that we can rtn
	ldn	[THREAD_REG + T_ONFAULT], %o0
	membar	#Sync				! sync error barrier
	stn	%g0, [THREAD_REG + T_ONFAULT]	! turn off onfault
	b	longjmp			! let longjmp do the rest
	stn	%g0, [THREAD_REG + T_LOFAULT]	! turn off lofault
	SET_SIZE(on_fault)

/*
 * no_fault()
 * turn off fault catching.
 */

	ENTRY(no_fault)
	membar	#Sync				! sync error barrier
	stn	%g0, [THREAD_REG + T_ONFAULT]
	retl
	stn	%g0, [THREAD_REG + T_LOFAULT]	! turn off lofault
	SET_SIZE(no_fault)

/*
 * Default trampoline code for on_trap() (see <sys/ontrap.h>).  On sparcv9,
 * the trap code will complete trap processing but reset the return %pc to
 * ot_trampoline, which will by default be set to the address of this code.
 * We longjmp(&curthread->t_ontrap->ot_jmpbuf) to return back to on_trap().
 */

	ENTRY(on_trap_trampoline)
	ldn	[THREAD_REG + T_ONTRAP], %o0    
	b	longjmp                 
	add	%o0, OT_JMPBUF, %o0
	SET_SIZE(on_trap_trampoline)

/*
 * Push a new element on to the t_ontrap stack.  Refer to <sys/ontrap.h> for
 * more information about the on_trap() mechanism.  If the on_trap_data is the
 * same as the topmost stack element, we just modify that element.
 * On UltraSPARC, we need to issue a membar #Sync before modifying t_ontrap.
 * The issue barrier is defined to force all deferred errors to complete before
 * we go any further.  We want these errors to be processed before we modify
 * our current error protection.
 */

	ENTRY(on_trap)
	membar	#Sync				! force error barrier
	sth	%o1, [%o0 + OT_PROT]		! ot_prot = prot
	sth	%g0, [%o0 + OT_TRAP]		! ot_trap = 0
	set	on_trap_trampoline, %o2		! %o2 = &on_trap_trampoline
	stn	%o2, [%o0 + OT_TRAMPOLINE]	! ot_trampoline = %o2
	stn	%g0, [%o0 + OT_HANDLE]		! ot_handle = NULL
	ldn	[THREAD_REG + T_ONTRAP], %o2	! %o2 = curthread->t_ontrap
	cmp	%o0, %o2			! if (otp == %o2)
	be	0f				!    don't modify t_ontrap
	stn	%g0, [%o0 + OT_PAD1]		! delay - ot_pad1 = NULL

	stn	%o2, [%o0 + OT_PREV]		! ot_prev = t_ontrap
	membar	#Sync				! force error barrier
	stn	%o0, [THREAD_REG + T_ONTRAP]	! t_ontrap = otp

0:	b	setjmp				! let setjmp do the rest
	add	%o0, OT_JMPBUF, %o0		! %o0 = &ot_jmpbuf
	SET_SIZE(on_trap)

/*
 * Setjmp and longjmp implement non-local gotos using state vectors
 * type label_t.
 */

	ENTRY(setjmp)
	stn	%o7, [%o0 + L_PC]	! save return address
	stn	%sp, [%o0 + L_SP]	! save stack ptr
	retl
	clr	%o0			! return 0
	SET_SIZE(setjmp)


	ENTRY(longjmp)
	!
        ! The following save is required so that an extra register
        ! window is flushed.  Flushw flushes nwindows-2
        ! register windows.  If setjmp and longjmp are called from
        ! within the same window, that window will not get pushed
        ! out onto the stack without the extra save below.  Tail call
        ! optimization can lead to callers of longjmp executing
        ! from a window that could be the same as the setjmp,
        ! thus the need for the following save.
        !
	save    %sp, -SA(MINFRAME), %sp
	flushw				! flush all but this window
	ldn	[%i0 + L_PC], %i7	! restore return addr
	ldn	[%i0 + L_SP], %fp	! restore sp for dest on foreign stack
	ret				! return 1
	restore	%g0, 1, %o0		! takes underflow, switches stacks
	SET_SIZE(longjmp)

/*
 * movtuc(length, from, to, table)
 *
 * VAX movtuc instruction (sort of).
 */

	ENTRY(movtuc)
	tst     %o0
	ble,pn	%ncc, 2f		! check length
	clr     %o4

	ldub    [%o1 + %o4], %g1        ! get next byte in string
0:
	ldub    [%o3 + %g1], %g1        ! get corresponding table entry
	tst     %g1                     ! escape char?
	bnz     1f
	stb     %g1, [%o2 + %o4]        ! delay slot, store it

	retl                            ! return (bytes moved)
	mov     %o4, %o0
1:
	inc     %o4                     ! increment index
	cmp     %o4, %o0                ! index < length ?
	bl,a,pt	%ncc, 0b
	ldub    [%o1 + %o4], %g1        ! delay slot, get next byte in string
2:
	retl                            ! return (bytes moved)
	mov     %o4, %o0
	SET_SIZE(movtuc)

/*
 * scanc(length, string, table, mask)
 *
 * VAX scanc instruction.
 */

	ENTRY(scanc)
	tst	%o0	
	ble,pn	%ncc, 1f		! check length
	clr	%o4
0:
	ldub	[%o1 + %o4], %g1	! get next byte in string
	cmp	%o4, %o0		! interlock slot, index < length ?
	ldub	[%o2 + %g1], %g1	! get corresponding table entry
	bge,pn	%ncc, 1f		! interlock slot
	btst	%o3, %g1		! apply the mask
	bz,a	0b
	inc	%o4			! delay slot, increment index
1:
	retl				! return(length - index)
	sub	%o0, %o4, %o0
	SET_SIZE(scanc)

/*
 * if a() calls b() calls caller(),
 * caller() returns return address in a().
 */

	ENTRY(caller)
	retl
	mov	%i7, %o0
	SET_SIZE(caller)

/*
 * if a() calls callee(), callee() returns the
 * return address in a();
 */

	ENTRY(callee)
	retl
	mov	%o7, %o0
	SET_SIZE(callee)

/*
 * return the current frame pointer
 */

	ENTRY(getfp)
	retl
	mov	%fp, %o0
	SET_SIZE(getfp)

/*
 * Get vector base register
 */

	ENTRY(gettbr)
	retl
	mov     %tbr, %o0
	SET_SIZE(gettbr)

/*
 * Get processor state register, V9 faked to look like V8.
 * Note: does not provide ccr.xcc and provides FPRS.FEF instead of
 * PSTATE.PEF, because PSTATE.PEF is always on in order to allow the
 * libc_psr memcpy routines to run without hitting the fp_disabled trap.
 */

	ENTRY(getpsr)
	rd	%ccr, %o1			! get ccr
        sll	%o1, PSR_ICC_SHIFT, %o0		! move icc to V8 psr.icc
	rd	%fprs, %o1			! get fprs
	and	%o1, FPRS_FEF, %o1		! mask out dirty upper/lower
	sllx	%o1, PSR_FPRS_FEF_SHIFT, %o1	! shift fef to V8 psr.ef
        or	%o0, %o1, %o0			! or into psr.ef
        set	V9_PSR_IMPLVER, %o1		! SI assigned impl/ver: 0xef
        retl
        or	%o0, %o1, %o0			! or into psr.impl/ver
	SET_SIZE(getpsr)

/*
 * Get current processor interrupt level
 */

	ENTRY(getpil)
	retl
	rdpr	%pil, %o0
	SET_SIZE(getpil)

	ENTRY(setpil)
	retl
	wrpr	%g0, %o0, %pil
	SET_SIZE(setpil)


/*
 * _insque(entryp, predp)
 *
 * Insert entryp after predp in a doubly linked list.
 */

	ENTRY(_insque)
	ldn	[%o1], %g1		! predp->forw
	stn	%o1, [%o0 + CPTRSIZE]	! entryp->back = predp
	stn	%g1, [%o0]		! entryp->forw = predp->forw
	stn	%o0, [%o1]		! predp->forw = entryp
	retl
	stn	%o0, [%g1 + CPTRSIZE]	! predp->forw->back = entryp
	SET_SIZE(_insque)

/*
 * _remque(entryp)
 *
 * Remove entryp from a doubly linked list
 */

	ENTRY(_remque)
	ldn	[%o0], %g1		! entryp->forw
	ldn	[%o0 + CPTRSIZE], %g2	! entryp->back
	stn	%g1, [%g2]		! entryp->back->forw = entryp->forw
	retl
	stn	%g2, [%g1 + CPTRSIZE]	! entryp->forw->back = entryp->back
	SET_SIZE(_remque)


/*
 * strlen(str)
 *
 * Returns the number of non-NULL bytes in string argument.
 *
 * XXX -  why is this here, rather than the traditional file?
 *	  why does it have local labels which don't start with a `.'?
 */

	ENTRY(strlen)
	mov	%o0, %o1
	andcc	%o1, 3, %o3		! is src word aligned
	bz	$nowalgnd
	clr	%o0			! length of non-zero bytes
	cmp	%o3, 2			! is src half-word aligned
	be	$s2algn
	cmp	%o3, 3			! src is byte aligned
	ldub	[%o1], %o3		! move 1 or 3 bytes to align it
	inc	1, %o1			! in either case, safe to do a byte
	be	$s3algn
	tst	%o3
$s1algn:
	bnz,a	$s2algn			! now go align dest
	inc	1, %o0
	b,a	$done

$s2algn:
	lduh	[%o1], %o3		! know src is half-byte aligned
	inc	2, %o1
	srl	%o3, 8, %o4
	tst	%o4			! is the first byte zero
	bnz,a	1f
	inc	%o0
	b,a	$done
1:	andcc	%o3, 0xff, %o3		! is the second byte zero
	bnz,a	$nowalgnd
	inc	%o0
	b,a	$done
$s3algn:
	bnz,a	$nowalgnd
	inc	1, %o0
	b,a	$done

$nowalgnd:
	! use trick to check if any read bytes of a word are zero
	! the following two constants will generate "byte carries"
	! and check if any bit in a byte is set, if all characters
	! are 7bits (unsigned) this allways works, otherwise
	! there is a specil case that rarely happens, see below

	set	0x7efefeff, %o3
	set	0x81010100, %o4

3:	ld	[%o1], %o2		! main loop
	inc	4, %o1
	add	%o2, %o3, %o5		! generate byte-carries
	xor	%o5, %o2, %o5		! see if orignal bits set
	and	%o5, %o4, %o5
	cmp	%o5, %o4		! if ==,  no zero bytes
	be,a	3b
	inc	4, %o0

	! check for the zero byte and increment the count appropriately
	! some information (the carry bit) is lost if bit 31
	! was set (very rare), if this is the rare condition,
	! return to the main loop again

	sethi	%hi(0xff000000), %o5	! mask used to test for terminator
	andcc	%o2, %o5, %g0		! check if first byte was zero
	bnz	1f
	srl	%o5, 8, %o5
$done:
	retl
	nop
1:	andcc	%o2, %o5, %g0		! check if second byte was zero
	bnz	1f
	srl	%o5, 8, %o5
$done1:
	retl
	inc	%o0
1:	andcc 	%o2, %o5, %g0		! check if third byte was zero
	bnz	1f
	andcc	%o2, 0xff, %g0		! check if last byte is zero
$done2:
	retl
	inc	2, %o0
1:	bnz,a	3b
	inc	4, %o0			! count of bytes
$done3:
	retl
	inc	3, %o0
	SET_SIZE(strlen)

/*
 * Provide a C callable interface to the membar instruction.
 */

	ENTRY(membar_ldld)
	retl
	membar	#LoadLoad
	SET_SIZE(membar_ldld)

	ENTRY(membar_stld)
	retl
	membar	#StoreLoad
	SET_SIZE(membar_stld)

	ENTRY(membar_ldst)
	retl
	membar	#LoadStore
	SET_SIZE(membar_ldst)

	ENTRY(membar_stst)
	retl
	membar	#StoreStore
	SET_SIZE(membar_stst)

	ENTRY(membar_ldld_stld)
	ALTENTRY(membar_stld_ldld)
	retl
	membar	#LoadLoad|#StoreLoad
	SET_SIZE(membar_stld_ldld)
	SET_SIZE(membar_ldld_stld)

	ENTRY(membar_ldld_ldst)
	ALTENTRY(membar_ldst_ldld)
	retl
	membar	#LoadLoad|#LoadStore
	SET_SIZE(membar_ldst_ldld)
	SET_SIZE(membar_ldld_ldst)

	ENTRY(membar_ldld_stst)
	ALTENTRY(membar_stst_ldld)
	retl
	membar	#LoadLoad|#StoreStore
	SET_SIZE(membar_stst_ldld)
	SET_SIZE(membar_ldld_stst)

	ENTRY(membar_stld_ldst)
	ALTENTRY(membar_ldst_stld)
	retl
	membar	#StoreLoad|#LoadStore
	SET_SIZE(membar_ldst_stld)
	SET_SIZE(membar_stld_ldst)

	ENTRY(membar_stld_stst)
	ALTENTRY(membar_stst_stld)
	retl
	membar	#StoreLoad|#StoreStore
	SET_SIZE(membar_stst_stld)
	SET_SIZE(membar_stld_stst)

	ENTRY(membar_ldst_stst)
	ALTENTRY(membar_stst_ldst)
	retl
	membar	#LoadStore|#StoreStore
	SET_SIZE(membar_stst_ldst)
	SET_SIZE(membar_ldst_stst)

	ENTRY(membar_lookaside)
	retl
	membar	#Lookaside
	SET_SIZE(membar_lookaside)

	ENTRY(membar_memissue)
	retl
	membar	#MemIssue
	SET_SIZE(membar_memissue)

	ENTRY(membar_sync)
	retl
	membar	#Sync
	SET_SIZE(membar_sync)


/*
 * Since all of the fuword() variants are so similar, we have a macro to spit
 * them out.
 */

#define	FUWORD(NAME, LOAD, STORE, COPYOP)	\
	ENTRY(NAME);				\
	sethi	%hi(1f), %o5;			\
	ldn	[THREAD_REG + T_LOFAULT], %o3;	\
	or	%o5, %lo(1f), %o5;		\
	membar	#Sync;				\
	stn	%o5, [THREAD_REG + T_LOFAULT];	\
	LOAD	[%o0]ASI_USER, %o2;		\
	membar	#Sync;				\
	stn	%o3, [THREAD_REG + T_LOFAULT];	\
	mov	0, %o0;				\
	retl;					\
	STORE	%o2, [%o1];			\
1:						\
	membar	#Sync;				\
	stn	%o3, [THREAD_REG + T_LOFAULT];	\
	ldn	[THREAD_REG + T_COPYOPS], %o2;	\
	brz	%o2, 2f;			\
	nop;					\
	ldn	[%o2 + COPYOP], %g1;		\
	jmp	%g1;				\
	nop;					\
2:						\
	retl;					\
	mov	-1, %o0;			\
	SET_SIZE(NAME)

	FUWORD(fuword64, ldxa, stx, CP_FUWORD64)
	FUWORD(fuword32, lda, st, CP_FUWORD32)
	FUWORD(fuword16, lduha, sth, CP_FUWORD16)
	FUWORD(fuword8, lduba, stb, CP_FUWORD8)


/*
 * Since all of the suword() variants are so similar, we have a macro to spit
 * them out.
 */

#define	SUWORD(NAME, STORE, COPYOP)		\
	ENTRY(NAME)				\
	sethi	%hi(1f), %o5;			\
	ldn	[THREAD_REG + T_LOFAULT], %o3;	\
	or	%o5, %lo(1f), %o5;		\
	membar	#Sync;				\
	stn	%o5, [THREAD_REG + T_LOFAULT];	\
	STORE	%o1, [%o0]ASI_USER;		\
	membar	#Sync;				\
	stn	%o3, [THREAD_REG + T_LOFAULT];	\
	retl;					\
	clr	%o0;				\
1:						\
	membar	#Sync;				\
	stn	%o3, [THREAD_REG + T_LOFAULT];	\
	ldn	[THREAD_REG + T_COPYOPS], %o2;	\
	brz	%o2, 2f;			\
	nop;					\
	ldn	[%o2 + COPYOP], %g1;		\
	jmp	%g1;				\
	nop;					\
2:						\
	retl;					\
	mov	-1, %o0;			\
	SET_SIZE(NAME)

	SUWORD(suword64, stxa, CP_SUWORD64)
	SUWORD(suword32, sta, CP_SUWORD32)
	SUWORD(suword16, stha, CP_SUWORD16)
	SUWORD(suword8, stba, CP_SUWORD8)

	ENTRY(fuword8_noerr)
	lduba	[%o0]ASI_USER, %o0	
	retl
	stb	%o0, [%o1]
	SET_SIZE(fuword8_noerr)

	ENTRY(fuword16_noerr)
	lduha	[%o0]ASI_USER, %o0
	retl
	sth	%o0, [%o1]
	SET_SIZE(fuword16_noerr)

	ENTRY(fuword32_noerr)
	lda	[%o0]ASI_USER, %o0
	retl
	st	%o0, [%o1]
	SET_SIZE(fuword32_noerr)

	ENTRY(fuword64_noerr)
	ldxa	[%o0]ASI_USER, %o0
	retl
	stx	%o0, [%o1]
	SET_SIZE(fuword64_noerr)

	ENTRY(suword8_noerr)
	retl
	stba	%o1, [%o0]ASI_USER
	SET_SIZE(suword8_noerr)

	ENTRY(suword16_noerr)
	retl
	stha	%o1, [%o0]ASI_USER
	SET_SIZE(suword16_noerr)

	ENTRY(suword32_noerr)
	retl
	sta	%o1, [%o0]ASI_USER
	SET_SIZE(suword32_noerr)

	ENTRY(suword64_noerr)
	retl
	stxa	%o1, [%o0]ASI_USER
	SET_SIZE(suword64_noerr)

	.weak	subyte
	subyte=suword8
	.weak	subyte_noerr
	subyte_noerr=suword8_noerr
#ifdef _LP64
	.weak	fulword
	fulword=fuword64
	.weak	fulword_noerr
	fulword_noerr=fuword64_noerr
	.weak	sulword
	sulword=suword64
	.weak	sulword_noerr
	sulword_noerr=suword64_noerr
#else
	.weak	fulword
	fulword=fuword32
	.weak	fulword_noerr
	fulword_noerr=fuword32_noerr
	.weak	sulword
	sulword=suword32
	.weak	sulword_noerr
	sulword_noerr=suword32_noerr
#endif	/* LP64 */

/*
 * We define rdtick here, but not for sun4v. On sun4v systems, the %tick
 * and %stick should not be read directly without considering the tick
 * and stick offset kernel variables introduced to support sun4v OS
 * suspension.
 */
#if !defined (sun4v)

	ENTRY(rdtick)
	retl
	rd	%tick, %o0
        SET_SIZE(rdtick)

#endif /* !sun4v */

/*
 * Set tba to given address, no side effects.
 */

	ENTRY(set_tba)
	mov	%o0, %o1
	rdpr	%tba, %o0
	wrpr	%o1, %tba
	retl
	nop
	SET_SIZE(set_tba)

	ENTRY(get_tba)
	retl
	rdpr	%tba, %o0
	SET_SIZE(get_tba)

	ENTRY_NP(setpstate)
	retl
	wrpr	%g0, %o0, %pstate
	SET_SIZE(setpstate)

	ENTRY_NP(getpstate)
	retl
	rdpr	%pstate, %o0
	SET_SIZE(getpstate)

	ENTRY_NP(dtrace_interrupt_disable)
	rdpr	%pstate, %o0
	andn	%o0, PSTATE_IE, %o1
	retl
	wrpr	%g0, %o1, %pstate
	SET_SIZE(dtrace_interrupt_disable)

	ENTRY_NP(dtrace_interrupt_enable)
	retl
	wrpr	%g0, %o0, %pstate 
	SET_SIZE(dtrace_interrupt_enable)

#ifdef SF_ERRATA_51
	.align 32
	ENTRY(dtrace_membar_return)
	retl
	nop
	SET_SIZE(dtrace_membar_return)
#define	DTRACE_MEMBAR_RETURN	ba,pt %icc, dtrace_membar_return
#else
#define	DTRACE_MEMBAR_RETURN	retl
#endif

	ENTRY(dtrace_membar_producer)
	DTRACE_MEMBAR_RETURN
	membar	#StoreStore
	SET_SIZE(dtrace_membar_producer)

	ENTRY(dtrace_membar_consumer)
	DTRACE_MEMBAR_RETURN
	membar	#LoadLoad
	SET_SIZE(dtrace_membar_consumer)

	ENTRY_NP(dtrace_flush_windows)
	retl
	flushw
	SET_SIZE(dtrace_flush_windows)

	/*
	 * %g1	pcstack
	 * %g2	iteration count
	 * %g3	final %fp
	 * %g4	final %i7
	 * %g5	saved %cwp (so we can get back to the original window)
	 *
	 * %o0	pcstack / return value (iteration count)
	 * %o1	limit / saved %cansave
	 * %o2	lastfp
	 * %o3	lastpc
	 * %o4	saved %canrestore
	 * %o5	saved %pstate (to restore interrupts)
	 *
	 * Note:  The frame pointer returned via lastfp is safe to use as
	 *	long as getpcstack_top() returns either (0) or a value less
	 *	than (limit).
	 */
	ENTRY_NP(getpcstack_top)

	rdpr	%pstate, %o5
	andn	%o5, PSTATE_IE, %g1
	wrpr	%g0, %g1, %pstate	! disable interrupts

	mov	%o0, %g1		! we need the pcstack pointer while
					! we're visiting other windows

	rdpr	%canrestore, %g2	! number of available windows
	sub	%g2, 1, %g2		! account for skipped frame
	cmp	%g2, %o1		! compare with limit
	movg	%icc, %o1, %g2		! %g2 = min(%canrestore-1, limit)

	brlez,a,pn %g2, 3f		! Use slow path if count <= 0 --
	clr	%o0			! return zero.

	mov	%g2, %o0		! set up return value

	rdpr	%cwp, %g5		! remember the register window state
	rdpr	%cansave, %o1		! 'restore' changes, so we can undo
	rdpr	%canrestore, %o4	! its effects when we finish.

	restore				! skip caller's frame
1:
	st	%i7, [%g1]		! stash return address in pcstack
	restore				! go to the next frame
	subcc	%g2, 1, %g2		! decrement the count
	bnz,pt	%icc, 1b		! loop until count reaches 0
	add	%g1, 4, %g1		! increment pcstack

	mov	%i6, %g3		! copy the final %fp and return PC
	mov	%i7, %g4		! aside so we can return them to our
					! caller

	wrpr	%g0, %g5, %cwp		! jump back to the original window
	wrpr	%g0, %o1, %cansave	! and restore the original register
	wrpr	%g0, %o4, %canrestore	! window state.
2:
	stn	%g3, [%o2]		! store the frame pointer and pc
	st	%g4, [%o3]		! so our caller can continue the trace

	retl				! return to caller
	wrpr	%g0, %o5, %pstate	! restore interrupts

3:
	flushw				! flush register windows, then
	ldn	[%fp + STACK_BIAS + 14*CLONGSIZE], %g3	! load initial fp
	ba	2b
	ldn	[%fp + STACK_BIAS + 15*CLONGSIZE], %g4	! and pc
	SET_SIZE(getpcstack_top)

	ENTRY_NP(setwstate)
	retl
	wrpr	%g0, %o0, %wstate
	SET_SIZE(setwstate)


	ENTRY_NP(getwstate)
	retl
	rdpr	%wstate, %o0
	SET_SIZE(getwstate)


/*
 * int panic_trigger(int *tp)
 *
 * A panic trigger is a word which is updated atomically and can only be set
 * once.  We atomically store 0xFF into the high byte and load the old value.
 * If the byte was 0xFF, the trigger has already been activated and we fail.
 * If the previous value was 0 or not 0xFF, we succeed.  This allows a
 * partially corrupt trigger to still trigger correctly.  DTrace has its own
 * version of this function to allow it to panic correctly from probe context.
 */

	ENTRY_NP(panic_trigger)
	ldstub	[%o0], %o0		! store 0xFF, load byte into %o0
	cmp	%o0, 0xFF		! compare %o0 to 0xFF
	set	1, %o1			! %o1 = 1
	be,a	0f			! if (%o0 == 0xFF) goto 0f (else annul)
	set	0, %o1			! delay - %o1 = 0
0:	retl
	mov	%o1, %o0		! return (%o1);
	SET_SIZE(panic_trigger)

	ENTRY_NP(dtrace_panic_trigger)
	ldstub	[%o0], %o0		! store 0xFF, load byte into %o0
	cmp	%o0, 0xFF		! compare %o0 to 0xFF
	set	1, %o1			! %o1 = 1
	be,a	0f			! if (%o0 == 0xFF) goto 0f (else annul)
	set	0, %o1			! delay - %o1 = 0
0:	retl
	mov	%o1, %o0		! return (%o1);
	SET_SIZE(dtrace_panic_trigger)

/*
 * void vpanic(const char *format, va_list alist)
 *
 * The panic() and cmn_err() functions invoke vpanic() as a common entry point
 * into the panic code implemented in panicsys().  vpanic() is responsible
 * for passing through the format string and arguments, and constructing a
 * regs structure on the stack into which it saves the current register
 * values.  If we are not dying due to a fatal trap, these registers will
 * then be preserved in panicbuf as the current processor state.  Before
 * invoking panicsys(), vpanic() activates the first panic trigger (see
 * common/os/panic.c) and switches to the panic_stack if successful.  Note that
 * DTrace takes a slightly different panic path if it must panic from probe
 * context.  Instead of calling panic, it calls into dtrace_vpanic(), which
 * sets up the initial stack as vpanic does, calls dtrace_panic_trigger(), and
 * branches back into vpanic().
 */

	ENTRY_NP(vpanic)

	save	%sp, -SA(MINFRAME + REGSIZE), %sp	! save and allocate regs

	!
	! The v9 struct regs has a 64-bit r_tstate field, which we use here
	! to store the %ccr, %asi, %pstate, and %cwp as they would appear
	! in %tstate if a trap occurred.  We leave it up to the debugger to
	! realize what happened and extract the register values.
	!
	rd	%ccr, %l0				! %l0 = %ccr
	sllx	%l0, TSTATE_CCR_SHIFT, %l0		! %l0 <<= CCR_SHIFT
	rd	%asi, %l1				! %l1 = %asi
	sllx	%l1, TSTATE_ASI_SHIFT, %l1		! %l1 <<= ASI_SHIFT
	or	%l0, %l1, %l0				! %l0 |= %l1
	rdpr	%pstate, %l1				! %l1 = %pstate
	sllx	%l1, TSTATE_PSTATE_SHIFT, %l1		! %l1 <<= PSTATE_SHIFT
	or	%l0, %l1, %l0				! %l0 |= %l1
	rdpr	%cwp, %l1				! %l1 = %cwp
	sllx	%l1, TSTATE_CWP_SHIFT, %l1		! %l1 <<= CWP_SHIFT
	or	%l0, %l1, %l0				! %l0 |= %l1

	set	vpanic, %l1				! %l1 = %pc (vpanic)
	add	%l1, 4, %l2				! %l2 = %npc (vpanic+4)
	rd	%y, %l3					! %l3 = %y
	!
	! Flush register windows before panic_trigger() in order to avoid a
	! problem that a dump hangs if flush_windows() causes another panic.
	!
	call	flush_windows
	nop

	sethi	%hi(panic_quiesce), %o0
	call	panic_trigger
	or	%o0, %lo(panic_quiesce), %o0		! if (!panic_trigger(

vpanic_common:
	tst	%o0					!     &panic_quiesce))
	be	0f					!   goto 0f;
	mov	%o0, %l4				!   delay - %l4 = %o0

	!
	! If panic_trigger() was successful, we are the first to initiate a
	! panic: switch to the panic_stack.
	!
	set	panic_stack, %o0			! %o0 = panic_stack
	set	PANICSTKSIZE, %o1			! %o1 = size of stack
	add	%o0, %o1, %o0				! %o0 = top of stack

	sub	%o0, SA(MINFRAME + REGSIZE) + STACK_BIAS, %sp

	!
	! Now that we've got everything set up, store each register to its
	! designated location in the regs structure allocated on the stack.
	! The register set we store is the equivalent of the registers at
	! the time the %pc was pointing to vpanic, thus the %i's now contain
	! what the %o's contained prior to the save instruction.
	!
0:	stx	%l0, [%sp + STACK_BIAS + SA(MINFRAME) + TSTATE_OFF]
	stx	%g1, [%sp + STACK_BIAS + SA(MINFRAME) + G1_OFF]
	stx	%g2, [%sp + STACK_BIAS + SA(MINFRAME) + G2_OFF]
	stx	%g3, [%sp + STACK_BIAS + SA(MINFRAME) + G3_OFF]
	stx	%g4, [%sp + STACK_BIAS + SA(MINFRAME) + G4_OFF]
	stx	%g5, [%sp + STACK_BIAS + SA(MINFRAME) + G5_OFF]
	stx	%g6, [%sp + STACK_BIAS + SA(MINFRAME) + G6_OFF]
	stx	%g7, [%sp + STACK_BIAS + SA(MINFRAME) + G7_OFF]
	stx	%i0, [%sp + STACK_BIAS + SA(MINFRAME) + O0_OFF]
	stx	%i1, [%sp + STACK_BIAS + SA(MINFRAME) + O1_OFF]
	stx	%i2, [%sp + STACK_BIAS + SA(MINFRAME) + O2_OFF]
	stx	%i3, [%sp + STACK_BIAS + SA(MINFRAME) + O3_OFF]
	stx	%i4, [%sp + STACK_BIAS + SA(MINFRAME) + O4_OFF]
	stx	%i5, [%sp + STACK_BIAS + SA(MINFRAME) + O5_OFF]
	stx	%i6, [%sp + STACK_BIAS + SA(MINFRAME) + O6_OFF]
	stx	%i7, [%sp + STACK_BIAS + SA(MINFRAME) + O7_OFF]
	stn	%l1, [%sp + STACK_BIAS + SA(MINFRAME) + PC_OFF]
	stn	%l2, [%sp + STACK_BIAS + SA(MINFRAME) + NPC_OFF]
	st	%l3, [%sp + STACK_BIAS + SA(MINFRAME) + Y_OFF]

	mov	%l4, %o3				! %o3 = on_panic_stack
	add	%sp, STACK_BIAS + SA(MINFRAME), %o2	! %o2 = &regs
	mov	%i1, %o1				! %o1 = alist
	call	panicsys				! panicsys();
	mov	%i0, %o0				! %o0 = format
	ret
	restore

	SET_SIZE(vpanic)

	ENTRY_NP(dtrace_vpanic)

	save	%sp, -SA(MINFRAME + REGSIZE), %sp	! save and allocate regs

	!
	! The v9 struct regs has a 64-bit r_tstate field, which we use here
	! to store the %ccr, %asi, %pstate, and %cwp as they would appear
	! in %tstate if a trap occurred.  We leave it up to the debugger to
	! realize what happened and extract the register values.
	!
	rd	%ccr, %l0				! %l0 = %ccr
	sllx	%l0, TSTATE_CCR_SHIFT, %l0		! %l0 <<= CCR_SHIFT
	rd	%asi, %l1				! %l1 = %asi
	sllx	%l1, TSTATE_ASI_SHIFT, %l1		! %l1 <<= ASI_SHIFT
	or	%l0, %l1, %l0				! %l0 |= %l1
	rdpr	%pstate, %l1				! %l1 = %pstate
	sllx	%l1, TSTATE_PSTATE_SHIFT, %l1		! %l1 <<= PSTATE_SHIFT
	or	%l0, %l1, %l0				! %l0 |= %l1
	rdpr	%cwp, %l1				! %l1 = %cwp
	sllx	%l1, TSTATE_CWP_SHIFT, %l1		! %l1 <<= CWP_SHIFT
	or	%l0, %l1, %l0				! %l0 |= %l1

	set	dtrace_vpanic, %l1			! %l1 = %pc (vpanic)
	add	%l1, 4, %l2				! %l2 = %npc (vpanic+4)
	rd	%y, %l3					! %l3 = %y
	!
	! Flush register windows before panic_trigger() in order to avoid a
	! problem that a dump hangs if flush_windows() causes another panic.
	!
	call	dtrace_flush_windows
	nop

	sethi	%hi(panic_quiesce), %o0
	call	dtrace_panic_trigger
	or	%o0, %lo(panic_quiesce), %o0		! if (!panic_trigger(

	ba,a	vpanic_common
	SET_SIZE(dtrace_vpanic)
	
	ENTRY(get_subcc_ccr)
	wr	%g0, %ccr	! clear condition codes
	subcc	%o0, %o1, %g0
	retl
	rd	%ccr, %o0	! return condition codes
	SET_SIZE(get_subcc_ccr)

	ENTRY_NP(ftrace_interrupt_disable)
	rdpr	%pstate, %o0
	andn	%o0, PSTATE_IE, %o1
	retl
	wrpr	%g0, %o1, %pstate
	SET_SIZE(ftrace_interrupt_disable)

	ENTRY_NP(ftrace_interrupt_enable)
	retl
	wrpr	%g0, %o0, %pstate 
	SET_SIZE(ftrace_interrupt_enable)