path: root/usr/src/uts/sparc/v9/ml/lock_prim.s

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 *
 * Copyright 2020 Joyent, Inc.
 */

#include "assym.h"

#include <sys/t_lock.h>
#include <sys/mutex.h>
#include <sys/mutex_impl.h>
#include <sys/rwlock_impl.h>
#include <sys/asm_linkage.h>
#include <sys/machlock.h>
#include <sys/machthread.h>
#include <sys/lockstat.h>

/* #define DEBUG */

#ifdef DEBUG
#include <sys/machparam.h>
#endif /* DEBUG */

/************************************************************************
 *		ATOMIC OPERATIONS
 */

/*
 * uint8_t	ldstub(uint8_t *cp)
 *
 * Store 0xFF at the specified location, and return its previous content.
 */

	ENTRY(ldstub)
	retl
	ldstub	[%o0], %o0
	SET_SIZE(ldstub)

/************************************************************************
 *		MEMORY BARRIERS -- see atomic.h for full descriptions.
 */

#ifdef SF_ERRATA_51
	.align 32
	ENTRY(membar_return)
	retl
	nop
	SET_SIZE(membar_return)
#define	MEMBAR_RETURN	ba,pt %icc, membar_return
#else
#define	MEMBAR_RETURN	retl
#endif

	ENTRY(membar_enter)
	MEMBAR_RETURN
	membar	#StoreLoad|#StoreStore
	SET_SIZE(membar_enter)

	ENTRY(membar_exit)
	MEMBAR_RETURN
	membar	#LoadStore|#StoreStore
	SET_SIZE(membar_exit)

	ENTRY(membar_producer)
	MEMBAR_RETURN
	membar	#StoreStore
	SET_SIZE(membar_producer)

	ENTRY(membar_consumer)
	MEMBAR_RETURN
	membar	#LoadLoad
	SET_SIZE(membar_consumer)

/************************************************************************
 *		MINIMUM LOCKS
 */

/*
 * lock_try(lp), ulock_try(lp)
 * - returns non-zero on success.
 * - doesn't block interrupts so don't use this to spin on a lock.
 * - uses "0xFF is busy, anything else is free" model.
 *
 * ulock_try() is for a lock in the user address space.
 */

	.align	32
	ENTRY(lock_try)
	ldstub	[%o0], %o1		! try to set lock, get value in %o1
	brnz,pn	%o1, 1f
	membar	#LoadLoad
.lock_try_lockstat_patch_point:
	retl
	or	%o0, 1, %o0		! ensure lo32 != 0
1:
	retl
	clr	%o0
	SET_SIZE(lock_try)

	.align	32
	ENTRY(lock_spin_try)
	ldstub	[%o0], %o1		! try to set lock, get value in %o1
	brnz,pn	%o1, 1f
	membar	#LoadLoad
	retl
	or	%o0, 1, %o0		! ensure lo32 != 0
1:
	retl
	clr	%o0
	SET_SIZE(lock_spin_try)

	.align	32
	ENTRY(lock_set)
	ldstub	[%o0], %o1
	brnz,pn	%o1, 1f			! go to C for the hard case
	membar	#LoadLoad
.lock_set_lockstat_patch_point:
	retl
	nop
1:
	sethi	%hi(lock_set_spin), %o2	! load up for jump to C
	jmp	%o2 + %lo(lock_set_spin)
	nop				! delay: do nothing
	SET_SIZE(lock_set)

	ENTRY(lock_clear)
	membar	#LoadStore|#StoreStore
.lock_clear_lockstat_patch_point:
	retl
	clrb	[%o0]
	SET_SIZE(lock_clear)

	.align	32
	ENTRY(ulock_try)
	ldstuba	[%o0]ASI_USER, %o1	! try to set lock, get value in %o1
	xor	%o1, 0xff, %o0		! delay - return non-zero if success
	retl
	  membar	#LoadLoad
	SET_SIZE(ulock_try)

	ENTRY(ulock_clear)
	membar  #LoadStore|#StoreStore
	retl
	  stba	%g0, [%o0]ASI_USER	! clear lock
	SET_SIZE(ulock_clear)


/*
 * lock_set_spl(lp, new_pil, *old_pil_addr)
 *	Sets pil to new_pil, grabs lp, stores old pil in *old_pil_addr.
 */

	ENTRY(lock_set_spl)
	rdpr	%pil, %o3			! %o3 = current pil
	cmp	%o3, %o1			! is current pil high enough?
	bl,a,pt %icc, 1f			! if not, write %pil in delay
	wrpr	%g0, %o1, %pil
1:
	ldstub	[%o0], %o4			! try the lock
	brnz,pn	%o4, 2f				! go to C for the miss case
	membar	#LoadLoad
.lock_set_spl_lockstat_patch_point:
	retl
	sth	%o3, [%o2]			! delay - save original pil
2:
	sethi	%hi(lock_set_spl_spin), %o5	! load up jmp to C
	jmp	%o5 + %lo(lock_set_spl_spin)	! jmp to lock_set_spl_spin
	nop					! delay: do nothing
	SET_SIZE(lock_set_spl)

/*
 * lock_clear_splx(lp, s)
 */

	ENTRY(lock_clear_splx)
	ldn	[THREAD_REG + T_CPU], %o2	! get CPU pointer
	membar	#LoadStore|#StoreStore
	ld	[%o2 + CPU_BASE_SPL], %o2
	clrb	[%o0]				! clear lock
	cmp	%o2, %o1			! compare new to base
	movl	%xcc, %o1, %o2			! use new pri if base is less
.lock_clear_splx_lockstat_patch_point:
	retl
	wrpr	%g0, %o2, %pil
	SET_SIZE(lock_clear_splx)

/*
 * mutex_enter() and mutex_exit().
 *
 * These routines handle the simple cases of mutex_enter() (adaptive
 * lock, not held) and mutex_exit() (adaptive lock, held, no waiters).
 * If anything complicated is going on we punt to mutex_vector_enter().
 *
 * mutex_tryenter() is similar to mutex_enter() but returns zero if
 * the lock cannot be acquired, nonzero on success.
 *
 * If mutex_exit() gets preempted in the window between checking waiters
 * and clearing the lock, we can miss wakeups.  Disabling preemption
 * in the mutex code is prohibitively expensive, so instead we detect
 * mutex preemption by examining the trapped PC in the interrupt path.
 * If we interrupt a thread in mutex_exit() that has not yet cleared
 * the lock, pil_interrupt() resets its PC back to the beginning of
 * mutex_exit() so it will check again for waiters when it resumes.
 *
 * The lockstat code below is activated when the lockstat driver
 * calls lockstat_hot_patch() to hot-patch the kernel mutex code.
 * Note that we don't need to test lockstat_event_mask here -- we won't
 * patch this code in unless we're gathering ADAPTIVE_HOLD lockstats.
 */

	.align	32
	ENTRY(mutex_enter)
	mov	THREAD_REG, %o1
	casx	[%o0], %g0, %o1			! try to acquire as adaptive
	brnz,pn	%o1, 1f				! locked or wrong type
	membar	#LoadLoad
.mutex_enter_lockstat_patch_point:
	retl
	nop
1:
	sethi	%hi(mutex_vector_enter), %o2	! load up for jump to C
	jmp	%o2 + %lo(mutex_vector_enter)
	nop
	SET_SIZE(mutex_enter)

	ENTRY(mutex_tryenter)
	mov	THREAD_REG, %o1
	casx	[%o0], %g0, %o1			! try to acquire as adaptive
	brnz,pn	%o1, 1f				! locked or wrong type continue
	membar	#LoadLoad
.mutex_tryenter_lockstat_patch_point:
	retl
	or	%o0, 1, %o0			! ensure lo32 != 0
1:
	sethi	%hi(mutex_vector_tryenter), %o2		! hi bits
	jmp	%o2 + %lo(mutex_vector_tryenter)	! go to C
	nop
	SET_SIZE(mutex_tryenter)

	ENTRY(mutex_adaptive_tryenter)
	mov	THREAD_REG, %o1
	casx	[%o0], %g0, %o1			! try to acquire as adaptive
	brnz,pn	%o1, 0f				! locked or wrong type
	membar	#LoadLoad
	retl
	or	%o0, 1, %o0			! ensure lo32 != 0
0:
	retl
	mov	%g0, %o0
	SET_SIZE(mutex_adaptive_tryenter)

	! these need to be together and cache aligned for performance.
	.align 64
	.global	mutex_exit_critical_size
	.global	mutex_exit_critical_start
	.global mutex_owner_running_critical_size
	.global mutex_owner_running_critical_start

mutex_exit_critical_size = .mutex_exit_critical_end - mutex_exit_critical_start

	.align	32

	ENTRY(mutex_exit)
mutex_exit_critical_start:		! If we are interrupted, restart here
	ldn	[%o0], %o1		! get the owner field
	membar	#LoadStore|#StoreStore
	cmp	THREAD_REG, %o1		! do we own lock with no waiters?
	be,a,pt	%ncc, 1f		! if so, drive on ...
	stn	%g0, [%o0]		! delay: clear lock if we owned it
.mutex_exit_critical_end:		! for pil_interrupt() hook
	ba,a,pt	%xcc, mutex_vector_exit	! go to C for the hard cases
1:
.mutex_exit_lockstat_patch_point:
	retl
	nop
	SET_SIZE(mutex_exit)

mutex_owner_running_critical_size = .mutex_owner_running_critical_end - mutex_owner_running_critical_start

	.align  32

	ENTRY(mutex_owner_running)
mutex_owner_running_critical_start:	! If interrupted restart here
	ldn	[%o0], %o1		! get the owner field
	and	%o1, MUTEX_THREAD, %o1	! remove the waiters bit if any
	brz,pn	%o1, 1f			! if so, drive on ...
	nop
	ldn	[%o1+T_CPU], %o2	! get owner->t_cpu
	ldn	[%o2+CPU_THREAD], %o3	! get owner->t_cpu->cpu_thread
.mutex_owner_running_critical_end:	! for pil_interrupt() hook
	cmp	%o1, %o3		! owner == running thread?
	be,a,pt	%xcc, 2f		! yes, go return cpu
	nop
1:
	retl
	mov	%g0, %o0		! return 0 (owner not running)
2:
	retl
	mov	%o2, %o0		! owner running, return cpu
	SET_SIZE(mutex_owner_running)

/*
 * rw_enter() and rw_exit().
 *
 * These routines handle the simple cases of rw_enter (write-locking an unheld
 * lock or read-locking a lock that's neither write-locked nor write-wanted)
 * and rw_exit (no waiters or not the last reader).  If anything complicated
 * is going on we punt to rw_enter_sleep() and rw_exit_wakeup(), respectively.
 */

	.align	16
	ENTRY(rw_enter)
	cmp	%o1, RW_WRITER			! entering as writer?
	be,a,pn	%icc, 2f			! if so, go do it ...
	or	THREAD_REG, RW_WRITE_LOCKED, %o5 ! delay: %o5 = owner
	ldn	[%o0], %o4			! %o4 = old lock value
1:
	andcc	%o4, RW_WRITE_CLAIMED, %g0	! write-locked or write-wanted?
	bz,pt	%xcc, 3f			! if so, prepare to block
	add	%o4, RW_READ_LOCK, %o5		! delay: increment hold count
	sethi	%hi(rw_enter_sleep), %o2	! load up jump
	jmp	%o2 + %lo(rw_enter_sleep)	! jmp to rw_enter_sleep
	nop					! delay: do nothing
3:
	casx	[%o0], %o4, %o5			! try to grab read lock
	cmp	%o4, %o5			! did we get it?
#ifdef sun4v
	be,a,pt %xcc, 0f
	membar  #LoadLoad
	sethi	%hi(rw_enter_sleep), %o2	! load up jump
	jmp	%o2 + %lo(rw_enter_sleep)	! jmp to rw_enter_sleep
	nop					! delay: do nothing
0:
#else /* sun4v */
	bne,pn	%xcc, 1b			! if not, try again
	mov	%o5, %o4			! delay: %o4 = old lock value
	membar	#LoadLoad
#endif /* sun4v */
.rw_read_enter_lockstat_patch_point:
	retl
	nop
2:
	casx	[%o0], %g0, %o5			! try to grab write lock
	brz,pt %o5, 4f				! branch around if we got it
	membar	#LoadLoad			! done regardless of where we go
	sethi	%hi(rw_enter_sleep), %o2
	jmp	%o2 + %lo(rw_enter_sleep)	! jump to rw_enter_sleep if not
	nop					! delay: do nothing
4:
.rw_write_enter_lockstat_patch_point:
	retl
	nop
	SET_SIZE(rw_enter)

	.align	16
	ENTRY(rw_exit)
	ldn	[%o0], %o4			! %o4 = old lock value
	membar	#LoadStore|#StoreStore		! membar_exit()
	subcc	%o4, RW_READ_LOCK, %o5		! %o5 = new lock value if reader
	bnz,pn	%xcc, 2f			! single reader, no waiters?
	clr	%o1
1:
	srl	%o4, RW_HOLD_COUNT_SHIFT, %o3	! %o3 = hold count (lockstat)
	casx	[%o0], %o4, %o5			! try to drop lock
	cmp	%o4, %o5			! did we succeed?
	bne,pn	%xcc, rw_exit_wakeup		! if not, go to C
	nop					! delay: do nothing
.rw_read_exit_lockstat_patch_point:
	retl
	nop					! delay: do nothing
2:
	andcc	%o4, RW_WRITE_LOCKED, %g0	! are we a writer?
	bnz,a,pt %xcc, 3f
	or	THREAD_REG, RW_WRITE_LOCKED, %o4 ! delay: %o4 = owner
	cmp	%o5, RW_READ_LOCK		! would lock still be held?
	bge,pt	%xcc, 1b			! if so, go ahead and drop it
	nop
	ba,pt	%xcc, rw_exit_wakeup		! otherwise, wake waiters
	nop
3:
	casx	[%o0], %o4, %o1			! try to drop write lock
	cmp	%o4, %o1			! did we succeed?
	bne,pn	%xcc, rw_exit_wakeup		! if not, go to C
	nop
.rw_write_exit_lockstat_patch_point:
	retl
	nop
	SET_SIZE(rw_exit)

#define	RETL			0x81c3e008
#define	NOP			0x01000000
#define BA			0x10800000

#define	DISP22			((1 << 22) - 1)
#define	ANNUL			0x20000000

#define	HOT_PATCH_COMMON(addr, event, normal_instr, annul, rs)		\
	ba	1f;							\
	rd	%pc, %o0;						\
	save	%sp, -SA(MINFRAME), %sp;				\
	set	lockstat_probemap, %l1;					\
	ld	[%l1 + (event * DTRACE_IDSIZE)], %o0;			\
	brz,pn	%o0, 0f;						\
	ldub	[THREAD_REG + T_LOCKSTAT], %l0;				\
	add	%l0, 1, %l2;						\
	stub	%l2, [THREAD_REG + T_LOCKSTAT];				\
	set	lockstat_probe, %g1;					\
	ld	[%l1 + (event * DTRACE_IDSIZE)], %o0;			\
	brz,a,pn %o0, 0f;						\
	stub	%l0, [THREAD_REG + T_LOCKSTAT];				\
	ldn	[%g1], %g2;						\
	mov	rs, %o2;						\
	jmpl	%g2, %o7;						\
	mov	%i0, %o1;						\
	stub	%l0, [THREAD_REG + T_LOCKSTAT];				\
0:	ret;								\
	restore	%g0, 1, %o0;	/* for mutex_tryenter / lock_try */	\
1:	set	addr, %o1;						\
	sub	%o0, %o1, %o0;						\
	srl	%o0, 2, %o0;						\
	inc	%o0;							\
	set	DISP22, %o1;						\
	and	%o1, %o0, %o0;						\
	set	BA, %o1;						\
	or	%o1, %o0, %o0;						\
	sethi	%hi(annul), %o2;					\
	add	%o0, %o2, %o2;						\
	set	addr, %o0;						\
	set	normal_instr, %o1;					\
	ld	[%i0 + (event * DTRACE_IDSIZE)], %o3;			\
	tst	%o3;							\
	movnz	%icc, %o2, %o1;						\
	call	hot_patch_kernel_text;					\
	mov	4, %o2;							\
	membar	#Sync

#define	HOT_PATCH(addr, event, normal_instr)	\
	HOT_PATCH_COMMON(addr, event, normal_instr, 0, %i1)

#define	HOT_PATCH_ARG(addr, event, normal_instr, arg)	\
	HOT_PATCH_COMMON(addr, event, normal_instr, 0, arg)

#define HOT_PATCH_ANNULLED(addr, event, normal_instr)	\
	HOT_PATCH_COMMON(addr, event, normal_instr, ANNUL, %i1)

	ENTRY(lockstat_hot_patch)
	save	%sp, -SA(MINFRAME), %sp
	set	lockstat_probemap, %i0
	HOT_PATCH(.mutex_enter_lockstat_patch_point,
		LS_MUTEX_ENTER_ACQUIRE, RETL)
	HOT_PATCH_ANNULLED(.mutex_tryenter_lockstat_patch_point,
		LS_MUTEX_TRYENTER_ACQUIRE, RETL)
	HOT_PATCH(.mutex_exit_lockstat_patch_point,
		LS_MUTEX_EXIT_RELEASE, RETL)
	HOT_PATCH(.rw_write_enter_lockstat_patch_point,
		LS_RW_ENTER_ACQUIRE, RETL)
	HOT_PATCH(.rw_read_enter_lockstat_patch_point,
		LS_RW_ENTER_ACQUIRE, RETL)
	HOT_PATCH_ARG(.rw_write_exit_lockstat_patch_point,
		LS_RW_EXIT_RELEASE, RETL, RW_WRITER)
	HOT_PATCH_ARG(.rw_read_exit_lockstat_patch_point,
		LS_RW_EXIT_RELEASE, RETL, RW_READER)
	HOT_PATCH(.lock_set_lockstat_patch_point,
		LS_LOCK_SET_ACQUIRE, RETL)
	HOT_PATCH_ANNULLED(.lock_try_lockstat_patch_point,
		LS_LOCK_TRY_ACQUIRE, RETL)
	HOT_PATCH(.lock_clear_lockstat_patch_point,
		LS_LOCK_CLEAR_RELEASE, RETL)
	HOT_PATCH(.lock_set_spl_lockstat_patch_point,
		LS_LOCK_SET_SPL_ACQUIRE, RETL)
	HOT_PATCH(.lock_clear_splx_lockstat_patch_point,
		LS_LOCK_CLEAR_SPLX_RELEASE, RETL)
	ret
	restore
	SET_SIZE(lockstat_hot_patch)

/*
 * asm_mutex_spin_enter(mutex_t *)
 *
 * For use by assembly interrupt handler only.
 * Does not change spl, since the interrupt handler is assumed to be
 * running at high level already.
 * Traps may be off, so cannot panic.
 * Does not keep statistics on the lock.
 *
 * Entry:	%l6 - points to mutex
 *		%l7 - address of call (returns to %l7+8)
 * Uses:	%l6, %l5
 */
	.align 16
	ENTRY_NP(asm_mutex_spin_enter)
	ldstub	[%l6 + M_SPINLOCK], %l5	! try to set lock, get value in %l5
1:
	tst	%l5
	bnz	3f			! lock already held - go spin
	nop
2:
	jmp	%l7 + 8			! return
	membar	#LoadLoad
	!
	! Spin on lock without using an atomic operation to prevent the caches
	! from unnecessarily moving ownership of the line around.
	!
3:
	ldub	[%l6 + M_SPINLOCK], %l5
4:
	tst	%l5
	bz,a	1b			! lock appears to be free, try again
	ldstub	[%l6 + M_SPINLOCK], %l5	! delay slot - try to set lock

	sethi	%hi(panicstr) , %l5
	ldn	[%l5 + %lo(panicstr)], %l5
	tst	%l5
	bnz	2b			! after panic, feign success
	nop
	b	4b
	ldub	[%l6 + M_SPINLOCK], %l5	! delay - reload lock
	SET_SIZE(asm_mutex_spin_enter)

/*
 * asm_mutex_spin_exit(mutex_t *)
 *
 * For use by assembly interrupt handler only.
 * Does not change spl, since the interrupt handler is assumed to be
 * running at high level already.
 *
 * Entry:	%l6 - points to mutex
 *		%l7 - address of call (returns to %l7+8)
 * Uses:	none
 */
	ENTRY_NP(asm_mutex_spin_exit)
	membar	#LoadStore|#StoreStore
	jmp	%l7 + 8			! return
	clrb	[%l6 + M_SPINLOCK]	! delay - clear lock
	SET_SIZE(asm_mutex_spin_exit)

/*
 * thread_onproc()
 * Set thread in onproc state for the specified CPU.
 * Also set the thread lock pointer to the CPU's onproc lock.
 * Since the new lock isn't held, the store ordering is important.
 * If not done in assembler, the compiler could reorder the stores.
 */

	ENTRY(thread_onproc)
	set	TS_ONPROC, %o2		! TS_ONPROC state
	st	%o2, [%o0 + T_STATE]	! store state
	add	%o1, CPU_THREAD_LOCK, %o3 ! pointer to disp_lock while running
	retl				! return
	stn	%o3, [%o0 + T_LOCKP]	! delay - store new lock pointer
	SET_SIZE(thread_onproc)

/* delay function used in some mutex code - just do 3 nop cas ops */
	ENTRY(cas_delay)
	casx [%o0], %g0, %g0
	casx [%o0], %g0, %g0
	retl
	casx [%o0], %g0, %g0
	SET_SIZE(cas_delay)

/*
 * alternative delay function for some niagara processors.   The rd
 * instruction uses less resources than casx on those cpus.
 */
	ENTRY(rdccr_delay)
	rd	%ccr, %g0
	rd	%ccr, %g0
	retl
	rd	%ccr, %g0
	SET_SIZE(rdccr_delay)

/*
 * mutex_delay_default(void)
 * Spins for approx a few hundred processor cycles and returns to caller.
 */

	ENTRY(mutex_delay_default)
	mov	72,%o0
1:	brgz	%o0, 1b
	dec	%o0
	retl
	nop
	SET_SIZE(mutex_delay_default)