/* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2008 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. * * Copyright 2020 Joyent, Inc. */ #include "assym.h" #include <sys/t_lock.h> #include <sys/mutex.h> #include <sys/mutex_impl.h> #include <sys/rwlock_impl.h> #include <sys/asm_linkage.h> #include <sys/machlock.h> #include <sys/machthread.h> #include <sys/lockstat.h> /* #define DEBUG */ #ifdef DEBUG #include <sys/machparam.h> #endif /* DEBUG */ /************************************************************************ * ATOMIC OPERATIONS */ /* * uint8_t ldstub(uint8_t *cp) * * Store 0xFF at the specified location, and return its previous content. */ ENTRY(ldstub) retl ldstub [%o0], %o0 SET_SIZE(ldstub) /************************************************************************ * MEMORY BARRIERS -- see atomic.h for full descriptions. */ #ifdef SF_ERRATA_51 .align 32 ENTRY(membar_return) retl nop SET_SIZE(membar_return) #define MEMBAR_RETURN ba,pt %icc, membar_return #else #define MEMBAR_RETURN retl #endif ENTRY(membar_enter) MEMBAR_RETURN membar #StoreLoad|#StoreStore SET_SIZE(membar_enter) ENTRY(membar_exit) MEMBAR_RETURN membar #LoadStore|#StoreStore SET_SIZE(membar_exit) ENTRY(membar_producer) MEMBAR_RETURN membar #StoreStore SET_SIZE(membar_producer) ENTRY(membar_consumer) MEMBAR_RETURN membar #LoadLoad SET_SIZE(membar_consumer) /************************************************************************ * MINIMUM LOCKS */ /* * lock_try(lp), ulock_try(lp) * - returns non-zero on success. * - doesn't block interrupts so don't use this to spin on a lock. * - uses "0xFF is busy, anything else is free" model. * * ulock_try() is for a lock in the user address space. */ .align 32 ENTRY(lock_try) ldstub [%o0], %o1 ! try to set lock, get value in %o1 brnz,pn %o1, 1f membar #LoadLoad .lock_try_lockstat_patch_point: retl or %o0, 1, %o0 ! ensure lo32 != 0 1: retl clr %o0 SET_SIZE(lock_try) .align 32 ENTRY(lock_spin_try) ldstub [%o0], %o1 ! try to set lock, get value in %o1 brnz,pn %o1, 1f membar #LoadLoad retl or %o0, 1, %o0 ! ensure lo32 != 0 1: retl clr %o0 SET_SIZE(lock_spin_try) .align 32 ENTRY(lock_set) ldstub [%o0], %o1 brnz,pn %o1, 1f ! go to C for the hard case membar #LoadLoad .lock_set_lockstat_patch_point: retl nop 1: sethi %hi(lock_set_spin), %o2 ! load up for jump to C jmp %o2 + %lo(lock_set_spin) nop ! delay: do nothing SET_SIZE(lock_set) ENTRY(lock_clear) membar #LoadStore|#StoreStore .lock_clear_lockstat_patch_point: retl clrb [%o0] SET_SIZE(lock_clear) .align 32 ENTRY(ulock_try) ldstuba [%o0]ASI_USER, %o1 ! try to set lock, get value in %o1 xor %o1, 0xff, %o0 ! delay - return non-zero if success retl membar #LoadLoad SET_SIZE(ulock_try) ENTRY(ulock_clear) membar #LoadStore|#StoreStore retl stba %g0, [%o0]ASI_USER ! clear lock SET_SIZE(ulock_clear) /* * lock_set_spl(lp, new_pil, *old_pil_addr) * Sets pil to new_pil, grabs lp, stores old pil in *old_pil_addr. */ ENTRY(lock_set_spl) rdpr %pil, %o3 ! %o3 = current pil cmp %o3, %o1 ! is current pil high enough? bl,a,pt %icc, 1f ! if not, write %pil in delay wrpr %g0, %o1, %pil 1: ldstub [%o0], %o4 ! try the lock brnz,pn %o4, 2f ! go to C for the miss case membar #LoadLoad .lock_set_spl_lockstat_patch_point: retl sth %o3, [%o2] ! delay - save original pil 2: sethi %hi(lock_set_spl_spin), %o5 ! load up jmp to C jmp %o5 + %lo(lock_set_spl_spin) ! jmp to lock_set_spl_spin nop ! delay: do nothing SET_SIZE(lock_set_spl) /* * lock_clear_splx(lp, s) */ ENTRY(lock_clear_splx) ldn [THREAD_REG + T_CPU], %o2 ! get CPU pointer membar #LoadStore|#StoreStore ld [%o2 + CPU_BASE_SPL], %o2 clrb [%o0] ! clear lock cmp %o2, %o1 ! compare new to base movl %xcc, %o1, %o2 ! use new pri if base is less .lock_clear_splx_lockstat_patch_point: retl wrpr %g0, %o2, %pil SET_SIZE(lock_clear_splx) /* * mutex_enter() and mutex_exit(). * * These routines handle the simple cases of mutex_enter() (adaptive * lock, not held) and mutex_exit() (adaptive lock, held, no waiters). * If anything complicated is going on we punt to mutex_vector_enter(). * * mutex_tryenter() is similar to mutex_enter() but returns zero if * the lock cannot be acquired, nonzero on success. * * If mutex_exit() gets preempted in the window between checking waiters * and clearing the lock, we can miss wakeups. Disabling preemption * in the mutex code is prohibitively expensive, so instead we detect * mutex preemption by examining the trapped PC in the interrupt path. * If we interrupt a thread in mutex_exit() that has not yet cleared * the lock, pil_interrupt() resets its PC back to the beginning of * mutex_exit() so it will check again for waiters when it resumes. * * The lockstat code below is activated when the lockstat driver * calls lockstat_hot_patch() to hot-patch the kernel mutex code. * Note that we don't need to test lockstat_event_mask here -- we won't * patch this code in unless we're gathering ADAPTIVE_HOLD lockstats. */ .align 32 ENTRY(mutex_enter) mov THREAD_REG, %o1 casx [%o0], %g0, %o1 ! try to acquire as adaptive brnz,pn %o1, 1f ! locked or wrong type membar #LoadLoad .mutex_enter_lockstat_patch_point: retl nop 1: sethi %hi(mutex_vector_enter), %o2 ! load up for jump to C jmp %o2 + %lo(mutex_vector_enter) nop SET_SIZE(mutex_enter) ENTRY(mutex_tryenter) mov THREAD_REG, %o1 casx [%o0], %g0, %o1 ! try to acquire as adaptive brnz,pn %o1, 1f ! locked or wrong type continue membar #LoadLoad .mutex_tryenter_lockstat_patch_point: retl or %o0, 1, %o0 ! ensure lo32 != 0 1: sethi %hi(mutex_vector_tryenter), %o2 ! hi bits jmp %o2 + %lo(mutex_vector_tryenter) ! go to C nop SET_SIZE(mutex_tryenter) ENTRY(mutex_adaptive_tryenter) mov THREAD_REG, %o1 casx [%o0], %g0, %o1 ! try to acquire as adaptive brnz,pn %o1, 0f ! locked or wrong type membar #LoadLoad retl or %o0, 1, %o0 ! ensure lo32 != 0 0: retl mov %g0, %o0 SET_SIZE(mutex_adaptive_tryenter) ! these need to be together and cache aligned for performance. .align 64 .global mutex_exit_critical_size .global mutex_exit_critical_start .global mutex_owner_running_critical_size .global mutex_owner_running_critical_start mutex_exit_critical_size = .mutex_exit_critical_end - mutex_exit_critical_start .align 32 ENTRY(mutex_exit) mutex_exit_critical_start: ! If we are interrupted, restart here ldn [%o0], %o1 ! get the owner field membar #LoadStore|#StoreStore cmp THREAD_REG, %o1 ! do we own lock with no waiters? be,a,pt %ncc, 1f ! if so, drive on ... stn %g0, [%o0] ! delay: clear lock if we owned it .mutex_exit_critical_end: ! for pil_interrupt() hook ba,a,pt %xcc, mutex_vector_exit ! go to C for the hard cases 1: .mutex_exit_lockstat_patch_point: retl nop SET_SIZE(mutex_exit) mutex_owner_running_critical_size = .mutex_owner_running_critical_end - mutex_owner_running_critical_start .align 32 ENTRY(mutex_owner_running) mutex_owner_running_critical_start: ! If interrupted restart here ldn [%o0], %o1 ! get the owner field and %o1, MUTEX_THREAD, %o1 ! remove the waiters bit if any brz,pn %o1, 1f ! if so, drive on ... nop ldn [%o1+T_CPU], %o2 ! get owner->t_cpu ldn [%o2+CPU_THREAD], %o3 ! get owner->t_cpu->cpu_thread .mutex_owner_running_critical_end: ! for pil_interrupt() hook cmp %o1, %o3 ! owner == running thread? be,a,pt %xcc, 2f ! yes, go return cpu nop 1: retl mov %g0, %o0 ! return 0 (owner not running) 2: retl mov %o2, %o0 ! owner running, return cpu SET_SIZE(mutex_owner_running) /* * rw_enter() and rw_exit(). * * These routines handle the simple cases of rw_enter (write-locking an unheld * lock or read-locking a lock that's neither write-locked nor write-wanted) * and rw_exit (no waiters or not the last reader). If anything complicated * is going on we punt to rw_enter_sleep() and rw_exit_wakeup(), respectively. */ .align 16 ENTRY(rw_enter) cmp %o1, RW_WRITER ! entering as writer? be,a,pn %icc, 2f ! if so, go do it ... or THREAD_REG, RW_WRITE_LOCKED, %o5 ! delay: %o5 = owner ldn [%o0], %o4 ! %o4 = old lock value 1: andcc %o4, RW_WRITE_CLAIMED, %g0 ! write-locked or write-wanted? bz,pt %xcc, 3f ! if so, prepare to block add %o4, RW_READ_LOCK, %o5 ! delay: increment hold count sethi %hi(rw_enter_sleep), %o2 ! load up jump jmp %o2 + %lo(rw_enter_sleep) ! jmp to rw_enter_sleep nop ! delay: do nothing 3: casx [%o0], %o4, %o5 ! try to grab read lock cmp %o4, %o5 ! did we get it? #ifdef sun4v be,a,pt %xcc, 0f membar #LoadLoad sethi %hi(rw_enter_sleep), %o2 ! load up jump jmp %o2 + %lo(rw_enter_sleep) ! jmp to rw_enter_sleep nop ! delay: do nothing 0: #else /* sun4v */ bne,pn %xcc, 1b ! if not, try again mov %o5, %o4 ! delay: %o4 = old lock value membar #LoadLoad #endif /* sun4v */ .rw_read_enter_lockstat_patch_point: retl nop 2: casx [%o0], %g0, %o5 ! try to grab write lock brz,pt %o5, 4f ! branch around if we got it membar #LoadLoad ! done regardless of where we go sethi %hi(rw_enter_sleep), %o2 jmp %o2 + %lo(rw_enter_sleep) ! jump to rw_enter_sleep if not nop ! delay: do nothing 4: .rw_write_enter_lockstat_patch_point: retl nop SET_SIZE(rw_enter) .align 16 ENTRY(rw_exit) ldn [%o0], %o4 ! %o4 = old lock value membar #LoadStore|#StoreStore ! membar_exit() subcc %o4, RW_READ_LOCK, %o5 ! %o5 = new lock value if reader bnz,pn %xcc, 2f ! single reader, no waiters? clr %o1 1: srl %o4, RW_HOLD_COUNT_SHIFT, %o3 ! %o3 = hold count (lockstat) casx [%o0], %o4, %o5 ! try to drop lock cmp %o4, %o5 ! did we succeed? bne,pn %xcc, rw_exit_wakeup ! if not, go to C nop ! delay: do nothing .rw_read_exit_lockstat_patch_point: retl nop ! delay: do nothing 2: andcc %o4, RW_WRITE_LOCKED, %g0 ! are we a writer? bnz,a,pt %xcc, 3f or THREAD_REG, RW_WRITE_LOCKED, %o4 ! delay: %o4 = owner cmp %o5, RW_READ_LOCK ! would lock still be held? bge,pt %xcc, 1b ! if so, go ahead and drop it nop ba,pt %xcc, rw_exit_wakeup ! otherwise, wake waiters nop 3: casx [%o0], %o4, %o1 ! try to drop write lock cmp %o4, %o1 ! did we succeed? bne,pn %xcc, rw_exit_wakeup ! if not, go to C nop .rw_write_exit_lockstat_patch_point: retl nop SET_SIZE(rw_exit) #define RETL 0x81c3e008 #define NOP 0x01000000 #define BA 0x10800000 #define DISP22 ((1 << 22) - 1) #define ANNUL 0x20000000 #define HOT_PATCH_COMMON(addr, event, normal_instr, annul, rs) \ ba 1f; \ rd %pc, %o0; \ save %sp, -SA(MINFRAME), %sp; \ set lockstat_probemap, %l1; \ ld [%l1 + (event * DTRACE_IDSIZE)], %o0; \ brz,pn %o0, 0f; \ ldub [THREAD_REG + T_LOCKSTAT], %l0; \ add %l0, 1, %l2; \ stub %l2, [THREAD_REG + T_LOCKSTAT]; \ set lockstat_probe, %g1; \ ld [%l1 + (event * DTRACE_IDSIZE)], %o0; \ brz,a,pn %o0, 0f; \ stub %l0, [THREAD_REG + T_LOCKSTAT]; \ ldn [%g1], %g2; \ mov rs, %o2; \ jmpl %g2, %o7; \ mov %i0, %o1; \ stub %l0, [THREAD_REG + T_LOCKSTAT]; \ 0: ret; \ restore %g0, 1, %o0; /* for mutex_tryenter / lock_try */ \ 1: set addr, %o1; \ sub %o0, %o1, %o0; \ srl %o0, 2, %o0; \ inc %o0; \ set DISP22, %o1; \ and %o1, %o0, %o0; \ set BA, %o1; \ or %o1, %o0, %o0; \ sethi %hi(annul), %o2; \ add %o0, %o2, %o2; \ set addr, %o0; \ set normal_instr, %o1; \ ld [%i0 + (event * DTRACE_IDSIZE)], %o3; \ tst %o3; \ movnz %icc, %o2, %o1; \ call hot_patch_kernel_text; \ mov 4, %o2; \ membar #Sync #define HOT_PATCH(addr, event, normal_instr) \ HOT_PATCH_COMMON(addr, event, normal_instr, 0, %i1) #define HOT_PATCH_ARG(addr, event, normal_instr, arg) \ HOT_PATCH_COMMON(addr, event, normal_instr, 0, arg) #define HOT_PATCH_ANNULLED(addr, event, normal_instr) \ HOT_PATCH_COMMON(addr, event, normal_instr, ANNUL, %i1) ENTRY(lockstat_hot_patch) save %sp, -SA(MINFRAME), %sp set lockstat_probemap, %i0 HOT_PATCH(.mutex_enter_lockstat_patch_point, LS_MUTEX_ENTER_ACQUIRE, RETL) HOT_PATCH_ANNULLED(.mutex_tryenter_lockstat_patch_point, LS_MUTEX_TRYENTER_ACQUIRE, RETL) HOT_PATCH(.mutex_exit_lockstat_patch_point, LS_MUTEX_EXIT_RELEASE, RETL) HOT_PATCH(.rw_write_enter_lockstat_patch_point, LS_RW_ENTER_ACQUIRE, RETL) HOT_PATCH(.rw_read_enter_lockstat_patch_point, LS_RW_ENTER_ACQUIRE, RETL) HOT_PATCH_ARG(.rw_write_exit_lockstat_patch_point, LS_RW_EXIT_RELEASE, RETL, RW_WRITER) HOT_PATCH_ARG(.rw_read_exit_lockstat_patch_point, LS_RW_EXIT_RELEASE, RETL, RW_READER) HOT_PATCH(.lock_set_lockstat_patch_point, LS_LOCK_SET_ACQUIRE, RETL) HOT_PATCH_ANNULLED(.lock_try_lockstat_patch_point, LS_LOCK_TRY_ACQUIRE, RETL) HOT_PATCH(.lock_clear_lockstat_patch_point, LS_LOCK_CLEAR_RELEASE, RETL) HOT_PATCH(.lock_set_spl_lockstat_patch_point, LS_LOCK_SET_SPL_ACQUIRE, RETL) HOT_PATCH(.lock_clear_splx_lockstat_patch_point, LS_LOCK_CLEAR_SPLX_RELEASE, RETL) ret restore SET_SIZE(lockstat_hot_patch) /* * asm_mutex_spin_enter(mutex_t *) * * For use by assembly interrupt handler only. * Does not change spl, since the interrupt handler is assumed to be * running at high level already. * Traps may be off, so cannot panic. * Does not keep statistics on the lock. * * Entry: %l6 - points to mutex * %l7 - address of call (returns to %l7+8) * Uses: %l6, %l5 */ .align 16 ENTRY_NP(asm_mutex_spin_enter) ldstub [%l6 + M_SPINLOCK], %l5 ! try to set lock, get value in %l5 1: tst %l5 bnz 3f ! lock already held - go spin nop 2: jmp %l7 + 8 ! return membar #LoadLoad ! ! Spin on lock without using an atomic operation to prevent the caches ! from unnecessarily moving ownership of the line around. ! 3: ldub [%l6 + M_SPINLOCK], %l5 4: tst %l5 bz,a 1b ! lock appears to be free, try again ldstub [%l6 + M_SPINLOCK], %l5 ! delay slot - try to set lock sethi %hi(panicstr) , %l5 ldn [%l5 + %lo(panicstr)], %l5 tst %l5 bnz 2b ! after panic, feign success nop b 4b ldub [%l6 + M_SPINLOCK], %l5 ! delay - reload lock SET_SIZE(asm_mutex_spin_enter) /* * asm_mutex_spin_exit(mutex_t *) * * For use by assembly interrupt handler only. * Does not change spl, since the interrupt handler is assumed to be * running at high level already. * * Entry: %l6 - points to mutex * %l7 - address of call (returns to %l7+8) * Uses: none */ ENTRY_NP(asm_mutex_spin_exit) membar #LoadStore|#StoreStore jmp %l7 + 8 ! return clrb [%l6 + M_SPINLOCK] ! delay - clear lock SET_SIZE(asm_mutex_spin_exit) /* * thread_onproc() * Set thread in onproc state for the specified CPU. * Also set the thread lock pointer to the CPU's onproc lock. * Since the new lock isn't held, the store ordering is important. * If not done in assembler, the compiler could reorder the stores. */ ENTRY(thread_onproc) set TS_ONPROC, %o2 ! TS_ONPROC state st %o2, [%o0 + T_STATE] ! store state add %o1, CPU_THREAD_LOCK, %o3 ! pointer to disp_lock while running retl ! return stn %o3, [%o0 + T_LOCKP] ! delay - store new lock pointer SET_SIZE(thread_onproc) /* delay function used in some mutex code - just do 3 nop cas ops */ ENTRY(cas_delay) casx [%o0], %g0, %g0 casx [%o0], %g0, %g0 retl casx [%o0], %g0, %g0 SET_SIZE(cas_delay) /* * alternative delay function for some niagara processors. The rd * instruction uses less resources than casx on those cpus. */ ENTRY(rdccr_delay) rd %ccr, %g0 rd %ccr, %g0 retl rd %ccr, %g0 SET_SIZE(rdccr_delay) /* * mutex_delay_default(void) * Spins for approx a few hundred processor cycles and returns to caller. */ ENTRY(mutex_delay_default) mov 72,%o0 1: brgz %o0, 1b dec %o0 retl nop SET_SIZE(mutex_delay_default)