diff options
Diffstat (limited to 'usr/src/uts/common/os/timer.c')
-rw-r--r-- | usr/src/uts/common/os/timer.c | 446 |
1 files changed, 290 insertions, 156 deletions
diff --git a/usr/src/uts/common/os/timer.c b/usr/src/uts/common/os/timer.c index b25a6cbcf1..5453ebf380 100644 --- a/usr/src/uts/common/os/timer.c +++ b/usr/src/uts/common/os/timer.c @@ -25,11 +25,12 @@ */ /* - * Copyright (c) 2012, Joyent, Inc. All rights reserved. + * Copyright 2017 Joyent, Inc. */ #include <sys/timer.h> #include <sys/systm.h> +#include <sys/sysmacros.h> #include <sys/param.h> #include <sys/kmem.h> #include <sys/debug.h> @@ -81,6 +82,7 @@ timer_lock(proc_t *p, itimer_t *it) * waiters. p_lock must be held on entry; it will not be dropped by * timer_unlock(). */ +/* ARGSUSED */ static void timer_unlock(proc_t *p, itimer_t *it) { @@ -123,6 +125,7 @@ timer_delete_locked(proc_t *p, timer_t tid, itimer_t *it) timer_lock(p, it); } + ASSERT(p->p_itimer_sz > tid); ASSERT(p->p_itimer[tid] == it); p->p_itimer[tid] = NULL; @@ -137,7 +140,7 @@ timer_delete_locked(proc_t *p, timer_t tid, itimer_t *it) it->it_backend->clk_timer_delete(it); - if (it->it_portev) { + if (it->it_flags & IT_PORT) { mutex_enter(&it->it_mutex); if (it->it_portev) { port_kevent_t *pev; @@ -199,18 +202,20 @@ timer_delete_locked(proc_t *p, timer_t tid, itimer_t *it) static itimer_t * timer_grab(proc_t *p, timer_t tid) { - itimer_t **itp, *it; + itimer_t *it; - if (tid >= timer_max || tid < 0) + if (tid < 0) { return (NULL); + } mutex_enter(&p->p_lock); - - if ((itp = p->p_itimer) == NULL || (it = itp[tid]) == NULL) { + if (p->p_itimer == NULL || tid >= p->p_itimer_sz || + (it = p->p_itimer[tid]) == NULL) { mutex_exit(&p->p_lock); return (NULL); } + /* This may drop p_lock temporarily. */ timer_lock(p, it); if (it->it_lock & ITLK_REMOVE) { @@ -232,7 +237,7 @@ timer_grab(proc_t *p, timer_t tid) * should not be held on entry; timer_release() will acquire p_lock but * will drop it before returning. */ -static void +void timer_release(proc_t *p, itimer_t *it) { mutex_enter(&p->p_lock); @@ -245,7 +250,7 @@ timer_release(proc_t *p, itimer_t *it) * p_lock should not be held on entry; timer_delete_grabbed() will acquire * p_lock, but will drop it before returning. */ -static void +void timer_delete_grabbed(proc_t *p, timer_t tid, itimer_t *it) { mutex_enter(&p->p_lock); @@ -258,6 +263,13 @@ clock_timer_init() { clock_timer_cache = kmem_cache_create("timer_cache", sizeof (itimer_t), 0, NULL, NULL, NULL, NULL, NULL, 0); + + /* + * Push the timer_max limit up to at least 4 * NCPU. Due to the way + * NCPU is defined, proper initialization of the timer limit is + * performed at runtime. + */ + timer_max = MAX(NCPU * 4, timer_max); } void @@ -453,6 +465,9 @@ timer_fire(itimer_t *it) it->it_pending = 1; port_send_event((port_kevent_t *)it->it_portev); mutex_exit(&it->it_mutex); + } else if (it->it_flags & IT_CALLBACK) { + it->it_cb_func(it); + ASSERT(MUTEX_NOT_HELD(&it->it_mutex)); } else if (it->it_flags & IT_SIGNAL) { it->it_pending = 1; mutex_exit(&it->it_mutex); @@ -466,159 +481,175 @@ timer_fire(itimer_t *it) mutex_exit(&p->p_lock); } -int -timer_create(clockid_t clock, struct sigevent *evp, timer_t *tid) +/* + * Allocate an itimer_t and find and appropriate slot for it in p_itimer. + * Acquires p_lock and holds it on return, regardless of success. + */ +static itimer_t * +timer_alloc(proc_t *p, timer_t *id) { - struct sigevent ev; - proc_t *p = curproc; - clock_backend_t *backend; - itimer_t *it, **itp; - sigqueue_t *sigq; - cred_t *cr = CRED(); - int error = 0; - timer_t i; - port_notify_t tim_pnevp; - port_kevent_t *pkevp = NULL; + itimer_t *it, **itp = NULL; + uint_t i; - if ((backend = CLOCK_BACKEND(clock)) == NULL) - return (set_errno(EINVAL)); + ASSERT(MUTEX_NOT_HELD(&p->p_lock)); - if (evp != NULL) { - /* - * short copyin() for binary compatibility - * fetch oldsigevent to determine how much to copy in. - */ - if (get_udatamodel() == DATAMODEL_NATIVE) { - if (copyin(evp, &ev, sizeof (struct oldsigevent))) - return (set_errno(EFAULT)); + it = kmem_cache_alloc(clock_timer_cache, KM_SLEEP); + bzero(it, sizeof (itimer_t)); + mutex_init(&it->it_mutex, NULL, MUTEX_DEFAULT, NULL); - if (ev.sigev_notify == SIGEV_PORT || - ev.sigev_notify == SIGEV_THREAD) { - if (copyin(ev.sigev_value.sival_ptr, &tim_pnevp, - sizeof (port_notify_t))) - return (set_errno(EFAULT)); + mutex_enter(&p->p_lock); +retry: + if (p->p_itimer != NULL) { + for (i = 0; i < p->p_itimer_sz; i++) { + if (p->p_itimer[i] == NULL) { + itp = &(p->p_itimer[i]); + break; } -#ifdef _SYSCALL32_IMPL - } else { - struct sigevent32 ev32; - port_notify32_t tim_pnevp32; + } + } - if (copyin(evp, &ev32, sizeof (struct oldsigevent32))) - return (set_errno(EFAULT)); - ev.sigev_notify = ev32.sigev_notify; - ev.sigev_signo = ev32.sigev_signo; + /* + * A suitable slot was not found. If possible, allocate (or resize) + * the p_itimer array and try again. + */ + if (itp == NULL) { + uint_t target_sz = _TIMER_ALLOC_INIT; + itimer_t **itp_new; + + if (p->p_itimer != NULL) { + ASSERT(p->p_itimer_sz != 0); + + target_sz = p->p_itimer_sz * 2; + } + /* + * Protect against exceeding the max or overflow + */ + if (target_sz > timer_max || target_sz > INT_MAX || + target_sz < p->p_itimer_sz) { + kmem_cache_free(clock_timer_cache, it); + return (NULL); + } + mutex_exit(&p->p_lock); + itp_new = kmem_zalloc(target_sz * sizeof (itimer_t *), + KM_SLEEP); + mutex_enter(&p->p_lock); + if (target_sz <= p->p_itimer_sz) { /* - * See comment in sigqueue32() on handling of 32-bit - * sigvals in a 64-bit kernel. + * A racing thread performed the resize while we were + * waiting outside p_lock. Discard our now-useless + * allocation and retry. */ - ev.sigev_value.sival_int = ev32.sigev_value.sival_int; - if (ev.sigev_notify == SIGEV_PORT || - ev.sigev_notify == SIGEV_THREAD) { - if (copyin((void *)(uintptr_t) - ev32.sigev_value.sival_ptr, - (void *)&tim_pnevp32, - sizeof (port_notify32_t))) - return (set_errno(EFAULT)); - tim_pnevp.portnfy_port = - tim_pnevp32.portnfy_port; - tim_pnevp.portnfy_user = - (void *)(uintptr_t)tim_pnevp32.portnfy_user; + kmem_free(itp_new, target_sz * sizeof (itimer_t *)); + goto retry; + } else { + /* + * Instantiate the larger allocation and select the + * first fresh entry for use. + */ + if (p->p_itimer != NULL) { + uint_t old_sz; + + old_sz = p->p_itimer_sz; + bcopy(p->p_itimer, itp_new, + old_sz * sizeof (itimer_t *)); + kmem_free(p->p_itimer, + old_sz * sizeof (itimer_t *)); + + /* + * Short circuit to use the first free entry in + * the new allocation. It's possible that + * other lower-indexed timers were freed while + * p_lock was dropped, but skipping over them + * is not harmful at all. In the common case, + * we skip the need to walk over an array + * filled with timers before arriving at the + * slot we know is fresh from the allocation. + */ + i = old_sz; + } else { + /* + * For processes lacking any existing timers, + * we can simply select the first entry. + */ + i = 0; } -#endif + p->p_itimer = itp_new; + p->p_itimer_sz = target_sz; } - switch (ev.sigev_notify) { - case SIGEV_NONE: - break; - case SIGEV_SIGNAL: - if (ev.sigev_signo < 1 || ev.sigev_signo >= NSIG) - return (set_errno(EINVAL)); - break; - case SIGEV_THREAD: - case SIGEV_PORT: - break; - default: - return (set_errno(EINVAL)); - } - } else { - /* - * Use the clock's default sigevent (this is a structure copy). - */ - ev = backend->clk_default; } + ASSERT(i <= INT_MAX); + *id = (timer_t)i; + return (it); +} + +/* + * Setup a timer + * + * This allocates an itimer_t (including a timer_t ID and slot in the process), + * wires it up according to the provided sigevent, and associates it with the + * desired clock backend. Upon successful completion, the timer will be + * locked, preventing it from being armed via timer_settime() or deleted via + * timer_delete(). This gives the caller a chance to perform any last minute + * manipulations (such as configuring the IT_CALLBACK functionality and/or + * copying the timer_t out to userspace) before using timer_release() to unlock + * it or timer_delete_grabbed() to delete it. + */ +int +timer_setup(clock_backend_t *backend, struct sigevent *evp, port_notify_t *pnp, + itimer_t **itp, timer_t *tidp) +{ + proc_t *p = curproc; + int error = 0; + itimer_t *it; + sigqueue_t *sigq; + timer_t tid; + /* - * We'll allocate our timer and sigqueue now, before we grab p_lock. - * If we can't find an empty slot, we'll free them before returning. + * We'll allocate our sigqueue now, before we grab p_lock. + * If we can't find an empty slot, we'll free it before returning. */ - it = kmem_cache_alloc(clock_timer_cache, KM_SLEEP); - bzero(it, sizeof (itimer_t)); - mutex_init(&it->it_mutex, NULL, MUTEX_DEFAULT, NULL); sigq = kmem_zalloc(sizeof (sigqueue_t), KM_SLEEP); - mutex_enter(&p->p_lock); - /* - * If this is this process' first timer, we need to attempt to allocate - * an array of timerstr_t pointers. We drop p_lock to perform the - * allocation; if we return to discover that p_itimer is non-NULL, - * we will free our allocation and drive on. + * Allocate a timer and choose a slot for it. This acquires p_lock. */ - if ((itp = p->p_itimer) == NULL) { - mutex_exit(&p->p_lock); - itp = kmem_zalloc(timer_max * sizeof (itimer_t *), KM_SLEEP); - mutex_enter(&p->p_lock); - - if (p->p_itimer == NULL) - p->p_itimer = itp; - else { - kmem_free(itp, timer_max * sizeof (itimer_t *)); - itp = p->p_itimer; - } - } - - for (i = 0; i < timer_max && itp[i] != NULL; i++) - continue; + it = timer_alloc(p, &tid); + ASSERT(MUTEX_HELD(&p->p_lock)); - if (i == timer_max) { - /* - * We couldn't find a slot. Drop p_lock, free the preallocated - * timer and sigqueue, and return an error. - */ + if (it == NULL) { mutex_exit(&p->p_lock); - kmem_cache_free(clock_timer_cache, it); kmem_free(sigq, sizeof (sigqueue_t)); - - return (set_errno(EAGAIN)); + return (EAGAIN); } - ASSERT(i < timer_max && itp[i] == NULL); + ASSERT(tid < p->p_itimer_sz && p->p_itimer[tid] == NULL); + ASSERT(evp != NULL); /* * If we develop other notification mechanisms, this will need * to call into (yet another) backend. */ - sigq->sq_info.si_signo = ev.sigev_signo; - if (evp == NULL) - sigq->sq_info.si_value.sival_int = i; - else - sigq->sq_info.si_value = ev.sigev_value; + sigq->sq_info.si_signo = evp->sigev_signo; + sigq->sq_info.si_value = evp->sigev_value; sigq->sq_info.si_code = SI_TIMER; sigq->sq_info.si_pid = p->p_pid; sigq->sq_info.si_ctid = PRCTID(p); sigq->sq_info.si_zoneid = getzoneid(); - sigq->sq_info.si_uid = crgetruid(cr); + sigq->sq_info.si_uid = crgetruid(CRED()); sigq->sq_func = timer_signal; sigq->sq_next = NULL; sigq->sq_backptr = it; it->it_sigq = sigq; it->it_backend = backend; it->it_lock = ITLK_LOCKED; - itp[i] = it; - - if (ev.sigev_notify == SIGEV_THREAD || - ev.sigev_notify == SIGEV_PORT) { + if (evp->sigev_notify == SIGEV_THREAD || + evp->sigev_notify == SIGEV_PORT) { int port; + port_kevent_t *pkevp = NULL; + + ASSERT(pnp != NULL); /* * This timer is programmed to use event port notification when @@ -638,18 +669,17 @@ timer_create(clockid_t clock, struct sigevent *evp, timer_t *tid) */ it->it_flags |= IT_PORT; - port = tim_pnevp.portnfy_port; + port = pnp->portnfy_port; /* associate timer as event source with the port */ error = port_associate_ksource(port, PORT_SOURCE_TIMER, (port_source_t **)&it->it_portsrc, timer_close_port, (void *)it, NULL); if (error) { - itp[i] = NULL; /* clear slot */ mutex_exit(&p->p_lock); kmem_cache_free(clock_timer_cache, it); kmem_free(sigq, sizeof (sigqueue_t)); - return (set_errno(error)); + return (error); } /* allocate an event structure/slot */ @@ -658,23 +688,24 @@ timer_create(clockid_t clock, struct sigevent *evp, timer_t *tid) if (error) { (void) port_dissociate_ksource(port, PORT_SOURCE_TIMER, (port_source_t *)it->it_portsrc); - itp[i] = NULL; /* clear slot */ mutex_exit(&p->p_lock); kmem_cache_free(clock_timer_cache, it); kmem_free(sigq, sizeof (sigqueue_t)); - return (set_errno(error)); + return (error); } /* initialize event data */ - port_init_event(pkevp, i, tim_pnevp.portnfy_user, + port_init_event(pkevp, tid, pnp->portnfy_user, timer_port_callback, it); it->it_portev = pkevp; it->it_portfd = port; } else { - if (ev.sigev_notify == SIGEV_SIGNAL) + if (evp->sigev_notify == SIGEV_SIGNAL) it->it_flags |= IT_SIGNAL; } + /* Populate the slot now that the timer is prepped. */ + p->p_itimer[tid] = it; mutex_exit(&p->p_lock); /* @@ -687,17 +718,8 @@ timer_create(clockid_t clock, struct sigevent *evp, timer_t *tid) it->it_lwp = ttolwp(curthread); it->it_proc = p; - if (copyout(&i, tid, sizeof (timer_t)) != 0) { - error = EFAULT; - goto err; - } - - /* - * If we're here, then we have successfully created the timer; we - * just need to release the timer and return. - */ - timer_release(p, it); - + *itp = it; + *tidp = tid; return (0); err: @@ -708,11 +730,115 @@ err: * impossible for a removal to be pending. */ ASSERT(!(it->it_lock & ITLK_REMOVE)); - timer_delete_grabbed(p, i, it); + timer_delete_grabbed(p, tid, it); - return (set_errno(error)); + return (error); } + +int +timer_create(clockid_t clock, struct sigevent *evp, timer_t *tidp) +{ + int error = 0; + proc_t *p = curproc; + clock_backend_t *backend; + struct sigevent ev; + itimer_t *it; + timer_t tid; + port_notify_t tim_pnevp; + + if ((backend = CLOCK_BACKEND(clock)) == NULL) + return (set_errno(EINVAL)); + + if (evp != NULL) { + /* + * short copyin() for binary compatibility + * fetch oldsigevent to determine how much to copy in. + */ + if (get_udatamodel() == DATAMODEL_NATIVE) { + if (copyin(evp, &ev, sizeof (struct oldsigevent))) + return (set_errno(EFAULT)); + + if (ev.sigev_notify == SIGEV_PORT || + ev.sigev_notify == SIGEV_THREAD) { + if (copyin(ev.sigev_value.sival_ptr, &tim_pnevp, + sizeof (port_notify_t))) + return (set_errno(EFAULT)); + } +#ifdef _SYSCALL32_IMPL + } else { + struct sigevent32 ev32; + port_notify32_t tim_pnevp32; + + if (copyin(evp, &ev32, sizeof (struct oldsigevent32))) + return (set_errno(EFAULT)); + ev.sigev_notify = ev32.sigev_notify; + ev.sigev_signo = ev32.sigev_signo; + /* + * See comment in sigqueue32() on handling of 32-bit + * sigvals in a 64-bit kernel. + */ + ev.sigev_value.sival_int = ev32.sigev_value.sival_int; + if (ev.sigev_notify == SIGEV_PORT || + ev.sigev_notify == SIGEV_THREAD) { + if (copyin((void *)(uintptr_t) + ev32.sigev_value.sival_ptr, + (void *)&tim_pnevp32, + sizeof (port_notify32_t))) + return (set_errno(EFAULT)); + tim_pnevp.portnfy_port = + tim_pnevp32.portnfy_port; + tim_pnevp.portnfy_user = + (void *)(uintptr_t)tim_pnevp32.portnfy_user; + } +#endif + } + switch (ev.sigev_notify) { + case SIGEV_NONE: + break; + case SIGEV_SIGNAL: + if (ev.sigev_signo < 1 || ev.sigev_signo >= NSIG) + return (set_errno(EINVAL)); + break; + case SIGEV_THREAD: + case SIGEV_PORT: + break; + default: + return (set_errno(EINVAL)); + } + } else { + /* + * Use the clock's default sigevent (this is a structure copy). + */ + ev = backend->clk_default; + } + + if ((error = timer_setup(backend, &ev, &tim_pnevp, &it, &tid)) != 0) { + return (set_errno(error)); + } + + /* + * Populate si_value with the timer ID if no sigevent was passed in. + */ + if (evp == NULL) { + it->it_sigq->sq_info.si_value.sival_int = tid; + } + + if (copyout(&tid, tidp, sizeof (timer_t)) != 0) { + timer_delete_grabbed(p, tid, it); + return (set_errno(EFAULT)); + } + + /* + * If we're here, then we have successfully created the timer; we + * just need to release the timer and return. + */ + timer_release(p, it); + + return (0); +} + + int timer_gettime(timer_t tid, itimerspec_t *val) { @@ -832,20 +958,23 @@ timer_getoverrun(timer_t tid) void timer_lwpexit(void) { - timer_t i; + uint_t i; proc_t *p = curproc; klwp_t *lwp = ttolwp(curthread); - itimer_t *it, **itp; + itimer_t *it; ASSERT(MUTEX_HELD(&p->p_lock)); - if ((itp = p->p_itimer) == NULL) + if (p->p_itimer == NULL) { return; + } - for (i = 0; i < timer_max; i++) { - if ((it = itp[i]) == NULL) + for (i = 0; i < p->p_itimer_sz; i++) { + if ((it = p->p_itimer[i]) == NULL) { continue; + } + /* This may drop p_lock temporarily. */ timer_lock(p, it); if ((it->it_lock & ITLK_REMOVE) || it->it_lwp != lwp) { @@ -876,20 +1005,22 @@ timer_lwpexit(void) void timer_lwpbind() { - timer_t i; + uint_t i; proc_t *p = curproc; klwp_t *lwp = ttolwp(curthread); - itimer_t *it, **itp; + itimer_t *it; ASSERT(MUTEX_HELD(&p->p_lock)); - if ((itp = p->p_itimer) == NULL) + if (p->p_itimer == NULL) { return; + } - for (i = 0; i < timer_max; i++) { - if ((it = itp[i]) == NULL) + for (i = 0; i < p->p_itimer_sz; i++) { + if ((it = p->p_itimer[i]) == NULL) continue; + /* This may drop p_lock temporarily. */ timer_lock(p, it); if (!(it->it_lock & ITLK_REMOVE) && it->it_lwp == lwp) { @@ -911,16 +1042,19 @@ timer_lwpbind() void timer_exit(void) { - timer_t i; + uint_t i; proc_t *p = curproc; ASSERT(p->p_itimer != NULL); + ASSERT(p->p_itimer_sz != 0); - for (i = 0; i < timer_max; i++) - (void) timer_delete(i); + for (i = 0; i < p->p_itimer_sz; i++) { + (void) timer_delete((timer_t)i); + } - kmem_free(p->p_itimer, timer_max * sizeof (itimer_t *)); + kmem_free(p->p_itimer, p->p_itimer_sz * sizeof (itimer_t *)); p->p_itimer = NULL; + p->p_itimer_sz = 0; } /* @@ -977,7 +1111,7 @@ timer_close_port(void *arg, int port, pid_t pid, int lastclose) for (tid = 0; tid < timer_max; tid++) { if ((it = timer_grab(p, tid)) == NULL) continue; - if (it->it_portev) { + if (it->it_flags & IT_PORT) { mutex_enter(&it->it_mutex); if (it->it_portfd == port) { port_kevent_t *pev; |