summaryrefslogtreecommitdiff
path: root/usr/src/uts/common/os/timer.c
diff options
context:
space:
mode:
Diffstat (limited to 'usr/src/uts/common/os/timer.c')
-rw-r--r--usr/src/uts/common/os/timer.c446
1 files changed, 290 insertions, 156 deletions
diff --git a/usr/src/uts/common/os/timer.c b/usr/src/uts/common/os/timer.c
index b25a6cbcf1..5453ebf380 100644
--- a/usr/src/uts/common/os/timer.c
+++ b/usr/src/uts/common/os/timer.c
@@ -25,11 +25,12 @@
*/
/*
- * Copyright (c) 2012, Joyent, Inc. All rights reserved.
+ * Copyright 2017 Joyent, Inc.
*/
#include <sys/timer.h>
#include <sys/systm.h>
+#include <sys/sysmacros.h>
#include <sys/param.h>
#include <sys/kmem.h>
#include <sys/debug.h>
@@ -81,6 +82,7 @@ timer_lock(proc_t *p, itimer_t *it)
* waiters. p_lock must be held on entry; it will not be dropped by
* timer_unlock().
*/
+/* ARGSUSED */
static void
timer_unlock(proc_t *p, itimer_t *it)
{
@@ -123,6 +125,7 @@ timer_delete_locked(proc_t *p, timer_t tid, itimer_t *it)
timer_lock(p, it);
}
+ ASSERT(p->p_itimer_sz > tid);
ASSERT(p->p_itimer[tid] == it);
p->p_itimer[tid] = NULL;
@@ -137,7 +140,7 @@ timer_delete_locked(proc_t *p, timer_t tid, itimer_t *it)
it->it_backend->clk_timer_delete(it);
- if (it->it_portev) {
+ if (it->it_flags & IT_PORT) {
mutex_enter(&it->it_mutex);
if (it->it_portev) {
port_kevent_t *pev;
@@ -199,18 +202,20 @@ timer_delete_locked(proc_t *p, timer_t tid, itimer_t *it)
static itimer_t *
timer_grab(proc_t *p, timer_t tid)
{
- itimer_t **itp, *it;
+ itimer_t *it;
- if (tid >= timer_max || tid < 0)
+ if (tid < 0) {
return (NULL);
+ }
mutex_enter(&p->p_lock);
-
- if ((itp = p->p_itimer) == NULL || (it = itp[tid]) == NULL) {
+ if (p->p_itimer == NULL || tid >= p->p_itimer_sz ||
+ (it = p->p_itimer[tid]) == NULL) {
mutex_exit(&p->p_lock);
return (NULL);
}
+ /* This may drop p_lock temporarily. */
timer_lock(p, it);
if (it->it_lock & ITLK_REMOVE) {
@@ -232,7 +237,7 @@ timer_grab(proc_t *p, timer_t tid)
* should not be held on entry; timer_release() will acquire p_lock but
* will drop it before returning.
*/
-static void
+void
timer_release(proc_t *p, itimer_t *it)
{
mutex_enter(&p->p_lock);
@@ -245,7 +250,7 @@ timer_release(proc_t *p, itimer_t *it)
* p_lock should not be held on entry; timer_delete_grabbed() will acquire
* p_lock, but will drop it before returning.
*/
-static void
+void
timer_delete_grabbed(proc_t *p, timer_t tid, itimer_t *it)
{
mutex_enter(&p->p_lock);
@@ -258,6 +263,13 @@ clock_timer_init()
{
clock_timer_cache = kmem_cache_create("timer_cache",
sizeof (itimer_t), 0, NULL, NULL, NULL, NULL, NULL, 0);
+
+ /*
+ * Push the timer_max limit up to at least 4 * NCPU. Due to the way
+ * NCPU is defined, proper initialization of the timer limit is
+ * performed at runtime.
+ */
+ timer_max = MAX(NCPU * 4, timer_max);
}
void
@@ -453,6 +465,9 @@ timer_fire(itimer_t *it)
it->it_pending = 1;
port_send_event((port_kevent_t *)it->it_portev);
mutex_exit(&it->it_mutex);
+ } else if (it->it_flags & IT_CALLBACK) {
+ it->it_cb_func(it);
+ ASSERT(MUTEX_NOT_HELD(&it->it_mutex));
} else if (it->it_flags & IT_SIGNAL) {
it->it_pending = 1;
mutex_exit(&it->it_mutex);
@@ -466,159 +481,175 @@ timer_fire(itimer_t *it)
mutex_exit(&p->p_lock);
}
-int
-timer_create(clockid_t clock, struct sigevent *evp, timer_t *tid)
+/*
+ * Allocate an itimer_t and find and appropriate slot for it in p_itimer.
+ * Acquires p_lock and holds it on return, regardless of success.
+ */
+static itimer_t *
+timer_alloc(proc_t *p, timer_t *id)
{
- struct sigevent ev;
- proc_t *p = curproc;
- clock_backend_t *backend;
- itimer_t *it, **itp;
- sigqueue_t *sigq;
- cred_t *cr = CRED();
- int error = 0;
- timer_t i;
- port_notify_t tim_pnevp;
- port_kevent_t *pkevp = NULL;
+ itimer_t *it, **itp = NULL;
+ uint_t i;
- if ((backend = CLOCK_BACKEND(clock)) == NULL)
- return (set_errno(EINVAL));
+ ASSERT(MUTEX_NOT_HELD(&p->p_lock));
- if (evp != NULL) {
- /*
- * short copyin() for binary compatibility
- * fetch oldsigevent to determine how much to copy in.
- */
- if (get_udatamodel() == DATAMODEL_NATIVE) {
- if (copyin(evp, &ev, sizeof (struct oldsigevent)))
- return (set_errno(EFAULT));
+ it = kmem_cache_alloc(clock_timer_cache, KM_SLEEP);
+ bzero(it, sizeof (itimer_t));
+ mutex_init(&it->it_mutex, NULL, MUTEX_DEFAULT, NULL);
- if (ev.sigev_notify == SIGEV_PORT ||
- ev.sigev_notify == SIGEV_THREAD) {
- if (copyin(ev.sigev_value.sival_ptr, &tim_pnevp,
- sizeof (port_notify_t)))
- return (set_errno(EFAULT));
+ mutex_enter(&p->p_lock);
+retry:
+ if (p->p_itimer != NULL) {
+ for (i = 0; i < p->p_itimer_sz; i++) {
+ if (p->p_itimer[i] == NULL) {
+ itp = &(p->p_itimer[i]);
+ break;
}
-#ifdef _SYSCALL32_IMPL
- } else {
- struct sigevent32 ev32;
- port_notify32_t tim_pnevp32;
+ }
+ }
- if (copyin(evp, &ev32, sizeof (struct oldsigevent32)))
- return (set_errno(EFAULT));
- ev.sigev_notify = ev32.sigev_notify;
- ev.sigev_signo = ev32.sigev_signo;
+ /*
+ * A suitable slot was not found. If possible, allocate (or resize)
+ * the p_itimer array and try again.
+ */
+ if (itp == NULL) {
+ uint_t target_sz = _TIMER_ALLOC_INIT;
+ itimer_t **itp_new;
+
+ if (p->p_itimer != NULL) {
+ ASSERT(p->p_itimer_sz != 0);
+
+ target_sz = p->p_itimer_sz * 2;
+ }
+ /*
+ * Protect against exceeding the max or overflow
+ */
+ if (target_sz > timer_max || target_sz > INT_MAX ||
+ target_sz < p->p_itimer_sz) {
+ kmem_cache_free(clock_timer_cache, it);
+ return (NULL);
+ }
+ mutex_exit(&p->p_lock);
+ itp_new = kmem_zalloc(target_sz * sizeof (itimer_t *),
+ KM_SLEEP);
+ mutex_enter(&p->p_lock);
+ if (target_sz <= p->p_itimer_sz) {
/*
- * See comment in sigqueue32() on handling of 32-bit
- * sigvals in a 64-bit kernel.
+ * A racing thread performed the resize while we were
+ * waiting outside p_lock. Discard our now-useless
+ * allocation and retry.
*/
- ev.sigev_value.sival_int = ev32.sigev_value.sival_int;
- if (ev.sigev_notify == SIGEV_PORT ||
- ev.sigev_notify == SIGEV_THREAD) {
- if (copyin((void *)(uintptr_t)
- ev32.sigev_value.sival_ptr,
- (void *)&tim_pnevp32,
- sizeof (port_notify32_t)))
- return (set_errno(EFAULT));
- tim_pnevp.portnfy_port =
- tim_pnevp32.portnfy_port;
- tim_pnevp.portnfy_user =
- (void *)(uintptr_t)tim_pnevp32.portnfy_user;
+ kmem_free(itp_new, target_sz * sizeof (itimer_t *));
+ goto retry;
+ } else {
+ /*
+ * Instantiate the larger allocation and select the
+ * first fresh entry for use.
+ */
+ if (p->p_itimer != NULL) {
+ uint_t old_sz;
+
+ old_sz = p->p_itimer_sz;
+ bcopy(p->p_itimer, itp_new,
+ old_sz * sizeof (itimer_t *));
+ kmem_free(p->p_itimer,
+ old_sz * sizeof (itimer_t *));
+
+ /*
+ * Short circuit to use the first free entry in
+ * the new allocation. It's possible that
+ * other lower-indexed timers were freed while
+ * p_lock was dropped, but skipping over them
+ * is not harmful at all. In the common case,
+ * we skip the need to walk over an array
+ * filled with timers before arriving at the
+ * slot we know is fresh from the allocation.
+ */
+ i = old_sz;
+ } else {
+ /*
+ * For processes lacking any existing timers,
+ * we can simply select the first entry.
+ */
+ i = 0;
}
-#endif
+ p->p_itimer = itp_new;
+ p->p_itimer_sz = target_sz;
}
- switch (ev.sigev_notify) {
- case SIGEV_NONE:
- break;
- case SIGEV_SIGNAL:
- if (ev.sigev_signo < 1 || ev.sigev_signo >= NSIG)
- return (set_errno(EINVAL));
- break;
- case SIGEV_THREAD:
- case SIGEV_PORT:
- break;
- default:
- return (set_errno(EINVAL));
- }
- } else {
- /*
- * Use the clock's default sigevent (this is a structure copy).
- */
- ev = backend->clk_default;
}
+ ASSERT(i <= INT_MAX);
+ *id = (timer_t)i;
+ return (it);
+}
+
+/*
+ * Setup a timer
+ *
+ * This allocates an itimer_t (including a timer_t ID and slot in the process),
+ * wires it up according to the provided sigevent, and associates it with the
+ * desired clock backend. Upon successful completion, the timer will be
+ * locked, preventing it from being armed via timer_settime() or deleted via
+ * timer_delete(). This gives the caller a chance to perform any last minute
+ * manipulations (such as configuring the IT_CALLBACK functionality and/or
+ * copying the timer_t out to userspace) before using timer_release() to unlock
+ * it or timer_delete_grabbed() to delete it.
+ */
+int
+timer_setup(clock_backend_t *backend, struct sigevent *evp, port_notify_t *pnp,
+ itimer_t **itp, timer_t *tidp)
+{
+ proc_t *p = curproc;
+ int error = 0;
+ itimer_t *it;
+ sigqueue_t *sigq;
+ timer_t tid;
+
/*
- * We'll allocate our timer and sigqueue now, before we grab p_lock.
- * If we can't find an empty slot, we'll free them before returning.
+ * We'll allocate our sigqueue now, before we grab p_lock.
+ * If we can't find an empty slot, we'll free it before returning.
*/
- it = kmem_cache_alloc(clock_timer_cache, KM_SLEEP);
- bzero(it, sizeof (itimer_t));
- mutex_init(&it->it_mutex, NULL, MUTEX_DEFAULT, NULL);
sigq = kmem_zalloc(sizeof (sigqueue_t), KM_SLEEP);
- mutex_enter(&p->p_lock);
-
/*
- * If this is this process' first timer, we need to attempt to allocate
- * an array of timerstr_t pointers. We drop p_lock to perform the
- * allocation; if we return to discover that p_itimer is non-NULL,
- * we will free our allocation and drive on.
+ * Allocate a timer and choose a slot for it. This acquires p_lock.
*/
- if ((itp = p->p_itimer) == NULL) {
- mutex_exit(&p->p_lock);
- itp = kmem_zalloc(timer_max * sizeof (itimer_t *), KM_SLEEP);
- mutex_enter(&p->p_lock);
-
- if (p->p_itimer == NULL)
- p->p_itimer = itp;
- else {
- kmem_free(itp, timer_max * sizeof (itimer_t *));
- itp = p->p_itimer;
- }
- }
-
- for (i = 0; i < timer_max && itp[i] != NULL; i++)
- continue;
+ it = timer_alloc(p, &tid);
+ ASSERT(MUTEX_HELD(&p->p_lock));
- if (i == timer_max) {
- /*
- * We couldn't find a slot. Drop p_lock, free the preallocated
- * timer and sigqueue, and return an error.
- */
+ if (it == NULL) {
mutex_exit(&p->p_lock);
- kmem_cache_free(clock_timer_cache, it);
kmem_free(sigq, sizeof (sigqueue_t));
-
- return (set_errno(EAGAIN));
+ return (EAGAIN);
}
- ASSERT(i < timer_max && itp[i] == NULL);
+ ASSERT(tid < p->p_itimer_sz && p->p_itimer[tid] == NULL);
+ ASSERT(evp != NULL);
/*
* If we develop other notification mechanisms, this will need
* to call into (yet another) backend.
*/
- sigq->sq_info.si_signo = ev.sigev_signo;
- if (evp == NULL)
- sigq->sq_info.si_value.sival_int = i;
- else
- sigq->sq_info.si_value = ev.sigev_value;
+ sigq->sq_info.si_signo = evp->sigev_signo;
+ sigq->sq_info.si_value = evp->sigev_value;
sigq->sq_info.si_code = SI_TIMER;
sigq->sq_info.si_pid = p->p_pid;
sigq->sq_info.si_ctid = PRCTID(p);
sigq->sq_info.si_zoneid = getzoneid();
- sigq->sq_info.si_uid = crgetruid(cr);
+ sigq->sq_info.si_uid = crgetruid(CRED());
sigq->sq_func = timer_signal;
sigq->sq_next = NULL;
sigq->sq_backptr = it;
it->it_sigq = sigq;
it->it_backend = backend;
it->it_lock = ITLK_LOCKED;
- itp[i] = it;
-
- if (ev.sigev_notify == SIGEV_THREAD ||
- ev.sigev_notify == SIGEV_PORT) {
+ if (evp->sigev_notify == SIGEV_THREAD ||
+ evp->sigev_notify == SIGEV_PORT) {
int port;
+ port_kevent_t *pkevp = NULL;
+
+ ASSERT(pnp != NULL);
/*
* This timer is programmed to use event port notification when
@@ -638,18 +669,17 @@ timer_create(clockid_t clock, struct sigevent *evp, timer_t *tid)
*/
it->it_flags |= IT_PORT;
- port = tim_pnevp.portnfy_port;
+ port = pnp->portnfy_port;
/* associate timer as event source with the port */
error = port_associate_ksource(port, PORT_SOURCE_TIMER,
(port_source_t **)&it->it_portsrc, timer_close_port,
(void *)it, NULL);
if (error) {
- itp[i] = NULL; /* clear slot */
mutex_exit(&p->p_lock);
kmem_cache_free(clock_timer_cache, it);
kmem_free(sigq, sizeof (sigqueue_t));
- return (set_errno(error));
+ return (error);
}
/* allocate an event structure/slot */
@@ -658,23 +688,24 @@ timer_create(clockid_t clock, struct sigevent *evp, timer_t *tid)
if (error) {
(void) port_dissociate_ksource(port, PORT_SOURCE_TIMER,
(port_source_t *)it->it_portsrc);
- itp[i] = NULL; /* clear slot */
mutex_exit(&p->p_lock);
kmem_cache_free(clock_timer_cache, it);
kmem_free(sigq, sizeof (sigqueue_t));
- return (set_errno(error));
+ return (error);
}
/* initialize event data */
- port_init_event(pkevp, i, tim_pnevp.portnfy_user,
+ port_init_event(pkevp, tid, pnp->portnfy_user,
timer_port_callback, it);
it->it_portev = pkevp;
it->it_portfd = port;
} else {
- if (ev.sigev_notify == SIGEV_SIGNAL)
+ if (evp->sigev_notify == SIGEV_SIGNAL)
it->it_flags |= IT_SIGNAL;
}
+ /* Populate the slot now that the timer is prepped. */
+ p->p_itimer[tid] = it;
mutex_exit(&p->p_lock);
/*
@@ -687,17 +718,8 @@ timer_create(clockid_t clock, struct sigevent *evp, timer_t *tid)
it->it_lwp = ttolwp(curthread);
it->it_proc = p;
- if (copyout(&i, tid, sizeof (timer_t)) != 0) {
- error = EFAULT;
- goto err;
- }
-
- /*
- * If we're here, then we have successfully created the timer; we
- * just need to release the timer and return.
- */
- timer_release(p, it);
-
+ *itp = it;
+ *tidp = tid;
return (0);
err:
@@ -708,11 +730,115 @@ err:
* impossible for a removal to be pending.
*/
ASSERT(!(it->it_lock & ITLK_REMOVE));
- timer_delete_grabbed(p, i, it);
+ timer_delete_grabbed(p, tid, it);
- return (set_errno(error));
+ return (error);
}
+
+int
+timer_create(clockid_t clock, struct sigevent *evp, timer_t *tidp)
+{
+ int error = 0;
+ proc_t *p = curproc;
+ clock_backend_t *backend;
+ struct sigevent ev;
+ itimer_t *it;
+ timer_t tid;
+ port_notify_t tim_pnevp;
+
+ if ((backend = CLOCK_BACKEND(clock)) == NULL)
+ return (set_errno(EINVAL));
+
+ if (evp != NULL) {
+ /*
+ * short copyin() for binary compatibility
+ * fetch oldsigevent to determine how much to copy in.
+ */
+ if (get_udatamodel() == DATAMODEL_NATIVE) {
+ if (copyin(evp, &ev, sizeof (struct oldsigevent)))
+ return (set_errno(EFAULT));
+
+ if (ev.sigev_notify == SIGEV_PORT ||
+ ev.sigev_notify == SIGEV_THREAD) {
+ if (copyin(ev.sigev_value.sival_ptr, &tim_pnevp,
+ sizeof (port_notify_t)))
+ return (set_errno(EFAULT));
+ }
+#ifdef _SYSCALL32_IMPL
+ } else {
+ struct sigevent32 ev32;
+ port_notify32_t tim_pnevp32;
+
+ if (copyin(evp, &ev32, sizeof (struct oldsigevent32)))
+ return (set_errno(EFAULT));
+ ev.sigev_notify = ev32.sigev_notify;
+ ev.sigev_signo = ev32.sigev_signo;
+ /*
+ * See comment in sigqueue32() on handling of 32-bit
+ * sigvals in a 64-bit kernel.
+ */
+ ev.sigev_value.sival_int = ev32.sigev_value.sival_int;
+ if (ev.sigev_notify == SIGEV_PORT ||
+ ev.sigev_notify == SIGEV_THREAD) {
+ if (copyin((void *)(uintptr_t)
+ ev32.sigev_value.sival_ptr,
+ (void *)&tim_pnevp32,
+ sizeof (port_notify32_t)))
+ return (set_errno(EFAULT));
+ tim_pnevp.portnfy_port =
+ tim_pnevp32.portnfy_port;
+ tim_pnevp.portnfy_user =
+ (void *)(uintptr_t)tim_pnevp32.portnfy_user;
+ }
+#endif
+ }
+ switch (ev.sigev_notify) {
+ case SIGEV_NONE:
+ break;
+ case SIGEV_SIGNAL:
+ if (ev.sigev_signo < 1 || ev.sigev_signo >= NSIG)
+ return (set_errno(EINVAL));
+ break;
+ case SIGEV_THREAD:
+ case SIGEV_PORT:
+ break;
+ default:
+ return (set_errno(EINVAL));
+ }
+ } else {
+ /*
+ * Use the clock's default sigevent (this is a structure copy).
+ */
+ ev = backend->clk_default;
+ }
+
+ if ((error = timer_setup(backend, &ev, &tim_pnevp, &it, &tid)) != 0) {
+ return (set_errno(error));
+ }
+
+ /*
+ * Populate si_value with the timer ID if no sigevent was passed in.
+ */
+ if (evp == NULL) {
+ it->it_sigq->sq_info.si_value.sival_int = tid;
+ }
+
+ if (copyout(&tid, tidp, sizeof (timer_t)) != 0) {
+ timer_delete_grabbed(p, tid, it);
+ return (set_errno(EFAULT));
+ }
+
+ /*
+ * If we're here, then we have successfully created the timer; we
+ * just need to release the timer and return.
+ */
+ timer_release(p, it);
+
+ return (0);
+}
+
+
int
timer_gettime(timer_t tid, itimerspec_t *val)
{
@@ -832,20 +958,23 @@ timer_getoverrun(timer_t tid)
void
timer_lwpexit(void)
{
- timer_t i;
+ uint_t i;
proc_t *p = curproc;
klwp_t *lwp = ttolwp(curthread);
- itimer_t *it, **itp;
+ itimer_t *it;
ASSERT(MUTEX_HELD(&p->p_lock));
- if ((itp = p->p_itimer) == NULL)
+ if (p->p_itimer == NULL) {
return;
+ }
- for (i = 0; i < timer_max; i++) {
- if ((it = itp[i]) == NULL)
+ for (i = 0; i < p->p_itimer_sz; i++) {
+ if ((it = p->p_itimer[i]) == NULL) {
continue;
+ }
+ /* This may drop p_lock temporarily. */
timer_lock(p, it);
if ((it->it_lock & ITLK_REMOVE) || it->it_lwp != lwp) {
@@ -876,20 +1005,22 @@ timer_lwpexit(void)
void
timer_lwpbind()
{
- timer_t i;
+ uint_t i;
proc_t *p = curproc;
klwp_t *lwp = ttolwp(curthread);
- itimer_t *it, **itp;
+ itimer_t *it;
ASSERT(MUTEX_HELD(&p->p_lock));
- if ((itp = p->p_itimer) == NULL)
+ if (p->p_itimer == NULL) {
return;
+ }
- for (i = 0; i < timer_max; i++) {
- if ((it = itp[i]) == NULL)
+ for (i = 0; i < p->p_itimer_sz; i++) {
+ if ((it = p->p_itimer[i]) == NULL)
continue;
+ /* This may drop p_lock temporarily. */
timer_lock(p, it);
if (!(it->it_lock & ITLK_REMOVE) && it->it_lwp == lwp) {
@@ -911,16 +1042,19 @@ timer_lwpbind()
void
timer_exit(void)
{
- timer_t i;
+ uint_t i;
proc_t *p = curproc;
ASSERT(p->p_itimer != NULL);
+ ASSERT(p->p_itimer_sz != 0);
- for (i = 0; i < timer_max; i++)
- (void) timer_delete(i);
+ for (i = 0; i < p->p_itimer_sz; i++) {
+ (void) timer_delete((timer_t)i);
+ }
- kmem_free(p->p_itimer, timer_max * sizeof (itimer_t *));
+ kmem_free(p->p_itimer, p->p_itimer_sz * sizeof (itimer_t *));
p->p_itimer = NULL;
+ p->p_itimer_sz = 0;
}
/*
@@ -977,7 +1111,7 @@ timer_close_port(void *arg, int port, pid_t pid, int lastclose)
for (tid = 0; tid < timer_max; tid++) {
if ((it = timer_grab(p, tid)) == NULL)
continue;
- if (it->it_portev) {
+ if (it->it_flags & IT_PORT) {
mutex_enter(&it->it_mutex);
if (it->it_portfd == port) {
port_kevent_t *pev;