diff options
Diffstat (limited to 'usr/src/uts/common/os/timer.c')
| -rw-r--r-- | usr/src/uts/common/os/timer.c | 446 | 
1 files changed, 290 insertions, 156 deletions
| diff --git a/usr/src/uts/common/os/timer.c b/usr/src/uts/common/os/timer.c index b25a6cbcf1..5453ebf380 100644 --- a/usr/src/uts/common/os/timer.c +++ b/usr/src/uts/common/os/timer.c @@ -25,11 +25,12 @@   */  /* - * Copyright (c) 2012, Joyent, Inc. All rights reserved. + * Copyright 2017 Joyent, Inc.   */  #include <sys/timer.h>  #include <sys/systm.h> +#include <sys/sysmacros.h>  #include <sys/param.h>  #include <sys/kmem.h>  #include <sys/debug.h> @@ -81,6 +82,7 @@ timer_lock(proc_t *p, itimer_t *it)   * waiters.  p_lock must be held on entry; it will not be dropped by   * timer_unlock().   */ +/* ARGSUSED */  static void  timer_unlock(proc_t *p, itimer_t *it)  { @@ -123,6 +125,7 @@ timer_delete_locked(proc_t *p, timer_t tid, itimer_t *it)  		timer_lock(p, it);  	} +	ASSERT(p->p_itimer_sz > tid);  	ASSERT(p->p_itimer[tid] == it);  	p->p_itimer[tid] = NULL; @@ -137,7 +140,7 @@ timer_delete_locked(proc_t *p, timer_t tid, itimer_t *it)  	it->it_backend->clk_timer_delete(it); -	if (it->it_portev) { +	if (it->it_flags & IT_PORT) {  		mutex_enter(&it->it_mutex);  		if (it->it_portev) {  			port_kevent_t	*pev; @@ -199,18 +202,20 @@ timer_delete_locked(proc_t *p, timer_t tid, itimer_t *it)  static itimer_t *  timer_grab(proc_t *p, timer_t tid)  { -	itimer_t **itp, *it; +	itimer_t *it; -	if (tid >= timer_max || tid < 0) +	if (tid < 0) {  		return (NULL); +	}  	mutex_enter(&p->p_lock); - -	if ((itp = p->p_itimer) == NULL || (it = itp[tid]) == NULL) { +	if (p->p_itimer == NULL || tid >= p->p_itimer_sz || +	    (it = p->p_itimer[tid]) == NULL) {  		mutex_exit(&p->p_lock);  		return (NULL);  	} +	/* This may drop p_lock temporarily. */  	timer_lock(p, it);  	if (it->it_lock & ITLK_REMOVE) { @@ -232,7 +237,7 @@ timer_grab(proc_t *p, timer_t tid)   * should not be held on entry; timer_release() will acquire p_lock but   * will drop it before returning.   */ -static void +void  timer_release(proc_t *p, itimer_t *it)  {  	mutex_enter(&p->p_lock); @@ -245,7 +250,7 @@ timer_release(proc_t *p, itimer_t *it)   * p_lock should not be held on entry; timer_delete_grabbed() will acquire   * p_lock, but will drop it before returning.   */ -static void +void  timer_delete_grabbed(proc_t *p, timer_t tid, itimer_t *it)  {  	mutex_enter(&p->p_lock); @@ -258,6 +263,13 @@ clock_timer_init()  {  	clock_timer_cache = kmem_cache_create("timer_cache",  	    sizeof (itimer_t), 0, NULL, NULL, NULL, NULL, NULL, 0); + +	/* +	 * Push the timer_max limit up to at least 4 * NCPU.  Due to the way +	 * NCPU is defined, proper initialization of the timer limit is +	 * performed at runtime. +	 */ +	timer_max = MAX(NCPU * 4, timer_max);  }  void @@ -453,6 +465,9 @@ timer_fire(itimer_t *it)  			it->it_pending = 1;  			port_send_event((port_kevent_t *)it->it_portev);  			mutex_exit(&it->it_mutex); +		} else if (it->it_flags & IT_CALLBACK) { +			it->it_cb_func(it); +			ASSERT(MUTEX_NOT_HELD(&it->it_mutex));  		} else if (it->it_flags & IT_SIGNAL) {  			it->it_pending = 1;  			mutex_exit(&it->it_mutex); @@ -466,159 +481,175 @@ timer_fire(itimer_t *it)  		mutex_exit(&p->p_lock);  } -int -timer_create(clockid_t clock, struct sigevent *evp, timer_t *tid) +/* + * Allocate an itimer_t and find and appropriate slot for it in p_itimer. + * Acquires p_lock and holds it on return, regardless of success. + */ +static itimer_t * +timer_alloc(proc_t *p, timer_t *id)  { -	struct sigevent ev; -	proc_t *p = curproc; -	clock_backend_t *backend; -	itimer_t *it, **itp; -	sigqueue_t *sigq; -	cred_t *cr = CRED(); -	int error = 0; -	timer_t i; -	port_notify_t tim_pnevp; -	port_kevent_t *pkevp = NULL; +	itimer_t *it, **itp = NULL; +	uint_t i; -	if ((backend = CLOCK_BACKEND(clock)) == NULL) -		return (set_errno(EINVAL)); +	ASSERT(MUTEX_NOT_HELD(&p->p_lock)); -	if (evp != NULL) { -		/* -		 * short copyin() for binary compatibility -		 * fetch oldsigevent to determine how much to copy in. -		 */ -		if (get_udatamodel() == DATAMODEL_NATIVE) { -			if (copyin(evp, &ev, sizeof (struct oldsigevent))) -				return (set_errno(EFAULT)); +	it = kmem_cache_alloc(clock_timer_cache, KM_SLEEP); +	bzero(it, sizeof (itimer_t)); +	mutex_init(&it->it_mutex, NULL, MUTEX_DEFAULT, NULL); -			if (ev.sigev_notify == SIGEV_PORT || -			    ev.sigev_notify == SIGEV_THREAD) { -				if (copyin(ev.sigev_value.sival_ptr, &tim_pnevp, -				    sizeof (port_notify_t))) -					return (set_errno(EFAULT)); +	mutex_enter(&p->p_lock); +retry: +	if (p->p_itimer != NULL) { +		for (i = 0; i < p->p_itimer_sz; i++) { +			if (p->p_itimer[i] == NULL) { +				itp = &(p->p_itimer[i]); +				break;  			} -#ifdef	_SYSCALL32_IMPL -		} else { -			struct sigevent32 ev32; -			port_notify32_t tim_pnevp32; +		} +	} -			if (copyin(evp, &ev32, sizeof (struct oldsigevent32))) -				return (set_errno(EFAULT)); -			ev.sigev_notify = ev32.sigev_notify; -			ev.sigev_signo = ev32.sigev_signo; +	/* +	 * A suitable slot was not found.  If possible, allocate (or resize) +	 * the p_itimer array and try again. +	 */ +	if (itp == NULL) { +		uint_t target_sz = _TIMER_ALLOC_INIT; +		itimer_t **itp_new; + +		if (p->p_itimer != NULL) { +			ASSERT(p->p_itimer_sz != 0); + +			target_sz = p->p_itimer_sz * 2; +		} +		/* +		 * Protect against exceeding the max or overflow +		 */ +		if (target_sz > timer_max || target_sz > INT_MAX || +		    target_sz < p->p_itimer_sz) { +			kmem_cache_free(clock_timer_cache, it); +			return (NULL); +		} +		mutex_exit(&p->p_lock); +		itp_new = kmem_zalloc(target_sz * sizeof (itimer_t *), +		    KM_SLEEP); +		mutex_enter(&p->p_lock); +		if (target_sz <= p->p_itimer_sz) {  			/* -			 * See comment in sigqueue32() on handling of 32-bit -			 * sigvals in a 64-bit kernel. +			 * A racing thread performed the resize while we were +			 * waiting outside p_lock.  Discard our now-useless +			 * allocation and retry.  			 */ -			ev.sigev_value.sival_int = ev32.sigev_value.sival_int; -			if (ev.sigev_notify == SIGEV_PORT || -			    ev.sigev_notify == SIGEV_THREAD) { -				if (copyin((void *)(uintptr_t) -				    ev32.sigev_value.sival_ptr, -				    (void *)&tim_pnevp32, -				    sizeof (port_notify32_t))) -					return (set_errno(EFAULT)); -				tim_pnevp.portnfy_port = -				    tim_pnevp32.portnfy_port; -				tim_pnevp.portnfy_user = -				    (void *)(uintptr_t)tim_pnevp32.portnfy_user; +			kmem_free(itp_new, target_sz * sizeof (itimer_t *)); +			goto retry; +		} else { +			/* +			 * Instantiate the larger allocation and select the +			 * first fresh entry for use. +			 */ +			if (p->p_itimer != NULL) { +				uint_t old_sz; + +				old_sz = p->p_itimer_sz; +				bcopy(p->p_itimer, itp_new, +				    old_sz * sizeof (itimer_t *)); +				kmem_free(p->p_itimer, +				    old_sz * sizeof (itimer_t *)); + +				/* +				 * Short circuit to use the first free entry in +				 * the new allocation.  It's possible that +				 * other lower-indexed timers were freed while +				 * p_lock was dropped, but skipping over them +				 * is not harmful at all.  In the common case, +				 * we skip the need to walk over an array +				 * filled with timers before arriving at the +				 * slot we know is fresh from the allocation. +				 */ +				i = old_sz; +			} else { +				/* +				 * For processes lacking any existing timers, +				 * we can simply select the first entry. +				 */ +				i = 0;  			} -#endif +			p->p_itimer = itp_new; +			p->p_itimer_sz = target_sz;  		} -		switch (ev.sigev_notify) { -		case SIGEV_NONE: -			break; -		case SIGEV_SIGNAL: -			if (ev.sigev_signo < 1 || ev.sigev_signo >= NSIG) -				return (set_errno(EINVAL)); -			break; -		case SIGEV_THREAD: -		case SIGEV_PORT: -			break; -		default: -			return (set_errno(EINVAL)); -		} -	} else { -		/* -		 * Use the clock's default sigevent (this is a structure copy). -		 */ -		ev = backend->clk_default;  	} +	ASSERT(i <= INT_MAX); +	*id = (timer_t)i; +	return (it); +} + +/* + * Setup a timer + * + * This allocates an itimer_t (including a timer_t ID and slot in the process), + * wires it up according to the provided sigevent, and associates it with the + * desired clock backend.  Upon successful completion, the timer will be + * locked, preventing it from being armed via timer_settime() or deleted via + * timer_delete().  This gives the caller a chance to perform any last minute + * manipulations (such as configuring the IT_CALLBACK functionality and/or + * copying the timer_t out to userspace) before using timer_release() to unlock + * it or timer_delete_grabbed() to delete it. + */ +int +timer_setup(clock_backend_t *backend, struct sigevent *evp, port_notify_t *pnp, +    itimer_t **itp, timer_t *tidp) +{ +	proc_t *p = curproc; +	int error = 0; +	itimer_t *it; +	sigqueue_t *sigq; +	timer_t tid; +  	/* -	 * We'll allocate our timer and sigqueue now, before we grab p_lock. -	 * If we can't find an empty slot, we'll free them before returning. +	 * We'll allocate our sigqueue now, before we grab p_lock. +	 * If we can't find an empty slot, we'll free it before returning.  	 */ -	it = kmem_cache_alloc(clock_timer_cache, KM_SLEEP); -	bzero(it, sizeof (itimer_t)); -	mutex_init(&it->it_mutex, NULL, MUTEX_DEFAULT, NULL);  	sigq = kmem_zalloc(sizeof (sigqueue_t), KM_SLEEP); -	mutex_enter(&p->p_lock); -  	/* -	 * If this is this process' first timer, we need to attempt to allocate -	 * an array of timerstr_t pointers.  We drop p_lock to perform the -	 * allocation; if we return to discover that p_itimer is non-NULL, -	 * we will free our allocation and drive on. +	 * Allocate a timer and choose a slot for it. This acquires p_lock.  	 */ -	if ((itp = p->p_itimer) == NULL) { -		mutex_exit(&p->p_lock); -		itp = kmem_zalloc(timer_max * sizeof (itimer_t *), KM_SLEEP); -		mutex_enter(&p->p_lock); - -		if (p->p_itimer == NULL) -			p->p_itimer = itp; -		else { -			kmem_free(itp, timer_max * sizeof (itimer_t *)); -			itp = p->p_itimer; -		} -	} - -	for (i = 0; i < timer_max && itp[i] != NULL; i++) -		continue; +	it = timer_alloc(p, &tid); +	ASSERT(MUTEX_HELD(&p->p_lock)); -	if (i == timer_max) { -		/* -		 * We couldn't find a slot.  Drop p_lock, free the preallocated -		 * timer and sigqueue, and return an error. -		 */ +	if (it == NULL) {  		mutex_exit(&p->p_lock); -		kmem_cache_free(clock_timer_cache, it);  		kmem_free(sigq, sizeof (sigqueue_t)); - -		return (set_errno(EAGAIN)); +		return (EAGAIN);  	} -	ASSERT(i < timer_max && itp[i] == NULL); +	ASSERT(tid < p->p_itimer_sz && p->p_itimer[tid] == NULL); +	ASSERT(evp != NULL);  	/*  	 * If we develop other notification mechanisms, this will need  	 * to call into (yet another) backend.  	 */ -	sigq->sq_info.si_signo = ev.sigev_signo; -	if (evp == NULL) -		sigq->sq_info.si_value.sival_int = i; -	else -		sigq->sq_info.si_value = ev.sigev_value; +	sigq->sq_info.si_signo = evp->sigev_signo; +	sigq->sq_info.si_value = evp->sigev_value;  	sigq->sq_info.si_code = SI_TIMER;  	sigq->sq_info.si_pid = p->p_pid;  	sigq->sq_info.si_ctid = PRCTID(p);  	sigq->sq_info.si_zoneid = getzoneid(); -	sigq->sq_info.si_uid = crgetruid(cr); +	sigq->sq_info.si_uid = crgetruid(CRED());  	sigq->sq_func = timer_signal;  	sigq->sq_next = NULL;  	sigq->sq_backptr = it;  	it->it_sigq = sigq;  	it->it_backend = backend;  	it->it_lock = ITLK_LOCKED; -	itp[i] = it; - -	if (ev.sigev_notify == SIGEV_THREAD || -	    ev.sigev_notify == SIGEV_PORT) { +	if (evp->sigev_notify == SIGEV_THREAD || +	    evp->sigev_notify == SIGEV_PORT) {  		int port; +		port_kevent_t *pkevp = NULL; + +		ASSERT(pnp != NULL);  		/*  		 * This timer is programmed to use event port notification when @@ -638,18 +669,17 @@ timer_create(clockid_t clock, struct sigevent *evp, timer_t *tid)  		 */  		it->it_flags |= IT_PORT; -		port = tim_pnevp.portnfy_port; +		port = pnp->portnfy_port;  		/* associate timer as event source with the port */  		error = port_associate_ksource(port, PORT_SOURCE_TIMER,  		    (port_source_t **)&it->it_portsrc, timer_close_port,  		    (void *)it, NULL);  		if (error) { -			itp[i] = NULL;		/* clear slot */  			mutex_exit(&p->p_lock);  			kmem_cache_free(clock_timer_cache, it);  			kmem_free(sigq, sizeof (sigqueue_t)); -			return (set_errno(error)); +			return (error);  		}  		/* allocate an event structure/slot */ @@ -658,23 +688,24 @@ timer_create(clockid_t clock, struct sigevent *evp, timer_t *tid)  		if (error) {  			(void) port_dissociate_ksource(port, PORT_SOURCE_TIMER,  			    (port_source_t *)it->it_portsrc); -			itp[i] = NULL;		/* clear slot */  			mutex_exit(&p->p_lock);  			kmem_cache_free(clock_timer_cache, it);  			kmem_free(sigq, sizeof (sigqueue_t)); -			return (set_errno(error)); +			return (error);  		}  		/* initialize event data */ -		port_init_event(pkevp, i, tim_pnevp.portnfy_user, +		port_init_event(pkevp, tid, pnp->portnfy_user,  		    timer_port_callback, it);  		it->it_portev = pkevp;  		it->it_portfd = port;  	} else { -		if (ev.sigev_notify == SIGEV_SIGNAL) +		if (evp->sigev_notify == SIGEV_SIGNAL)  			it->it_flags |= IT_SIGNAL;  	} +	/* Populate the slot now that the timer is prepped. */ +	p->p_itimer[tid] = it;  	mutex_exit(&p->p_lock);  	/* @@ -687,17 +718,8 @@ timer_create(clockid_t clock, struct sigevent *evp, timer_t *tid)  	it->it_lwp = ttolwp(curthread);  	it->it_proc = p; -	if (copyout(&i, tid, sizeof (timer_t)) != 0) { -		error = EFAULT; -		goto err; -	} - -	/* -	 * If we're here, then we have successfully created the timer; we -	 * just need to release the timer and return. -	 */ -	timer_release(p, it); - +	*itp = it; +	*tidp = tid;  	return (0);  err: @@ -708,11 +730,115 @@ err:  	 * impossible for a removal to be pending.  	 */  	ASSERT(!(it->it_lock & ITLK_REMOVE)); -	timer_delete_grabbed(p, i, it); +	timer_delete_grabbed(p, tid, it); -	return (set_errno(error)); +	return (error);  } + +int +timer_create(clockid_t clock, struct sigevent *evp, timer_t *tidp) +{ +	int error = 0; +	proc_t *p = curproc; +	clock_backend_t *backend; +	struct sigevent ev; +	itimer_t *it; +	timer_t tid; +	port_notify_t tim_pnevp; + +	if ((backend = CLOCK_BACKEND(clock)) == NULL) +		return (set_errno(EINVAL)); + +	if (evp != NULL) { +		/* +		 * short copyin() for binary compatibility +		 * fetch oldsigevent to determine how much to copy in. +		 */ +		if (get_udatamodel() == DATAMODEL_NATIVE) { +			if (copyin(evp, &ev, sizeof (struct oldsigevent))) +				return (set_errno(EFAULT)); + +			if (ev.sigev_notify == SIGEV_PORT || +			    ev.sigev_notify == SIGEV_THREAD) { +				if (copyin(ev.sigev_value.sival_ptr, &tim_pnevp, +				    sizeof (port_notify_t))) +					return (set_errno(EFAULT)); +			} +#ifdef	_SYSCALL32_IMPL +		} else { +			struct sigevent32 ev32; +			port_notify32_t tim_pnevp32; + +			if (copyin(evp, &ev32, sizeof (struct oldsigevent32))) +				return (set_errno(EFAULT)); +			ev.sigev_notify = ev32.sigev_notify; +			ev.sigev_signo = ev32.sigev_signo; +			/* +			 * See comment in sigqueue32() on handling of 32-bit +			 * sigvals in a 64-bit kernel. +			 */ +			ev.sigev_value.sival_int = ev32.sigev_value.sival_int; +			if (ev.sigev_notify == SIGEV_PORT || +			    ev.sigev_notify == SIGEV_THREAD) { +				if (copyin((void *)(uintptr_t) +				    ev32.sigev_value.sival_ptr, +				    (void *)&tim_pnevp32, +				    sizeof (port_notify32_t))) +					return (set_errno(EFAULT)); +				tim_pnevp.portnfy_port = +				    tim_pnevp32.portnfy_port; +				tim_pnevp.portnfy_user = +				    (void *)(uintptr_t)tim_pnevp32.portnfy_user; +			} +#endif +		} +		switch (ev.sigev_notify) { +		case SIGEV_NONE: +			break; +		case SIGEV_SIGNAL: +			if (ev.sigev_signo < 1 || ev.sigev_signo >= NSIG) +				return (set_errno(EINVAL)); +			break; +		case SIGEV_THREAD: +		case SIGEV_PORT: +			break; +		default: +			return (set_errno(EINVAL)); +		} +	} else { +		/* +		 * Use the clock's default sigevent (this is a structure copy). +		 */ +		ev = backend->clk_default; +	} + +	if ((error = timer_setup(backend, &ev, &tim_pnevp, &it, &tid)) != 0) { +		return (set_errno(error)); +	} + +	/* +	 * Populate si_value with the timer ID if no sigevent was passed in. +	 */ +	if (evp == NULL) { +		it->it_sigq->sq_info.si_value.sival_int = tid; +	} + +	if (copyout(&tid, tidp, sizeof (timer_t)) != 0) { +		timer_delete_grabbed(p, tid, it); +		return (set_errno(EFAULT)); +	} + +	/* +	 * If we're here, then we have successfully created the timer; we +	 * just need to release the timer and return. +	 */ +	timer_release(p, it); + +	return (0); +} + +  int  timer_gettime(timer_t tid, itimerspec_t *val)  { @@ -832,20 +958,23 @@ timer_getoverrun(timer_t tid)  void  timer_lwpexit(void)  { -	timer_t i; +	uint_t i;  	proc_t *p = curproc;  	klwp_t *lwp = ttolwp(curthread); -	itimer_t *it, **itp; +	itimer_t *it;  	ASSERT(MUTEX_HELD(&p->p_lock)); -	if ((itp = p->p_itimer) == NULL) +	if (p->p_itimer == NULL) {  		return; +	} -	for (i = 0; i < timer_max; i++) { -		if ((it = itp[i]) == NULL) +	for (i = 0; i < p->p_itimer_sz; i++) { +		if ((it = p->p_itimer[i]) == NULL) {  			continue; +		} +		/* This may drop p_lock temporarily. */  		timer_lock(p, it);  		if ((it->it_lock & ITLK_REMOVE) || it->it_lwp != lwp) { @@ -876,20 +1005,22 @@ timer_lwpexit(void)  void  timer_lwpbind()  { -	timer_t i; +	uint_t i;  	proc_t *p = curproc;  	klwp_t *lwp = ttolwp(curthread); -	itimer_t *it, **itp; +	itimer_t *it;  	ASSERT(MUTEX_HELD(&p->p_lock)); -	if ((itp = p->p_itimer) == NULL) +	if (p->p_itimer == NULL) {  		return; +	} -	for (i = 0; i < timer_max; i++) { -		if ((it = itp[i]) == NULL) +	for (i = 0; i < p->p_itimer_sz; i++) { +		if ((it = p->p_itimer[i]) == NULL)  			continue; +		/* This may drop p_lock temporarily. */  		timer_lock(p, it);  		if (!(it->it_lock & ITLK_REMOVE) && it->it_lwp == lwp) { @@ -911,16 +1042,19 @@ timer_lwpbind()  void  timer_exit(void)  { -	timer_t i; +	uint_t i;  	proc_t *p = curproc;  	ASSERT(p->p_itimer != NULL); +	ASSERT(p->p_itimer_sz != 0); -	for (i = 0; i < timer_max; i++) -		(void) timer_delete(i); +	for (i = 0; i < p->p_itimer_sz; i++) { +		(void) timer_delete((timer_t)i); +	} -	kmem_free(p->p_itimer, timer_max * sizeof (itimer_t *)); +	kmem_free(p->p_itimer, p->p_itimer_sz * sizeof (itimer_t *));  	p->p_itimer = NULL; +	p->p_itimer_sz = 0;  }  /* @@ -977,7 +1111,7 @@ timer_close_port(void *arg, int port, pid_t pid, int lastclose)  	for (tid = 0; tid < timer_max; tid++) {  		if ((it = timer_grab(p, tid)) == NULL)  			continue; -		if (it->it_portev) { +		if (it->it_flags & IT_PORT) {  			mutex_enter(&it->it_mutex);  			if (it->it_portfd == port) {  				port_kevent_t *pev; | 
