5340 add kstats to track reason for fork failures

Reviewed by: Richard Elling <richard.elling@richardelling.com>
Reviewed by: Garrett D'Amore <garrett@damore.org>
Reviewed by: Igor Kozhukhov <ikozhukhov@gmail.com>
Reviewed by: Josef 'Jeff' Sipek <josef.sipek@nexenta.com>
Approved by: Dan McDonald <danmcd@omniti.com>

4 files changed, 43 insertions, 4 deletions

diff --git a/usr/src/uts/common/os/fork.c b/usr/src/uts/common/os/fork.c
index 5d8c144f99..210a301850 100644
--- a/usr/src/uts/common/os/fork.c
+++ b/usr/src/uts/common/os/fork.c
@@ -21,6 +21,7 @@
 
 /*
  * Copyright (c) 1988, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright 2013, Joyent, Inc. All rights reserved.
  */
 
 /*	Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T	*/
@@ -149,6 +150,7 @@ cfork(int isvfork, int isfork1, int flags)
 	 */
 	if ((flags & ~(FORK_NOSIGCHLD | FORK_WAITPID)) != 0) {
 		error = EINVAL;
+		atomic_inc_32(&curproc->p_zone->zone_ffmisc);
 		goto forkerr;
 	}
 
@@ -157,11 +159,14 @@ cfork(int isvfork, int isfork1, int flags)
 	 */
 	if (curthread == p->p_agenttp) {
 		error = ENOTSUP;
+		atomic_inc_32(&curproc->p_zone->zone_ffmisc);
 		goto forkerr;
 	}
 
-	if ((error = secpolicy_basic_fork(CRED())) != 0)
+	if ((error = secpolicy_basic_fork(CRED())) != 0) {
+		atomic_inc_32(&p->p_zone->zone_ffmisc);
 		goto forkerr;
+	}
 
 	/*
 	 * If the calling lwp is doing a fork1() then the
@@ -175,6 +180,7 @@ cfork(int isvfork, int isfork1, int flags)
 	if (!holdlwps(isfork1 ? SHOLDFORK1 : SHOLDFORK)) {
 		aston(curthread);
 		error = EINTR;
+		atomic_inc_32(&p->p_zone->zone_ffmisc);
 		goto forkerr;
 	}
 
@@ -290,6 +296,7 @@ cfork(int isvfork, int isfork1, int flags)
 			 * map all others to EAGAIN.
 			 */
 			error = (error == ENOMEM) ? ENOMEM : EAGAIN;
+			atomic_inc_32(&p->p_zone->zone_ffnomem);
 			goto forkerr;
 		}
 
@@ -424,8 +431,10 @@ cfork(int isvfork, int isfork1, int flags)
 	 * fork event (if requested) to whatever contract the child is
 	 * a member of.  Fails if the parent has been SIGKILLed.
 	 */
-	if (contract_process_fork(NULL, cp, p, B_TRUE) == NULL)
+	if (contract_process_fork(NULL, cp, p, B_TRUE) == NULL) {
+		atomic_inc_32(&p->p_zone->zone_ffmisc);
 		goto forklwperr;
+	}
 
 	/*
 	 * No fork failures occur beyond this point.
@@ -960,6 +969,7 @@ getproc(proc_t **cpp, pid_t pid, uint_t flags)
 		if (rctlfail) {
 			mutex_exit(&zone->zone_nlwps_lock);
 			mutex_exit(&pp->p_lock);
+			atomic_inc_32(&zone->zone_ffcap);
 			goto punish;
 		}
 	}
@@ -1233,6 +1243,7 @@ bad:
 	proj->kpj_nprocs--;
 	zone->zone_nprocs--;
 	mutex_exit(&zone->zone_nlwps_lock);
+	atomic_inc_32(&zone->zone_ffnoproc);
 
 punish:
 	/*
diff --git a/usr/src/uts/common/os/lwp.c b/usr/src/uts/common/os/lwp.c
index 6d518e1aae..feb8e76c42 100644
--- a/usr/src/uts/common/os/lwp.c
+++ b/usr/src/uts/common/os/lwp.c
@@ -146,6 +146,7 @@ lwp_create(void (*proc)(), caddr_t arg, size_t len, proc_t *p,
 	if (rctlfail) {
 		mutex_exit(&p->p_zone->zone_nlwps_lock);
 		mutex_exit(&p->p_lock);
+		atomic_inc_32(&p->p_zone->zone_ffcap);
 		return (NULL);
 	}
 	p->p_task->tk_nlwps++;
@@ -204,6 +205,7 @@ lwp_create(void (*proc)(), caddr_t arg, size_t len, proc_t *p,
 			p->p_zone->zone_nlwps--;
 			mutex_exit(&p->p_zone->zone_nlwps_lock);
 			mutex_exit(&p->p_lock);
+			atomic_inc_32(&p->p_zone->zone_ffnomem);
 			return (NULL);
 		}
 	} else {
@@ -217,6 +219,7 @@ lwp_create(void (*proc)(), caddr_t arg, size_t len, proc_t *p,
 			p->p_zone->zone_nlwps--;
 			mutex_exit(&p->p_zone->zone_nlwps_lock);
 			mutex_exit(&p->p_lock);
+			atomic_inc_32(&p->p_zone->zone_ffnomem);
 			return (NULL);
 		}
 	}
@@ -582,10 +585,12 @@ grow:
 			err = CL_FORK(curthread, t, bufp);
 			t->t_cid = cid;
 		}
-		if (err)
+		if (err) {
+			atomic_inc_32(&p->p_zone->zone_ffmisc);
 			goto error;
-		else
+		} else {
 			bufp = NULL;
+		}
 	}
 
 	/*
@@ -612,6 +617,7 @@ grow:
 				 * All lwpids are allocated; fail the request.
 				 */
 				err = 1;
+				atomic_inc_32(&p->p_zone->zone_ffnoproc);
 				goto error;
 			}
 			/*
@@ -631,6 +637,7 @@ grow:
 	if (PROC_IS_BRANDED(p)) {
 		if (BROP(p)->b_initlwp(lwp)) {
 			err = 1;
+			atomic_inc_32(&p->p_zone->zone_ffmisc);
 			goto error;
 		}
 		branded = 1;
diff --git a/usr/src/uts/common/os/zone.c b/usr/src/uts/common/os/zone.c
index 444b455e5e..884d52b224 100644
--- a/usr/src/uts/common/os/zone.c
+++ b/usr/src/uts/common/os/zone.c
@@ -1845,6 +1845,11 @@ zone_misc_kstat_update(kstat_t *ksp, int rw)
 	zmp->zm_avenrun5.value.ui32 = zone->zone_avenrun[1];
 	zmp->zm_avenrun15.value.ui32 = zone->zone_avenrun[2];
 
+	zmp->zm_ffcap.value.ui32 = zone->zone_ffcap;
+	zmp->zm_ffnoproc.value.ui32 = zone->zone_ffnoproc;
+	zmp->zm_ffnomem.value.ui32 = zone->zone_ffnomem;
+	zmp->zm_ffmisc.value.ui32 = zone->zone_ffmisc;
+
 	return (0);
 }
 
@@ -1878,6 +1883,12 @@ zone_misc_kstat_create(zone_t *zone)
 	kstat_named_init(&zmp->zm_avenrun5, "avenrun_5min", KSTAT_DATA_UINT32);
 	kstat_named_init(&zmp->zm_avenrun15, "avenrun_15min",
 	    KSTAT_DATA_UINT32);
+	kstat_named_init(&zmp->zm_ffcap, "forkfail_cap", KSTAT_DATA_UINT32);
+	kstat_named_init(&zmp->zm_ffnoproc, "forkfail_noproc",
+	    KSTAT_DATA_UINT32);
+	kstat_named_init(&zmp->zm_ffnomem, "forkfail_nomem", KSTAT_DATA_UINT32);
+	kstat_named_init(&zmp->zm_ffmisc, "forkfail_misc", KSTAT_DATA_UINT32);
+
 
 	ksp->ks_update = zone_misc_kstat_update;
 	ksp->ks_private = zone;
diff --git a/usr/src/uts/common/sys/zone.h b/usr/src/uts/common/sys/zone.h
index a4566be477..cb701bc713 100644
--- a/usr/src/uts/common/sys/zone.h
+++ b/usr/src/uts/common/sys/zone.h
@@ -20,6 +20,7 @@
  */
 /*
  * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright 2013, Joyent, Inc. All rights reserved.
  * Copyright 2014 Nexenta Systems, Inc. All rights reserved.
  * Copyright 2014 Igor Kozhukhov <ikozhukhov@gmail.com>.
  */
@@ -393,6 +394,10 @@ typedef struct {
 	kstat_named_t	zm_fss_shr_pct;
 	kstat_named_t	zm_fss_pri_hi;
 	kstat_named_t	zm_fss_pri_avg;
+	kstat_named_t	zm_ffcap;
+	kstat_named_t	zm_ffnoproc;
+	kstat_named_t	zm_ffnomem;
+	kstat_named_t	zm_ffmisc;
 } zone_misc_kstat_t;
 
 typedef struct zone {
@@ -574,6 +579,11 @@ typedef struct zone {
 	uint64_t	zone_stime;		/* total system time */
 	uint64_t	zone_utime;		/* total user time */
 	uint64_t	zone_wtime;		/* total time waiting in runq */
+	/* fork-fail kstat tracking */
+	uint32_t	zone_ffcap;		/* hit an rctl cap */
+	uint32_t	zone_ffnoproc;		/* get proc/lwp error */
+	uint32_t	zone_ffnomem;		/* as_dup/memory error */
+	uint32_t	zone_ffmisc;		/* misc. other error */
 
 	struct loadavg_s zone_loadavg;		/* loadavg for this zone */
 	uint64_t	zone_hp_avenrun[3];	/* high-precision avenrun */