6708797 mutex lock failed: Lock is not recoverable

6720135 go_filebench core dumping in snv_89 on sparc
6765549 go_filebench is core dumping on snv_101

6 files changed, 90 insertions, 107 deletions

diff --git a/usr/src/cmd/filebench/common/fb_localfs.c b/usr/src/cmd/filebench/common/fb_localfs.c
index ff62bb1114..df5ebad7cd 100644
--- a/usr/src/cmd/filebench/common/fb_localfs.c
+++ b/usr/src/cmd/filebench/common/fb_localfs.c
@@ -411,7 +411,7 @@ fb_lfsflow_aiowait(threadflow_t *threadflow, flowop_t *flowop)
 
 		flowop_beginop(threadflow, flowop);
 
-#ifdef HAVE_AIOWAITN
+#if (defined(HAVE_AIOWAITN) && defined(USE_PROCESS_MODEL))
 		if (((aio_waitn64((struct aiocb64 **)worklist,
 		    MAXREAP, &todo, &timeout)) == -1) &&
 		    errno && (errno != ETIME)) {
@@ -447,14 +447,14 @@ fb_lfsflow_aiowait(threadflow_t *threadflow, flowop_t *flowop)
 		for (ncompleted = 0, inprogress = 0,
 		    aio = flowop->fo_thread->tf_aiolist;
 		    ncompleted < todo, aio != NULL; aio = aio->al_next) {
-			int result = aio_error64(aio->al_aiocb);
+			int result = aio_error64(&aio->al_aiocb);
 
 			if (result == EINPROGRESS) {
 				inprogress++;
 				continue;
 			}
 
-			if ((aio_return64(aio->al_aiocb) == -1) || result) {
+			if ((aio_return64(&aio->al_aiocb) == -1) || result) {
 				filebench_log(LOG_ERROR, "aio failed: %s",
 				    strerror(result));
 				continue;
@@ -464,7 +464,7 @@ fb_lfsflow_aiowait(threadflow_t *threadflow, flowop_t *flowop)
 
 			if (aio_deallocate(flowop, &aio->al_aiocb) < 0) {
 				filebench_log(LOG_ERROR, "Could not remove "
-				    aio from list ");
+				    "aio from list ");
 				flowop_endop(threadflow, flowop, 0);
 				return (FILEBENCH_ERROR);
 			}
diff --git a/usr/src/cmd/filebench/common/filebench.h b/usr/src/cmd/filebench/common/filebench.h
index 8a6b785679..bcbe843723 100644
--- a/usr/src/cmd/filebench/common/filebench.h
+++ b/usr/src/cmd/filebench/common/filebench.h
@@ -118,12 +118,12 @@ void filebench_plugin_funcvecinit(void);
 #define	MIN(x, y) ((x) < (y) ? (x) : (y))
 #endif
 
-#define	FILEBENCH_VERSION	"1.4.3"
+#define	FILEBENCH_VERSION	"1.4.4"
 #define	FILEBENCHDIR	"/usr/benchmarks/filebench"
 #define	FILEBENCH_PROMPT	"filebench> "
 #define	MAX_LINE_LEN	1024
 #define	MAX_CMD_HIST	128
-#define	SHUTDOWN_WAIT_SECONDS	5 /* time to wait for proc / thrd to quit */
+#define	SHUTDOWN_WAIT_SECONDS	3 /* time to wait for proc / thrd to quit */
 
 #define	FILEBENCH_DONE	 1
 #define	FILEBENCH_OK	 0
diff --git a/usr/src/cmd/filebench/common/flowop.c b/usr/src/cmd/filebench/common/flowop.c
index a6b3617e2b..6bbe470fa3 100644
--- a/usr/src/cmd/filebench/common/flowop.c
+++ b/usr/src/cmd/filebench/common/flowop.c
@@ -718,6 +718,7 @@ void
 flowop_delete_all(flowop_t **flowoplist)
 {
 	flowop_t *flowop = *flowoplist;
+	flowop_t *next_flowop;
 
 	(void) ipc_mutex_lock(&filebench_shm->shm_flowop_lock);
 
@@ -730,8 +731,9 @@ flowop_delete_all(flowop_t **flowoplist)
 			flowop = flowop->fo_exec_next;
 			continue;
 		}
+		next_flowop = flowop->fo_exec_next;
 		flowop_delete(flowoplist, flowop);
-		flowop = flowop->fo_exec_next;
+		flowop = next_flowop;
 	}
 
 	(void) ipc_mutex_unlock(&filebench_shm->shm_flowop_lock);
diff --git a/usr/src/cmd/filebench/common/procflow.c b/usr/src/cmd/filebench/common/procflow.c
index 9d124912cc..d69cb0bcb3 100644
--- a/usr/src/cmd/filebench/common/procflow.c
+++ b/usr/src/cmd/filebench/common/procflow.c
@@ -41,6 +41,7 @@ procflow_t *my_procflow = NULL;
 
 static procflow_t *procflow_define_common(procflow_t **list, char *name,
     procflow_t *inherit, int instance);
+static void procflow_sleep(procflow_t *procflow, int wait_cnt);
 
 #ifdef USE_PROCESS_MODEL
 
@@ -403,6 +404,26 @@ procflow_createnwait(void *nothing)
 	/* NOTREACHED */
 	return (NULL);
 }
+
+/*
+ * Cancel all threads within a processes, as well as the process itself.
+ * Called by ^c or by sig_kill
+ */
+/* ARGSUSED */
+static void
+procflow_cancel(int arg1)
+{
+	filebench_log(LOG_DEBUG_IMPL, "Process signal handler on pid %",
+	    my_procflow->pf_pid);
+
+	procflow_sleep(my_procflow, SHUTDOWN_WAIT_SECONDS);
+
+	threadflow_delete_all(&my_procflow->pf_threads);
+
+	/* quit the main procflow thread and hence the process */
+	exit(0);
+}
+
 #endif	/* USE_PROCESS_MODEL */
 
 /*
@@ -431,6 +452,8 @@ procflow_init(void)
 
 	(void) ipc_mutex_lock(&filebench_shm->shm_procflow_lock);
 
+	(void) signal(SIGUSR1, procflow_cancel);
+
 	if ((ret = pthread_cond_wait(&filebench_shm->shm_procflow_procs_cv,
 	    &filebench_shm->shm_procflow_lock)) != 0)
 		return (ret);
@@ -468,57 +491,38 @@ procflow_wait(pid_t pid)
 #endif
 
 /*
- * Deletes the designated procflow and all its threadflows except
- * for FLOW_MASTER ones. Waits 10 seconds if the procflow is still
- * running, then kills the associated process. Finally it frees the
+ * Common routine to sleep for wait_cnt seconds or for pf_running to
+ * go false. Checks once a second to see if pf_running has gone false.
+ */
+static void
+procflow_sleep(procflow_t *procflow, int wait_cnt)
+{
+	while (procflow->pf_running & wait_cnt) {
+		sleep(1);
+		wait_cnt--;
+	}
+}
+
+/*
+ * Deletes the designated procflow. Finally it frees the
  * procflow entity. filebench_shm->shm_procflow_lock must be held on entry.
  *
  * If the designated procflow is not found on the list it returns -1 and
  * the procflow is not deleted. Otherwise it returns 0.
  */
 static int
-procflow_delete(procflow_t *procflow, int wait_cnt)
+procflow_cleanup(procflow_t *procflow)
 {
 	procflow_t *entry;
 
-	threadflow_delete_all(&procflow->pf_threads, wait_cnt);
-
 	filebench_log(LOG_DEBUG_SCRIPT,
 	    "Deleted proc: (%s-%d) pid %d",
 	    procflow->pf_name,
 	    procflow->pf_instance,
 	    procflow->pf_pid);
 
-	while (procflow->pf_running == 1) {
-		filebench_log(LOG_DEBUG_SCRIPT,
-		    "Waiting for process %s-%d %d",
-		    procflow->pf_name,
-		    procflow->pf_instance,
-		    procflow->pf_pid);
-
-		if (wait_cnt) {
-			(void) ipc_mutex_unlock(
-			    &filebench_shm->shm_procflow_lock);
-			(void) sleep(1);
-			(void) ipc_mutex_lock(
-			    &filebench_shm->shm_procflow_lock);
-			wait_cnt--;
-			continue;
-		}
-#ifdef USE_PROCESS_MODEL
-		(void) kill(procflow->pf_pid, SIGKILL);
-		filebench_log(LOG_DEBUG_SCRIPT,
-		    "procflow_delete: Had to kill process %s-%d %d!",
-		    procflow->pf_name,
-		    procflow->pf_instance,
-		    procflow->pf_pid);
-		procflow->pf_running = 0;
-#endif
-	}
+	procflow->pf_running = 0;
 
-#ifdef USE_PROCESS_MODEL
-	procflow_wait(procflow->pf_pid);
-#endif
 	/* remove entry from proclist */
 	entry = filebench_shm->shm_proclist;
 
@@ -628,8 +632,8 @@ procflow_allstarted()
 void
 procflow_shutdown(void)
 {
-	procflow_t *procflow;
-	int wait_cnt;
+	procflow_t *procflow, *next_procflow;
+	int wait_cnt = SHUTDOWN_WAIT_SECONDS;
 
 	(void) ipc_mutex_lock(&filebench_shm->shm_procs_running_lock);
 	if (filebench_shm->shm_procs_running <= 0) {
@@ -642,7 +646,6 @@ procflow_shutdown(void)
 	(void) ipc_mutex_lock(&filebench_shm->shm_procflow_lock);
 	procflow = filebench_shm->shm_proclist;
 	filebench_shm->shm_f_abort = 1;
-	wait_cnt = SHUTDOWN_WAIT_SECONDS;
 
 	while (procflow) {
 		if (procflow->pf_instance &&
@@ -654,10 +657,33 @@ procflow_shutdown(void)
 		    procflow->pf_name,
 		    procflow->pf_instance,
 		    procflow->pf_pid);
-		(void) procflow_delete(procflow, wait_cnt);
-		procflow = procflow->pf_next;
-		/* grow more impatient */
-		if (wait_cnt)
+
+		next_procflow = procflow->pf_next;
+
+		/*
+		 * Signalling the process with SIGUSR1 will result in it
+		 * gracefully shutting down and exiting
+		 */
+		procflow_sleep(procflow, wait_cnt);
+		if (procflow->pf_running) {
+#ifdef USE_PROCESS_MODEL
+			pid_t pid;
+
+			pid = procflow->pf_pid;
+#ifdef HAVE_SIGSEND
+			(void) sigsend(P_PID, pid, SIGUSR1);
+#else
+			(void) kill(pid, SIGUSR1);
+#endif
+			procflow_wait(pid);
+
+#else /* USE_PROCESS_MODEL */
+			threadflow_delete_all(&procflow->pf_threads);
+#endif /* USE_PROCESS_MODEL */
+		}
+		(void) procflow_cleanup(procflow);
+		procflow = next_procflow;
+		if (wait_cnt > 0)
 			wait_cnt--;
 	}
 
@@ -707,8 +733,6 @@ procflow_define_common(procflow_t **list, char *name,
 
 	filebench_log(LOG_DEBUG_IMPL, "defining process %s-%d", name, instance);
 
-	filebench_log(LOG_DEBUG_IMPL, "process %s-%d proclist %zx",
-	    name, instance, filebench_shm->shm_proclist);
 	/* Add procflow to list, lock is being held already */
 	if (*list == NULL) {
 		*list = procflow;
diff --git a/usr/src/cmd/filebench/common/threadflow.c b/usr/src/cmd/filebench/common/threadflow.c
index f723cc23af..70e28dd218 100644
--- a/usr/src/cmd/filebench/common/threadflow.c
+++ b/usr/src/cmd/filebench/common/threadflow.c
@@ -104,40 +104,6 @@ threadflow_createthread(threadflow_t *threadflow)
 }
 
 /*
- * Terminates (exits) all the threads of the procflow (process).
- * The procflow is determined from a process private pointer
- * initialized by threadflow_init().
- */
-/* ARGSUSED */
-static void
-threadflow_cancel(int arg1)
-{
-	threadflow_t *threadflow;
-
-#ifdef HAVE_LWPS
-	filebench_log(LOG_DEBUG_IMPL, "Thread signal handler on tid %d",
-	    _lwp_self());
-#endif
-
-	threadflow = my_procflow->pf_threads;
-	my_procflow->pf_running = 0;
-
-	while (threadflow) {
-		if (threadflow->tf_tid) {
-			/* make sure thread has been cleaned up */
-			flowop_destruct_all_flows(threadflow);
-
-			(void) pthread_cancel(threadflow->tf_tid);
-			filebench_log(LOG_DEBUG_IMPL, "Thread %d cancelled...",
-			    threadflow->tf_tid);
-		}
-		threadflow = threadflow->tf_next;
-	}
-
-	exit(0);
-}
-
-/*
  * Creates threads for the threadflows associated with a procflow.
  * The routine iterates through the list of threadflows in the
  * supplied procflow's pf_threads list. For each threadflow on
@@ -161,8 +127,6 @@ threadflow_init(procflow_t *procflow)
 
 	(void) ipc_mutex_lock(&filebench_shm->shm_threadflow_lock);
 
-	(void) signal(SIGUSR1, threadflow_cancel);
-
 	while (threadflow) {
 		threadflow_t *newthread;
 		int instances;
@@ -221,8 +185,10 @@ threadflow_init(procflow_t *procflow)
  * its associated process to end the thread.
  */
 static void
-threadflow_kill(threadflow_t *threadflow, int wait_cnt)
+threadflow_kill(threadflow_t *threadflow)
 {
+	int wait_cnt = 2;
+
 	/* Tell thread to finish */
 	threadflow->tf_abort = 1;
 
@@ -232,15 +198,10 @@ threadflow_kill(threadflow_t *threadflow, int wait_cnt)
 		wait_cnt--;
 	}
 
-#ifdef USE_PROCESS_MODEL
-#ifdef HAVE_SIGSEND
-	(void) sigsend(P_PID, threadflow->tf_process->pf_pid, SIGUSR1);
-#else
-	(void) kill(threadflow->tf_process->pf_pid, SIGUSR1);
-#endif
-#else /* USE_PROCESS_MODEL */
-	threadflow->tf_process->pf_running = 0;
-#endif /* USE_PROCESS_MODEL */
+	if (threadflow->tf_running) {
+		threadflow->tf_running = FALSE;
+		pthread_kill(threadflow->tf_tid, SIGKILL);
+	}
 }
 
 /*
@@ -253,8 +214,7 @@ threadflow_kill(threadflow_t *threadflow, int wait_cnt)
  * returns -1.
  */
 static int
-threadflow_delete(threadflow_t **threadlist, threadflow_t *threadflow,
-    int wait_cnt)
+threadflow_delete(threadflow_t **threadlist, threadflow_t *threadflow)
 {
 	threadflow_t *entry = *threadlist;
 
@@ -271,7 +231,7 @@ threadflow_delete(threadflow_t **threadlist, threadflow_t *threadflow,
 		    threadflow->tf_name,
 		    threadflow->tf_instance);
 
-		threadflow_kill(threadflow, wait_cnt);
+		threadflow_kill(threadflow);
 		flowop_delete_all(&threadflow->tf_thrd_fops);
 		*threadlist = threadflow->tf_next;
 		(void) pthread_mutex_destroy(&threadflow->tf_lock);
@@ -293,7 +253,7 @@ threadflow_delete(threadflow_t **threadlist, threadflow_t *threadflow,
 			    "Deleted thread: (%s-%d)",
 			    entry->tf_next->tf_name,
 			    entry->tf_next->tf_instance);
-			threadflow_kill(entry->tf_next, wait_cnt);
+			threadflow_kill(entry->tf_next);
 			flowop_delete_all(&entry->tf_next->tf_thrd_fops);
 			(void) pthread_mutex_destroy(&threadflow->tf_lock);
 			ipc_free(FILEBENCH_THREADFLOW, (char *)threadflow);
@@ -312,7 +272,7 @@ threadflow_delete(threadflow_t **threadlist, threadflow_t *threadflow,
  * except the FLOW_MASTER.
  */
 void
-threadflow_delete_all(threadflow_t **threadlist, int wait_cnt)
+threadflow_delete_all(threadflow_t **threadlist)
 {
 	threadflow_t *threadflow;
 
@@ -327,11 +287,8 @@ threadflow_delete_all(threadflow_t **threadlist, int wait_cnt)
 			threadflow = threadflow->tf_next;
 			continue;
 		}
-		(void) threadflow_delete(threadlist, threadflow, wait_cnt);
+		(void) threadflow_delete(threadlist, threadflow);
 		threadflow = threadflow->tf_next;
-		/* grow more impatient */
-		if (wait_cnt > 0)
-			wait_cnt--;
 	}
 
 	(void) ipc_mutex_unlock(&filebench_shm->shm_threadflow_lock);
diff --git a/usr/src/cmd/filebench/common/threadflow.h b/usr/src/cmd/filebench/common/threadflow.h
index ee08afbc2f..cb36a6bdcb 100644
--- a/usr/src/cmd/filebench/common/threadflow.h
+++ b/usr/src/cmd/filebench/common/threadflow.h
@@ -111,7 +111,7 @@ int threadflow_init(procflow_t *);
 void flowop_start(threadflow_t *threadflow);
 void threadflow_usage(void);
 void threadflow_allstarted(pid_t pid, threadflow_t *threadflow);
-void threadflow_delete_all(threadflow_t **threadlist, int wait_cnt);
+void threadflow_delete_all(threadflow_t **threadlist);
 
 #ifdef	__cplusplus
 }