Exemple #1
0
void
bu_kill_parallel(void)
{
    if (pid_of_initiating_thread == 0)
	return;

    if (pid_of_initiating_thread == bu_process_id())
	return;

    bu_terminate(pid_of_initiating_thread);

    return;
}
Exemple #2
0
/*
  Catch - invoked on interrupt
*/
static void
Catch(register int sig)
    /* signal number */
{
    register int pid;		/* this process's ID */
    register int *psig;		/* -> sigs[.] */
    register int i;

    for (i = 0; sigs[i]; ++i)
	(void)signal(sigs[i], SIG_IGN);

    (void)Foo( -13 );		/* clean up */

    (void)signal( sig, SIG_DFL );

    pid = bu_process_id();

    if ( pid > 1 ) {
#ifdef HAVE_KILL
	/* (re)signal process */
	(void)kill( pid, sig );
#endif
    }
}
int
bu_crashreport(const char *filename)
{
    if (UNLIKELY(!filename || strlen(filename) == 0)) {
	return 0;
    }

    /* vat time ist? */
    (void)time(&now);

    path = bu_argv0_full_path();

    /* do our own expansion to avoid heap allocation */
    snprintf(buffer, CR_BUFSIZE, "******************************************\n\n"
	     "%s\n"		/* version info */
	     "Command: %s\n"	/* program name */
	     "Process: %d\n"	/* pid */
	     "Path: %s\n"	/* which binary */
	     "Date: %s\n",	/* date/time */
	     brlcad_ident("Crash Report"),
	     bu_getprogname(),
	     bu_process_id(),
	     path ? path : "Unknown",
	     ctime(&now));

    fp = fopen(filename, "ab");
    if (UNLIKELY(!fp || ferror(fp))) {
	perror("unable to open crash report file");
	bu_log("ERROR: Unable to open crash report file [%s]\n", filename);
	return 0;
    }

    /* make the file stream unbuffered */
    if (setvbuf(fp, (char *)NULL, _IONBF, 0) != 0) {
	perror("unable to make stream unbuffered");
    }

    /* print the report header */
    if (fwrite(buffer, 1, strlen(buffer), fp) != strlen(buffer)) {
	/* cannot bomb */
	bu_log("ERROR: Unable to write to crash report file [%s]\n", filename);
	(void)fclose(fp);
	fp = NULL;
	return 0;
    }

    /* write out the backtrace */
    fprintf(fp, "Call stack backtrace (thread %d):\n", bu_parallel_id());
    fflush(fp);
    if (bu_backtrace(fp) == 0) {
	bu_log("WARNING: Unable to obtain a call stack backtrace\n");
    }

    /* write out operating system information */
    path = bu_which("uname");
    if (path) {
	snprintf(buffer, CR_BUFSIZE, "%s -a 2>&1", path);
#if defined(HAVE_POPEN) && !defined(STRICT_FLAGS)
	popenfp = popen(buffer, "r");
	if (!popenfp) {
	    perror("unable to popen uname");
	    bu_log("WARNING: Unable to obtain uname information\n");
	}
#endif
	if (popenfp) {
	    fprintf(fp, "\nSystem characteristics:\n");
	    fflush(fp);
	    while (bu_fgets(buffer, CR_BUFSIZE, popenfp)) {
		size_t ret;
		size_t len;

		len = strlen(buffer);
		ret = fwrite(buffer, 1, len, fp);
		if (ret != len)
		    perror("fwrite failed");
	    }
	}
#if defined(HAVE_POPEN) && !defined(STRICT_FLAGS)
	(void)pclose(popenfp);
#endif
	popenfp = NULL;
	path = NULL;
    }

    /* write out kernel and hardware information */
    path = bu_which("sysctl");
    if (path) {
	/* need 2>&1 to capture stderr junk from sysctl on Mac OS X for kern.exec */
	snprintf(buffer, CR_BUFSIZE, "%s -a 2>&1", path);
#if defined(HAVE_POPEN) && !defined(STRICT_FLAGS)
	popenfp = popen(buffer, "r");
	if (popenfp == (FILE *)NULL) {
	    perror("unable to popen sysctl");
	    bu_log("WARNING: Unable to obtain sysctl information\n");
	}
#endif
	if (popenfp != (FILE *)NULL) {
	    fprintf(fp, "\nSystem information:\n");
	    fflush(fp);
	    while (bu_fgets(buffer, CR_BUFSIZE, popenfp)) {
		size_t ret;
		size_t len;

		len = strlen(buffer);
		if ((len == 0)
		    || ((len == 1) && (buffer[0] == '\n')))
		{
		    continue;
		}

		ret = fwrite(buffer, 1, len, fp);
		if (ret != len)
		    perror("fwrite failed");
	    }
	}
#if defined(HAVE_POPEN) && !defined(STRICT_FLAGS)
	(void)pclose(popenfp);
#endif
	popenfp = NULL;
	path = NULL;
    }

    memset(buffer, 0, CR_BUFSIZE);
    fprintf(fp, "\n");
    fflush(fp);
    (void)fclose(fp);
    fp = NULL;

    /* success */
    return 1;
}
Exemple #4
0
int
main(int argc, char **argv)
{
    int	n;

    if (argc < 2)  {
	fprintf(stderr, "%s", srv_usage);
	return 1;
    }
    while (argv[1][0] == '-')  {
	if (BU_STR_EQUAL(argv[1], "-d"))  {
	    debug++;
	} else if (BU_STR_EQUAL(argv[1], "-x"))  {
	    sscanf(argv[2], "%x", (unsigned int *)&RTG.debug);
	    argc--; argv++;
	} else if (BU_STR_EQUAL(argv[1], "-X"))  {
	    sscanf(argv[2], "%x", (unsigned int *)&rdebug);
	    argc--; argv++;
	} else {
	    fprintf(stderr, "%s", srv_usage);
	    return 3;
	}
	argc--; argv++;
    }
    if (argc != 3 && argc != 4)  {
	fprintf(stderr, "%s", srv_usage);
	return 2;
    }

    control_host = argv[1];
    tcp_port = argv[2];

    /* Note that the LIBPKG error logger can not be
     * "bu_log", as that can cause bu_log to be entered recursively.
     * Given the special version of bu_log in use here,
     * that will result in a deadlock in bu_semaphore_acquire(res_syscall)!
     *  libpkg will default to stderr via pkg_errlog(), which is fine.
     */
    pcsrv = pkg_open(control_host, tcp_port, "tcp", "", "",
		      pkgswitch, NULL);
    if (pcsrv == PKC_ERROR)  {
	fprintf(stderr, "rtsrv: unable to contact %s, port %s\n",
		control_host, tcp_port);
	return 1;
    }

    if (argc == 4)  {
	/* Slip one command to dispatcher */
	(void)pkg_send(MSG_CMD, argv[3], strlen(argv[3])+1, pcsrv);

	/* Prevent chasing the package with an immediate TCP close */
	sleep(1);

	pkg_close(pcsrv);
	return 0;
    }

#ifdef SO_SNDBUF
    /* increase the default send buffer size to 32k since we're
     * sending pixels more than likely.
     */
    {
	int val = 32767;
	n = setsockopt(pcsrv->pkc_fd, SOL_SOCKET, SO_SNDBUF, (const void *)&val, sizeof(val));
	if (n < 0)  perror("setsockopt: SO_SNDBUF");
    }
#endif

    if (!debug)  {
	/* A fresh process */
	if (fork())
	    return 0;

	/* Go into our own process group */
	n = bu_process_id();

#ifdef HAVE_SETPGID
	if (setpgid(n, n) < 0)
	    perror("setpgid");
#else
	/* SysV uses setpgrp with no args and it can't fail,
	 * obsoleted by setpgid.
	 */
	setpgrp();
#endif

	/*
	 *  Unless controller process has specifically said
	 *  that this is an interactive session, e.g., for a demo,
	 *  drop to the lowest sensible priority.
	 */
	if (!interactive)  {
	    bu_nice_set(19);		/* lowest priority */
	}

	/* Close off the world */
	fclose(stdin);
	fclose(stdout);
	fclose(stderr);

	(void)close(0);
	(void)close(1);
	(void)close(2);

	/* For stdio & perror safety, reopen 0, 1, 2 */
	(void)open("/dev/null", 0);	/* to fd 0 */
	n = dup(0);			/* to fd 1 */
	if (n == -1)
	    perror("dup");
	n = dup(0);			/* to fd 2 */
	if (n == -1)
	    perror("dup");

#if defined(HAVE_SYS_IOCTL_H) && defined(TIOCNOTTY)
	n = open("/dev/tty", 2);
	if (n >= 0) {
	    (void)ioctl(n, TIOCNOTTY, 0);
	    (void)close(n);
	}
#endif
    }

    /* Send our version string */
    if (pkg_send(MSG_VERSION,
		   PROTOCOL_VERSION, strlen(PROTOCOL_VERSION)+1, pcsrv) < 0)  {
	fprintf(stderr, "pkg_send MSG_VERSION error\n");
	return 1;
    }
    if (debug)  fprintf(stderr, "PROTOCOL_VERSION='%s'\n", PROTOCOL_VERSION);

    /*
     *  Now that the fork() has been done, it is safe to initialize
     *  the parallel processing support.
     */

    avail_cpus = bu_avail_cpus();

    /* Need to set rtg_parallel non_zero here for RES_INIT to work */
    npsw = avail_cpus;
    if (npsw > 1)  {
	RTG.rtg_parallel = 1;
    } else
	RTG.rtg_parallel = 0;
    bu_semaphore_init(RT_SEM_LAST);

    bu_log("using %d of %d cpus\n",
	   npsw, avail_cpus);

    /*
     *  Initialize the non-parallel memory resource.
     *  The parallel guys are initialized after the rt_dirbuild().
     */
    rt_init_resource(&rt_uniresource, MAX_PSW, NULL);
    bn_rand_init(rt_uniresource.re_randptr, MAX_PSW);

    BU_LIST_INIT(&WorkHead);

    for (;;)  {
	struct pkg_queue	*lp;
	fd_set ifds;
	struct timeval tv;

	/* First, process any packages in library buffers */
	if (pkg_process(pcsrv) < 0)  {
	    bu_log("pkg_get error\n");
	    break;
	}

	/* Second, see if any input to read */
	FD_ZERO(&ifds);
	FD_SET(pcsrv->pkc_fd, &ifds);
	tv.tv_sec = BU_LIST_NON_EMPTY(&WorkHead) ? 0L : 9999L;
	tv.tv_usec = 0L;

	if (select(pcsrv->pkc_fd+1, &ifds, (fd_set *)0, (fd_set *)0,
		    &tv) != 0)  {
	    n = pkg_suckin(pcsrv);
	    if (n < 0)  {
		bu_log("pkg_suckin error\n");
		break;
	    } else if (n == 0)  {
		/* EOF detected */
		break;
	    } else {
		/* All is well */
	    }
	}

	/* Third, process any new packages in library buffers */
	if (pkg_process(pcsrv) < 0)  {
	    bu_log("pkg_get error\n");
	    break;
	}

	/* Finally, more work may have just arrived, check our list */
	if (BU_LIST_NON_EMPTY(&WorkHead))  {
	    lp = BU_LIST_FIRST(pkg_queue, &WorkHead);
	    BU_LIST_DEQUEUE(&lp->l);
	    switch (lp->type)  {
		case MSG_MATRIX:
		    ph_matrix((struct pkg_conn *)0, lp->buf);
		    break;
		case MSG_LINES:
		    ph_lines((struct pkg_conn *)0, lp->buf);
		    break;
		case MSG_OPTIONS:
		    ph_options((struct pkg_conn *)0, lp->buf);
		    break;
		case MSG_GETTREES:
		    ph_gettrees((struct pkg_conn *)0, lp->buf);
		    break;
		default:
		    bu_log("bad list element, type=%d\n", lp->type);
		    return 33;
	    }
	    BU_PUT(lp, struct pkg_queue);
	}
    }

    return 0;		/* bu_exit(0, NULL) */
}
Exemple #5
0
int
main (int argc, char **argv)
{
    char	buf[BUF_LEN];	/* Contents of current input line */
    char	*bp;		/* Pointer into buf */
    char	*nlp;		/* Location of newline in buf */
    char	rname[BUF_LEN];	/* Name of current region */
    char	rayname[BUF_LEN];	/* Name of ray */
    fastf_t	ray_radius = 1.0;	/* Thickness of the RCC */
    point_t	entryp;		/* Ray's entry into current region */
    point_t	exitp;		/* Ray's exit from current region */
    point_t	first_entryp;	/* Ray's entry into the entire geometry */
    int		i;		/* Index into rname */
    int		line_nm = 0;	/* Number of current line of input */
    int		opt;		/* Command-line option returned by bu_getopt */
    int		pid;		/* Process ID for unique group name */

    extern char *bu_optarg;
    extern int  bu_optind, bu_opterr;

    int         bu_getopt(int, char *const *, const char *);

    pid = bu_process_id();

    *rayname = '\0';
    /* Handle command-line options */
    while ((opt = bu_getopt(argc, argv, OPT_STRING)) != -1)
	switch (opt) {
	    case 'n':
		bu_strlcpy(rayname, bu_optarg, BUF_LEN);
		break;
	    case 'r':
		if (sscanf(bu_optarg, "%F", &ray_radius) != 1) {
		    bu_exit(1, "Illegal radius: '%s'\n", bu_optarg);
		}
		break;
	    case '?':
		print_usage();
	}

    /* Ensure proper command-line syntax */
    if (bu_optind != argc) {
	print_usage();
    }

    /* Construct the names of the objects to add to the database */
    if (*rayname == '\0')	/* Was one given on command line? */
	sprintf(rayname, "ray.%d", pid);

    if (strlen(rayname) > NAMESIZE) {
	fprintf(stderr,
		"Name '%s.s' for ray solid may not exceed %d characters\n",
		rayname, NAMESIZE);
	bu_exit(1, "Use the '-n name' option to specify a different name\n");
    }

    printf("in %s.s sph 0 0 0 1\n", rayname);
    printf("r %s.r u %s.s\n", rayname, rayname);
    printf("mater %s.r\n\n\n%s\n\n", rayname, RAY_COLOR);
    printf("g %s", rayname);

    /* Read the input */
    while (bu_fgets(buf, BUF_LEN, stdin) != NULL) {
	++line_nm;
	bp = buf;
	if ((nlp = index(bp, '\n')) != 0)
	    *nlp = '\0';

	/* Skip initial white space */
	while (isspace(*bp))
	    ++bp;

	/* Get region name */
	for (i = 0; ! isspace(*bp) && (*bp != '\0'); ++i, ++bp)
	    rname[i] = *bp;
	rname[i] = '\0';

	/* Read entry and exit coordinates for this partition */
	if (sscanf(bp, "%F%F%F%F%F%F",
		   &entryp[X], &entryp[Y], &entryp[Z],
		   &exitp[X], &exitp[Y], &exitp[Z]) != 6) {
	    bu_exit(1, "Illegal data on line %d: '%s'\n", line_nm, bp);
	}

	printf(" %s", rname);

	if (line_nm == 1) {
	    VMOVE(first_entryp, entryp);
	}
    }
    if (! feof(stdin)) {
	bu_exit(1, "Error from bu_fgets().  This shouldn't happen");
    }

    printf("\nkill %s.s\nin %s.s rcc\n\t%f %f %f\n\t%f %f %f\n\t%f\n",
	   rayname, rayname,
	   first_entryp[X], first_entryp[Y], first_entryp[Z],
	   exitp[X] - first_entryp[X],
	   exitp[Y] - first_entryp[Y],
	   exitp[Z] - first_entryp[Z],
	   ray_radius);
    printf("g %s %s.r\n", rayname, rayname);
    fprintf(stderr, "Group is '%s'\n", rayname);
}
Exemple #6
0
void
bu_parallel(void (*func)(int, void *), int ncpu, void *arg)
{
    /* avoid using the 'register' keyword in here "just in case" */

#ifndef PARALLEL

    bu_log("bu_parallel(%d., %p):  Not compiled for PARALLEL machine, running single-threaded\n", ncpu, arg);
    /* do the work anyways */
    (*func)(0, arg);

#else

    struct thread_data *user_thread_data_bu;
    int avail_cpus = 1;
    int x;

    char *libbu_affinity = NULL;

    /* OFF by default until linux issue is debugged */
    int affinity = 0;

    /*
     * multithreading support for SunOS 5.X / Solaris 2.x
     */
#  if defined(SUNOS) && SUNOS >= 52
    static int concurrency = 0; /* Max concurrency we have set */
#  endif
#  if (defined(SUNOS) && SUNOS >= 52) || defined(HAVE_PTHREAD_H)
    int nthreadc;
    int nthreade;
    rt_thread_t thread;
    rt_thread_t thread_tbl[MAX_PSW];
    int i;
#  endif /* SUNOS */
#  ifdef WIN32
    int nthreadc = ncpu;
    HANDLE hThreadArray[MAX_PSW] = {0};
    int i;
    DWORD returnCode;
#  endif /* WIN32 */

    if (UNLIKELY(bu_debug & BU_DEBUG_PARALLEL))
	bu_log("bu_parallel(%d, %p)\n", ncpu, arg);

    if (UNLIKELY(pid_of_initiating_thread))
	bu_bomb("bu_parallel() called from within parallel section\n");

    pid_of_initiating_thread = bu_process_id();

    if (ncpu > MAX_PSW) {
	bu_log("WARNING: bu_parallel() ncpu(%d) > MAX_PSW(%d), adjusting ncpu\n", ncpu, MAX_PSW);
	ncpu = MAX_PSW;
    }
    parallel_nthreads_started = 0;
    parallel_nthreads_finished = 0;

    libbu_affinity = getenv("LIBBU_AFFINITY");
    if (libbu_affinity)
	affinity = (int)strtol(libbu_affinity, NULL, 0x10);
    if (UNLIKELY(bu_debug & BU_DEBUG_PARALLEL)) {
	if (affinity)
	    bu_log("CPU affinity enabled. (LIBBU_AFFINITY=%d)\n", affinity);
	else
	    bu_log("CPU affinity disabled.\n", affinity);
    }

    user_thread_data_bu = (struct thread_data *)bu_calloc(ncpu, sizeof(*user_thread_data_bu), "struct thread_data *user_thread_data_bu");

    /* Fill in the data of user_thread_data_bu structures of all threads */
    for (x = 0; x < ncpu; x++) {
	user_thread_data_bu[x].user_func = func;
	user_thread_data_bu[x].user_arg  = arg;
	user_thread_data_bu[x].cpu_id    = x;
	user_thread_data_bu[x].counted   = 0;
	user_thread_data_bu[x].affinity  = affinity;
    }

    /* if we're in debug mode, allow additional cpus */
    if (!(bu_debug & BU_DEBUG_PARALLEL)) {
	avail_cpus = bu_avail_cpus();
	if (ncpu > avail_cpus) {
	    bu_log("%d cpus requested, but only %d available\n", ncpu, avail_cpus);
	    ncpu = avail_cpus;
	}
    }

    /*
     * multithreading support for SunOS 5.X / Solaris 2.x
     */
#  if defined(SUNOS) && SUNOS >= 52

    thread = 0;
    nthreadc = 0;

    /* Give the thread system a hint... */
    if (ncpu > concurrency) {
	if (thr_setconcurrency(ncpu)) {
	    bu_log("ERROR parallel.c/bu_parallel(): thr_setconcurrency(%d) failed\n",
		   ncpu);
	    /* Not much to do, lump it */
	} else {
	    concurrency = ncpu;
	}
    }

    /* Create the threads */
    for (x = 0; x < ncpu; x++) {

	if (thr_create(0, 0, (void *(*)(void *))parallel_interface_arg, &user_thread_data_bu[x], 0, &thread)) {
	    bu_log("ERROR: bu_parallel: thr_create(0x0, 0x0, 0x%x, 0x0, 0, 0x%x) failed for processor thread # %d\n",
		   parallel_interface_arg, &thread, x);
	    /* Not much to do, lump it */
	} else {
	    if (UNLIKELY(bu_debug & BU_DEBUG_PARALLEL))
		bu_log("bu_parallel(): created thread: (thread: 0x%x) (loop:%d) (nthreadc:%d)\n",
		       thread, x, nthreadc);

	    thread_tbl[nthreadc] = thread;
	    nthreadc++;
	}
    }

    if (UNLIKELY(bu_debug & BU_DEBUG_PARALLEL))
	for (i = 0; i < nthreadc; i++)
	    bu_log("bu_parallel(): thread_tbl[%d] = 0x%x\n",
		   i, thread_tbl[i]);

    /*
     * Wait for completion of all threads.  We don't wait for threads
     * in order.  We wait for any old thread but we keep track of how
     * many have returned and whether it is one that we started
     */
    thread = 0;
    nthreade = 0;
    for (x = 0; x < nthreadc; x++) {
	if (UNLIKELY(bu_debug & BU_DEBUG_PARALLEL))
	    bu_log("bu_parallel(): waiting for thread to complete:\t(loop:%d) (nthreadc:%d) (nthreade:%d)\n",
		   x, nthreadc, nthreade);

	if (thr_join((rt_thread_t)0, &thread, NULL)) {
	    /* badness happened */
	    perror("thr_join");
	    bu_log("thr_join() failed");
	}

	/* Check to see if this is one the threads we created */
	for (i = 0; i < nthreadc; i++) {
	    if (thread_tbl[i] == thread) {
		thread_tbl[i] = (rt_thread_t)-1;
		nthreade++;
		break;
	    }
	}

	if ((thread_tbl[i] != (rt_thread_t)-1) && i < nthreadc) {
	    bu_log("bu_parallel(): unknown thread %d completed.\n",
		   thread);
	}

	if (UNLIKELY(bu_debug & BU_DEBUG_PARALLEL))
	    bu_log("bu_parallel(): thread completed: (thread: %d)\t(loop:%d) (nthreadc:%d) (nthreade:%d)\n",
		   thread, x, nthreadc, nthreade);
    }

    if (UNLIKELY(bu_debug & BU_DEBUG_PARALLEL))
	bu_log("bu_parallel(): %d threads created.  %d threads exited.\n",
	       nthreadc, nthreade);
#  endif	/* SUNOS */

#  if defined(HAVE_PTHREAD_H)

    thread = 0;
    nthreadc = 0;

    /* Create the posix threads.
     *
     * Start at 1 so we can treat the parent as thread 0.
     */
    for (x = 0; x < ncpu; x++) {
	pthread_attr_t attrs;
	pthread_attr_init(&attrs);
	pthread_attr_setstacksize(&attrs, 10*1024*1024);

	if (pthread_create(&thread, &attrs, (void *(*)(void *))parallel_interface_arg, &user_thread_data_bu[x])) {
	    bu_log("ERROR: bu_parallel: pthread_create(0x0, 0x0, 0x%lx, 0x0, 0, %p) failed for processor thread # %d\n",
		   (unsigned long int)parallel_interface_arg, (void *)&thread, x);
	    /* Not much to do, lump it */
	} else {
	    if (UNLIKELY(bu_debug & BU_DEBUG_PARALLEL)) {
		bu_log("bu_parallel(): created thread: (thread: %p) (loop: %d) (nthreadc: %d)\n",
		       (void*)thread, x, nthreadc);
	    }

	    thread_tbl[nthreadc] = thread;
	    nthreadc++;
	}
	/* done with the attributes after create */
	pthread_attr_destroy(&attrs);
    }


    if (UNLIKELY(bu_debug & BU_DEBUG_PARALLEL)) {
	for (i = 0; i < nthreadc; i++) {
	    bu_log("bu_parallel(): thread_tbl[%d] = %p\n",
		   i, (void *)thread_tbl[i]);
	}
#    ifdef SIGINFO
	/* may be BSD-only (calls _thread_dump_info()) */
	raise(SIGINFO);
#    endif
    }

    /*
     * Wait for completion of all threads.
     * Wait for them in order.
     */
    thread = 0;
    nthreade = 0;
    for (x = 0; x < nthreadc; x++) {
	int ret;

	if (UNLIKELY(bu_debug & BU_DEBUG_PARALLEL))
	    bu_log("bu_parallel(): waiting for thread %p to complete:\t(loop:%d) (nthreadc:%d) (nthreade:%d)\n",
		   (void *)thread_tbl[x], x, nthreadc, nthreade);

	if ((ret = pthread_join(thread_tbl[x], NULL)) != 0) {
	    /* badness happened */
	    bu_log("pthread_join(thread_tbl[%d]=%p) ret=%d\n", x, (void *)thread_tbl[x], ret);
	}

	nthreade++;
	thread_tbl[x] = (rt_thread_t)-1;

	if (UNLIKELY(bu_debug & BU_DEBUG_PARALLEL))
	    bu_log("bu_parallel(): thread completed: (thread: %p)\t(loop:%d) (nthreadc:%d) (nthreade:%d)\n",
		   (void *)thread, x, nthreadc, nthreade);
    }

    if (UNLIKELY(bu_debug & BU_DEBUG_PARALLEL))
	bu_log("bu_parallel(): %d threads created.  %d threads exited.\n",
	       nthreadc, nthreade);

#  endif /* end if posix threads */


#  ifdef WIN32
    /* Create the Win32 threads */

    for (i = 0; i < nthreadc; i++) {
	hThreadArray[i] = CreateThread(
	    NULL,
	    0,
	    (LPTHREAD_START_ROUTINE)parallel_interface_arg_stub,
	    &user_thread_data_bu[i],
	    0,
	    NULL);

	/* Ensure that all successfully created threads are in sequential order.*/
	if (hThreadArray[i] == NULL) {
	    bu_log("bu_parallel(): Error in CreateThread, Win32 error code %d.\n", GetLastError());
	    --nthreadc;
	}
    }
    /* Wait for other threads in the array */


    returnCode = WaitForMultipleObjects(nthreadc, hThreadArray, TRUE, INFINITE);
    if (returnCode == WAIT_FAILED) {
	bu_log("bu_parallel(): Error in WaitForMultipleObjects, Win32 error code %d.\n", GetLastError());
    }

    for (x = 0; x < nthreadc; x++) {
	int ret;
	if ((ret = CloseHandle(hThreadArray[x]) == 0)) {
	    /* Thread didn't close properly if return value is zero; don't retry and potentially loop forever.  */
	    bu_log("bu_parallel(): Error closing thread %d of %d, Win32 error code %d.\n", x, nthreadc, GetLastError());
	}
    }
#  endif /* end if Win32 threads */

    /*
     * Ensure that all the threads are REALLY finished.  On some
     * systems, if threads core dump, the rest of the gang keeps
     * going, so this can actually happen (sigh).
     */
    if (UNLIKELY(parallel_nthreads_finished != parallel_nthreads_started)) {
	bu_log("*** ERROR bu_parallel(%d): %d workers did not finish!\n\n",
	       ncpu, ncpu - parallel_nthreads_finished);
    }
    if (UNLIKELY(parallel_nthreads_started != ncpu)) {
	bu_log("bu_parallel() NOTICE:  only %d workers started, expected %d\n",
	       parallel_nthreads_started, ncpu);
    }

    if (UNLIKELY(bu_debug & BU_DEBUG_PARALLEL))
	bu_log("bu_parallel(%d) complete, now serial\n", ncpu);

#  if defined(unix) || defined(__unix)
    /* Cray is known to wander among various pids, perhaps others.
     *
     * At this point, all multi-tasking activity should have ceased,
     * and we should be just a single UNIX process with our original
     * PID and open file table (kernel struct u).  If not, then any
     * output may be written into the wrong file.
     */
    x = bu_process_id();
    if (UNLIKELY(pid_of_initiating_thread != x)) {
	bu_log("WARNING: bu_parallel():  PID of initiating thread changed from %d to %d, open file table may be botched!\n",
	       pid_of_initiating_thread, x);
    }
#  endif
    pid_of_initiating_thread = 0;	/* No threads any more */

    bu_free(user_thread_data_bu, "struct thread_data *user_thread_data_bu");

#endif /* PARALLEL */

    return;
}