Пример #1
0
void
UserProc::execute()
{
	ArgList new_args;
	char    **argv;
	char	**argp;
	char	**envp;
	sigset_t	sigmask;
	MyString	a_out_name;
	MyString	shortname;
	int		user_syscall_fd = -1;
	const	int READ_END = 0;
	const	int WRITE_END = 1;
	int		pipe_fds[2];
	FILE	*cmd_fp;
	char	buf[128];
	ReliSock	*new_reli = NULL;

	pipe_fds[0] = -1;
	pipe_fds[1] = -1;

	shortname.formatstr( "condor_exec.%d.%d", cluster, proc );
	a_out_name.formatstr( "%s/%s/%s", Execute, local_dir, shortname.Value() );

		// Set up arg vector according to class of job
	switch( job_class ) {

	  case CONDOR_UNIVERSE_STANDARD:
		if( pipe(pipe_fds) < 0 ) {
			EXCEPT( "pipe()" );}

			dprintf( D_ALWAYS, "Pipe built\n" );
		
			// The user process should not try to read commands from
			// 0, 1, or 2 since we'll be using the commands to redirect
			// those.
		if( pipe_fds[READ_END] < 14 ) {
			dup2( pipe_fds[READ_END], 14 );
			close( pipe_fds[READ_END] );
			pipe_fds[READ_END] = 14;
		}
		dprintf( D_ALWAYS, "New pipe_fds[%d,%d]\n", pipe_fds[0], pipe_fds[1] );
		sprintf( buf, "%d", pipe_fds[READ_END] );
		dprintf( D_ALWAYS, "cmd_fd = %s\n", buf );

		new_args.AppendArg(shortname);
		new_args.AppendArg("-_condor_cmd_fd");
		new_args.AppendArg(buf);
		break;

	  case CONDOR_UNIVERSE_PVM:
#if 1
		EXCEPT( "Don't know how to deal with PVM jobs" );
#else
		new_args.AppendArg(shortname);
		new_args.AppendArg("-1");
		new_args.AppendArg(in);
		new_args.AppendArg(out);
		new_args.AppendArg(err);
#endif
		break;

	  case CONDOR_UNIVERSE_VANILLA:
	  	if (privsep_enabled()) {
			EXCEPT("Don't know how to deal with Vanilla jobs");
		}
		new_args.AppendArg(shortname.Value());
		break;
	}

	new_args.AppendArgsFromArgList(args);

		// take care of USER_JOB_WRAPPER
	support_job_wrapper(a_out_name,&new_args);

	MyString exec_name;
	exec_name = a_out_name;

	// If privsep is turned on, then we need to use the PrivSep
	// Switchboard to launch the job
	//
	FILE* switchboard_in_fp;
	FILE* switchboard_err_fp;
	int switchboard_child_in_fd;
	int switchboard_child_err_fd;
	if (privsep_enabled()) {

		// create the pipes that we'll use to communicate
		//
		if (!privsep_create_pipes(switchboard_in_fp,
		                          switchboard_child_in_fd,
		                          switchboard_err_fp,
		                          switchboard_child_err_fd)) {
			EXCEPT("can't launch job: privsep_create_pipes failure");
		}
	}

	argv = new_args.GetStringArray();

		// Set an environment variable that tells the job where it may put scratch data
		// even if it moves to a different directory.

		// get the environment vector
	envp = env_obj.getStringArray();

		// We may run more than one of these, so each needs its own
		// remote system call connection to the shadow
	if( job_class == CONDOR_UNIVERSE_PVM ) {
		new_reli = NewConnection( v_pid );
		user_syscall_fd = new_reli->get_file_desc();
	}

		// print out arguments to execve
	dprintf( D_ALWAYS, "Calling execve( \"%s\"", exec_name.Value() );
	for( argp = argv; *argp; argp++ ) {							// argv
		dprintf( D_ALWAYS | D_NOHEADER, ", \"%s\"", *argp );
	}
	dprintf( D_ALWAYS | D_NOHEADER, ", 0" );
	for( argp = envp; *argp; argp++ ) {							// envp
		dprintf( D_ALWAYS | D_NOHEADER, ", \"%s\"", *argp );
	}
	dprintf( D_ALWAYS | D_NOHEADER, ", 0 )\n" );


	if( (pid = fork()) < 0 ) {
		EXCEPT( "fork" );
	}

	if( pid == 0 ) {	// the child

			// Block only these 3 signals which have special meaning for
			// checkpoint/restart purposes.  Leave other signals ublocked
			// so that if we get an exception during the restart process,
			// we will get a core file to debug.
		sigemptyset( &sigmask );
		// for some reason if we block these, the user process is unable
		// to unblock some or all of them.
#if 0
		sigaddset( &sigmask, SIGUSR1 );
		sigaddset( &sigmask, SIGUSR2 );
		sigaddset( &sigmask, SIGTSTP );
#endif
		sigprocmask( SIG_SETMASK, &sigmask, 0 );

			// renice
		renice_self( "JOB_RENICE_INCREMENT" );

			// make certain the syscall sockets which are being passed
			// to the user job are setup to be blocking sockets.  this
			// is done by calling timeout(0) CEDAR method.
			// we must do this because the syscall lib does _not_ 
			// expect to see any failures due to errno EAGAIN...
		if ( SyscallStream ) {
			SyscallStream->timeout(0);
		}
		if ( new_reli ) {
			new_reli->timeout(0);
		}

			// If I'm using privledge separation, connect to the procd.
			// we need to register a family with the procd for the newly
			// created process, so that the ProcD will allow us to send
			// signals to it
			//
		if (privsep_enabled() == true) {
			MyString procd_address;
			bool response;
			bool ret;
			ProcFamilyClient pfc;

			procd_address = get_procd_address();
			ret = pfc.initialize(procd_address.Value());
			if (ret == false) {
				EXCEPT("Failure to initialize the ProcFamilyClient object");
			}

			ret = pfc.register_subfamily(getpid(), getppid(), 60, response);

			if (ret == false) {
				EXCEPT("Could not communicate with procd. Aborting.");
			}

			if (response == false) {
				EXCEPT("Procd refused to register job subfamily. Aborting.");
			}
		}

			// If there is a requested coresize for this job, enforce it.
			// Do it before the set_priv_final to ensure root can alter 
			// the coresize to the requested amount. Otherwise, just
			// use whatever the current default is.
		if (coredump_limit_exists == TRUE) {
			limit( RLIMIT_CORE, coredump_limit, CONDOR_HARD_LIMIT, "max core size" );
		}

			// child process should have only it's submitting uid, and cannot
			// switch back to root or some other uid.  
			// It'd be nice to check for errors here, but
			// unfortunately, we can't, since this only returns the
			// previous priv state, not whether it worked or not. 
			//  -Derek Wright 4/30/98
		set_user_priv_final();

		switch( job_class ) {
		  
		  case CONDOR_UNIVERSE_STANDARD:
			// if we're using PrivSep, the chdir here could fail. instead,
			// we pass the job's IWD to the switchboard via pipe
			//
		  	if (!privsep_enabled()) {
				if( chdir(local_dir) < 0 ) {
					EXCEPT( "chdir(%s)", local_dir );
				}
			}
			close( pipe_fds[WRITE_END] );
			break;

		  case CONDOR_UNIVERSE_PVM:
			if( chdir(local_dir) < 0 ) {
				EXCEPT( "chdir(%s)", local_dir );
			}
			close( pipe_fds[WRITE_END] );
			dup2( user_syscall_fd, RSC_SOCK );
			break;

		  case CONDOR_UNIVERSE_VANILLA:
			set_iwd();
			open_std_file( 0 );
			open_std_file( 1 );
			open_std_file( 2 );

			(void)close( RSC_SOCK );
			(void)close( CLIENT_LOG );

			break;
		}

			// Make sure we're not root
		if( getuid() == 0 ) {
				// EXCEPT( "We're about to start as root, aborting." );
				// You can't see this error message at all.  So, just
				// exit(4), which is what EXCEPT normally gives. 
			exit( 4 );
		}

#if defined( LINUX ) && (defined(I386) || defined(X86_64))
		// adjust the execution domain of the child to be suitable for
		// checkpointing.
		patch_personality();
#endif 

			// if we're using privsep, we'll exec the PrivSep Switchboard
			// first, which is setuid; it will then setuid to the user we
			// give it and exec the real job
			//
		if (privsep_enabled()) {
			close(fileno(switchboard_in_fp));
			close(fileno(switchboard_err_fp));
			privsep_get_switchboard_command("exec",
			                                switchboard_child_in_fd,
			                                switchboard_child_err_fd,
			                                exec_name,
			                                new_args);
			deleteStringArray(argv);
			argv = new_args.GetStringArray();
		}

			// Everything's ready, start it up...
		errno = 0;
		execve( exec_name.Value(), argv, envp );

			// A successful call to execve() never returns, so it is an
			// error if we get here.  A number of errors are possible
			// but the most likely is that there is insufficient swap
			// space to start the new process.  We don't try to log
			// anything, since we have the UID/GID of the job's owner
			// and cannot write into the log files...
		exit( JOB_EXEC_FAILED );
	}

		// The parent

		// PrivSep - we have at this point only spawned the switchboard
		// with the "exec" command. we need to use our pipe to it in
		// order to tell it how to execute the user job, and then use
		// the error pipe to make sure everything worked
		//
	if (privsep_enabled()) {

		close(switchboard_child_in_fd);
		close(switchboard_child_err_fd);

		privsep_exec_set_uid(switchboard_in_fp, uid);
		privsep_exec_set_path(switchboard_in_fp, exec_name.Value());
		privsep_exec_set_args(switchboard_in_fp, new_args);
		privsep_exec_set_env(switchboard_in_fp, env_obj);
		privsep_exec_set_iwd(switchboard_in_fp, local_dir);
		privsep_exec_set_inherit_fd(switchboard_in_fp, pipe_fds[0]);
		privsep_exec_set_inherit_fd(switchboard_in_fp, RSC_SOCK);
		privsep_exec_set_inherit_fd(switchboard_in_fp, CLIENT_LOG);
		privsep_exec_set_is_std_univ(switchboard_in_fp);
		fclose(switchboard_in_fp);

		if (!privsep_get_switchboard_response(switchboard_err_fp)) {
			EXCEPT("error starting job: "
			           "privsep get_switchboard_response failure");
		}
	}

	dprintf( D_ALWAYS, "Started user job - PID = %d\n", pid );
	if( job_class != CONDOR_UNIVERSE_VANILLA ) {
			// Send the user process its startup environment conditions
		close( pipe_fds[READ_END] );
		cmd_fp = fdopen( pipe_fds[WRITE_END], "w" );
		dprintf( D_ALWAYS, "cmd_fp = %p\n", cmd_fp );

		if( is_restart() ) {
#if 1
			fprintf( cmd_fp, "restart\n" );
			dprintf( D_ALWAYS, "restart\n" );
#else
			fprintf( cmd_fp, "restart %s\n", target_ckpt );
			dprintf( D_ALWAYS, "restart %s\n", target_ckpt );
#endif
			fprintf( cmd_fp, "end\n" );
			dprintf( D_ALWAYS, "end\n" );
		} else {
			fprintf( cmd_fp, "end\n" );
			dprintf( D_ALWAYS, "end\n" );
		}
		fclose( cmd_fp );
	}

	deleteStringArray(argv);
	deleteStringArray(envp);
	state = EXECUTING;

	if( new_reli ) {
		delete new_reli;
	}

	// removed some vanilla-specific code here
	//
	ASSERT(job_class != CONDOR_UNIVERSE_VANILLA);
}
Пример #2
0
static int
privsep_create_process(const char* cmd,
                       const char* path,
                       ArgList&    args,
                       Env*        env,
                       const char* iwd,
                       int         std_fds[3],
                       const char* std_file_names[3],
                       int         nice_inc,
                       size_t*     core_size_ptr,
                       int         reaper_id,
                       int         dc_job_opts,
                       FamilyInfo* family_info,
                       uid_t       uid,
					   int * 	   affinity_mask = 0)
{
	// if we're using privilege separation, we have to do a bit
	// of extra work to do to launch processes as other UIDs
	// since we need to use the PrivSep Swtichboard. the procedure
	// will be:
	//
	//   1) create two pipes over which we'll communicate with
	//      the switchboard
	//   2) call create process on the switchboard
	//   3) use the input pipe to give the switchboard info about
	//      what binary to run, what files to use for i/o, etc.
	//   4) use the error pipe to see if everything went ok

	// create the pipes
	//
	FILE* in_fp;
	int child_in_fd;
	FILE* err_fp;
	int child_err_fd;
	bool ok;
	ok = privsep_create_pipes(in_fp,
	                          child_in_fd,
	                          err_fp,
	                          child_err_fd);
	if (!ok) {
		dprintf(D_ALWAYS,
		        "privsep_create_process: privsep_create_pipes failure\n");
		errno = 0;
		return FALSE;
	}

	// fire up the switchboard
	//
	MyString sb_path;
	ArgList sb_args;
	privsep_get_switchboard_command(cmd,
	                                child_in_fd,
	                                child_err_fd,
	                                sb_path,
	                                sb_args);
	int pipe_fds[] = {child_in_fd, child_err_fd, 0};
	int pid = daemonCore->Create_Process(
		sb_path.Value(), // switchboard path
		sb_args,         // switchboard args
		PRIV_USER_FINAL, // priv states are ignored in priv sep mode
		reaper_id,       // reaper id
		FALSE,           // no command port
		FALSE,           // no command port
		NULL,            // we'll pass the job's real env via pipe
		NULL,            // we'll pass the job's real iwd via pipe
		family_info,     // tracking info for the ProcD
		NULL,            // no sockets to inherit
		std_fds,         // FDs that will become std{in,out,err}
		pipe_fds,        // tell DC to pass down the in/err pipe FDs
		nice_inc,        // niceness (TODO: need switchboard?)
		NULL,            // don't mess with the signal mask
		dc_job_opts,     // DC job options
		core_size_ptr,  // core size limit (TODO: need switchboard?)
		affinity_mask);  // Processor affinity mask
	close(child_in_fd);
	close(child_err_fd);
	if (pid == FALSE) {
		dprintf(D_ALWAYS, "privsep_create_process: DC::Create_Process error\n");
		fclose(in_fp);
		fclose(err_fp);
		return FALSE;
	}

	// feed the switchboard information on how to create the new
	// process
	//
	privsep_exec_set_uid(in_fp, uid);
	privsep_exec_set_path(in_fp, path);
	privsep_exec_set_args(in_fp, args);
	Env tmp_env;
	if (HAS_DCJOBOPT_ENV_INHERIT(dc_job_opts)) {
		tmp_env.MergeFrom(GetEnviron());
		if (env != NULL) {
			tmp_env.MergeFrom(*env);
		}
		env = &tmp_env;
	}
	if (env != NULL) {
		privsep_exec_set_env(in_fp, *env);
	}
	if (iwd != NULL) {
		privsep_exec_set_iwd(in_fp, iwd);
	}
	for (int i = 0; i < 3; i++) {
		if ((std_fds != NULL) && (std_fds[i] != -1)) {
			privsep_exec_set_inherit_fd(in_fp, i);
		}
		else if (std_file_names) {
			privsep_exec_set_std_file(in_fp, i, std_file_names[i]);
		}
	}
#if defined(LINUX)
	if ((family_info != NULL) && (family_info->group_ptr != NULL)) {
		privsep_exec_set_tracking_group(in_fp, *family_info->group_ptr);
	}
#endif
	fclose(in_fp);

	// check the switchboard's error pipe for any problems
	// (privsep_get_switchboard_response will fclose the error
	// pipe for us)
	//
	if (!privsep_get_switchboard_response(err_fp)) {
		dprintf(D_ALWAYS,
		        "privsep_create_process: "
		            "privsep_get_switchboard_response failure\n");
		errno = 0;
		return FALSE;
	}

	// if we've gotten here, everything worked
	//
	return pid;
}
Пример #3
0
static FILE *
my_popenv_impl( const char *const args[],
                const char * mode,
                int want_stderr,
                uid_t privsep_uid,
		Env *env_ptr = 0,
		bool drop_privs = true )
{
	int	pipe_d[2], pipe_d2[2];
	int	parent_reads;
	uid_t	euid;
	gid_t	egid;
	pid_t	pid;
	FILE*	retp;

		/* Figure out who reads and who writes on the pipe */
	parent_reads = (mode[0] == 'r');

		/* Create the pipe */
	if( pipe(pipe_d) < 0 ) {
		dprintf(D_ALWAYS, "my_popenv: Failed to create the pipe, "
				"errno=%d (%s)\n", errno, strerror(errno));
		return NULL;
	}

		/* Prepare for PrivSep if needed */
	PrivSepForkExec psforkexec;
	if ( privsep_uid != (uid_t)-1 ) {
		if (!psforkexec.init()) {
			dprintf(D_ALWAYS,
			        "my_popenv failure on %s\n",
			        args[0]);
			close(pipe_d[0]);
			close(pipe_d[1]);
			return NULL;
		}
	}

		/* Create a pipe to detect execv failures */
	if ( pipe(pipe_d2) < 0) {
		dprintf(D_ALWAYS, "my_popenv: Failed to create the pre-exec pipe, "
				"errno=%d (%s)\n", errno, strerror(errno));
		close(pipe_d[0]);
		close(pipe_d[1]);
		return NULL;
	}
	int fd_flags;
	if ((fd_flags = fcntl(pipe_d2[1], F_GETFD, NULL)) == -1) {
		dprintf(D_ALWAYS, "my_popenv: Failed to get fd flags: errno=%d (%s)\n", errno, strerror(errno));
		close( pipe_d[0] );
		close( pipe_d[1] );
		close( pipe_d2[0] );
		close( pipe_d2[1] );
		return NULL;
	}
	if (fcntl(pipe_d2[1], F_SETFD, fd_flags | FD_CLOEXEC) == -1) {
		dprintf(D_ALWAYS, "my_popenv: Failed to set new fd flags: errno=%d (%s)\n", errno, strerror(errno));
		close( pipe_d[0] );
		close( pipe_d[1] );
		close( pipe_d2[0] );
		close( pipe_d2[1] );
		return NULL;
	}

		/* Create a new process */
	if( (pid=fork()) < 0 ) {
		dprintf(D_ALWAYS, "my_popenv: Failed to fork child, errno=%d (%s)\n",
				errno, strerror(errno));
			/* Clean up file descriptors */
		close( pipe_d[0] );
		close( pipe_d[1] );
		close( pipe_d2[0] );
		close( pipe_d2[1] );
		return NULL;
	}

		/* The child */
	if( pid == 0 ) {

		/* Don't leak out fds from the parent to our child.
		 * Wish there was a more efficient way to do this, but
		 * this is how we do it in daemoncore CreateProcess...
		 * Of course, do not close stdin/out/err or the fds to
		 * the pipes we just created above.
		 */
		for (int jj=3; jj < getdtablesize(); jj++) {
			if (jj != pipe_d[0] &&
				jj != pipe_d[1] &&
				jj != pipe_d2[0] &&
				jj != pipe_d2[1])
			{
				close(jj);
			}
		}

		close(pipe_d2[0]);

		if( parent_reads ) {
				/* Close stdin, dup pipe to stdout */
			close( pipe_d[READ_END] );
			bool close_pipe_end = false;
			if( pipe_d[WRITE_END] != 1 ) {
				dup2( pipe_d[WRITE_END], 1 );
				close_pipe_end = true;
			}
			if (want_stderr) {
				if ( pipe_d[WRITE_END] != 2 ) {
					dup2( pipe_d[WRITE_END], 2 );
				}
				else {
					close_pipe_end = false;
				}
			}
			if (close_pipe_end) {
				close(pipe_d[WRITE_END]);
			}
		} else {
				/* Close stdout, dup pipe to stdin */
			close( pipe_d[WRITE_END] );
			if( pipe_d[READ_END] != 0 ) {
				dup2( pipe_d[READ_END], 0 );
				close( pipe_d[READ_END] );
			}
		}
			/* to be safe, we want to switch our real uid/gid to our
			   effective uid/gid (shedding any privledges we've got).
			   we also want to drop any supplimental groups we're in.
			   we want to run this popen()'ed thing as our effective
			   uid/gid, dropping the real uid/gid.  all of these calls
			   will fail if we don't have a ruid of 0 (root), but
			   that's harmless.  also, note that we have to stash our
			   effective uid, then switch our euid to 0 to be able to
			   set our real uid/gid.
			   We wrap some of the calls in if-statements to quiet some
			   compilers that object to us not checking the return values.
			*/
		if (drop_privs) {
			euid = geteuid();
			egid = getegid();
			if( seteuid( 0 ) ) { }
			setgroups( 1, &egid );
			if( setgid( egid ) ) { }
			if( setuid( euid ) ) _exit(ENOEXEC); // Unsafe?
		}

			/* before we exec(), clear the signal mask and reset SIGPIPE
			   to SIG_DFL
			*/
		install_sig_handler(SIGPIPE, SIG_DFL);
		sigset_t sigs;
		sigfillset(&sigs);
		sigprocmask(SIG_UNBLOCK, &sigs, NULL);

			/* handle PrivSep if needed */
		MyString cmd = args[0];
		if ( privsep_uid != (uid_t)-1 ) {
			ArgList al;
			psforkexec.in_child(cmd, al);
			args = al.GetStringArray();			
		}

			/* set environment if defined */
		if (env_ptr) {
			char **m_unix_env = NULL;
			m_unix_env = env_ptr->getStringArray();
			execve(cmd.Value(), const_cast<char *const*>(args), m_unix_env );

				// delete the memory even though we're on our way out
				// if exec failed.
			if (m_unix_env) {
				int i = 0;
				while (m_unix_env[i]) {
					delete m_unix_env[i];
					i++;
				}
				delete [] m_unix_env;
			}

		} else {
			execvp(cmd.Value(), const_cast<char *const*>(args) );
		}

			/* If we get here, inform the parent of our errno */
		char result_buf[10];
		int e = errno; // capture real errno

		int len = snprintf(result_buf, 10, "%d", errno);
		int ret = write(pipe_d2[1], result_buf, len);

			// Jump through some hoops just to use ret.
		if (ret <  1) {
			_exit( e );
		} else {
			_exit( e );
		}
	}

		/* The parent */
		/* First, wait until the exec is called - determine status */
	close(pipe_d2[1]);
	int exit_code;
	FILE *fh;
	if ((fh = fdopen(pipe_d2[0], "r")) == NULL) {
		dprintf(D_ALWAYS, "my_popenv: Failed to reopen file descriptor as file handle: errno=%d (%s)", errno, strerror(errno));
		close(pipe_d2[0]);
		close(pipe_d[0]);
		close(pipe_d[1]);

		/* Ensure child process is dead, then wait for it to exit */
		kill(pid, SIGKILL);
		while( waitpid(pid,NULL,0) < 0 && errno == EINTR ) {
			/* NOOP */
		}

		return NULL;
	}
		/* Handle case where exec fails */
	if (fscanf(fh, "%d", &exit_code) == 1) {
		fclose(fh);
		close(pipe_d[0]);
		close(pipe_d[1]);

		/* Ensure child process is dead, then wait for it to exit */
		kill(pid, SIGKILL);
		while( waitpid(pid,NULL,0) < 0 && errno == EINTR ) {
			/* NOOP */
		}

		errno = exit_code;
		return NULL;
	}
	fclose(fh);

	if( parent_reads ) {
		close( pipe_d[WRITE_END] );
		retp = fdopen(pipe_d[READ_END],mode);
	} else {
		close( pipe_d[READ_END] );
		retp = fdopen(pipe_d[WRITE_END],mode);
	}
	add_child(retp, pid);

		/* handle PrivSep if needed */
	if ( privsep_uid != (uid_t)-1 ) {
		FILE* fp = psforkexec.parent_begin();
		privsep_exec_set_uid(fp, privsep_uid);
		privsep_exec_set_path(fp, args[0]);
		ArgList al;
		for (const char* const* arg = args; *arg != NULL; arg++) {
			al.AppendArg(*arg);
		}
		privsep_exec_set_args(fp, al);
		Env env;
		env.Import();
		privsep_exec_set_env(fp, env);
		privsep_exec_set_iwd(fp, ".");
		if (parent_reads) {
			privsep_exec_set_inherit_fd(fp, 1);
			if (want_stderr) {
				privsep_exec_set_inherit_fd(fp, 2);
			}
		}
		else {
			privsep_exec_set_inherit_fd(fp, 0);
		}
		if (!psforkexec.parent_end()) {
			dprintf(D_ALWAYS,
			        "my_popenv failure on %s\n",
			        args[0]);
			fclose(retp);
			return NULL;
		}
	}

	return retp;
}
Пример #4
0
/**
 * merge_stderr_with_stdout is intended for clients of this function
 * that wish to have the old behavior, where stderr and stdout were
 * both added to the same StringList.
 */
int systemCommand( ArgList &args, priv_state priv, StringList *cmd_out, StringList * cmd_in,
		   StringList *cmd_err, bool merge_stderr_with_stdout)
{
	int result = 0;
	FILE *fp = NULL;
	FILE * fp_for_stdin = NULL;
	FILE * childerr = NULL;
	MyString line;
	char buff[1024];
	StringList *my_cmd_out = cmd_out;

	priv_state prev = PRIV_UNKNOWN;

	int stdout_pipes[2];
	int stdin_pipes[2];
	int pid;
	bool use_privsep = false;
	switch ( priv ) {
	case PRIV_ROOT:
		prev = set_root_priv();
		break;
	case PRIV_USER:
	case PRIV_USER_FINAL:
		prev = set_user_priv();
#if !defined(WIN32)
		if ( privsep_enabled() && (job_user_uid != get_condor_uid()) ) {
			use_privsep = true;
		}
#endif
		break;
	default:
		// Stay as Condor user
		;
	}
#if defined(WIN32)
	if((cmd_in != NULL) || (cmd_err != NULL))
	  {
	    vmprintf(D_ALWAYS, "Invalid use of systemCommand() in Windows.\n");
	    return -1;
	  }
	//if ( use_privsep ) {
	//	fp = privsep_popen(args, "r", want_stderr, job_user_uid);
	//}
	//else {
	fp = my_popen( args, "r", merge_stderr_with_stdout );
	//}
#else
	// The old way of doing things (and the Win32 way of doing
	//	things)
	// fp = my_popen( args, "r", want_stderr );
	if((cmd_err != NULL) && merge_stderr_with_stdout)
	  {
	    vmprintf(D_ALWAYS, "Invalid use of systemCommand().\n");
	    return -1;
	  }

	PrivSepForkExec psforkexec;
	char ** args_array = args.GetStringArray();
	int error_pipe[2];
		// AIX 5.2, Solaris 5.9, HPUX 11 don't have AF_LOCAL

	if(pipe(stdin_pipes) < 0)
	  {
	    vmprintf(D_ALWAYS, "Error creating pipe: %s\n", strerror(errno));
		deleteStringArray( args_array );
	    return -1;
	  }
	if(pipe(stdout_pipes) < 0)
	  {
	    vmprintf(D_ALWAYS, "Error creating pipe: %s\n", strerror(errno));
	    close(stdin_pipes[0]);
	    close(stdin_pipes[1]);
		deleteStringArray( args_array );
	    return -1;
	  }

	if ( use_privsep ) {
	  if(!psforkexec.init())
	    {
	      vmprintf(D_ALWAYS,
		       "my_popenv failure on %s\n",
		       args_array[0]);
	      close(stdin_pipes[0]);
	      close(stdin_pipes[1]);
	      close(stdout_pipes[0]);
	      close(stdout_pipes[1]);
		  deleteStringArray( args_array );
	      return -1;
	    }
	}

	if(cmd_err != NULL)
	  {
	    if(pipe(error_pipe) < 0)
	      {
		vmprintf(D_ALWAYS, "Could not open pipe for error output: %s\n", strerror(errno));
		close(stdin_pipes[0]);
		close(stdin_pipes[1]);
		close(stdout_pipes[0]);
		close(stdout_pipes[1]);
		deleteStringArray( args_array );
		return -1;
	      }
	  }
	// Now fork and do what my_popen used to do
	pid = fork();
	if(pid < 0)
	  {
	    vmprintf(D_ALWAYS, "Error forking: %s\n", strerror(errno));
		close(stdin_pipes[0]);
		close(stdin_pipes[1]);
		close(stdout_pipes[0]);
		close(stdout_pipes[1]);
		if(cmd_err != NULL) {
			close(error_pipe[0]);
			close(error_pipe[1]);
		}
		deleteStringArray( args_array );
	    return -1;
	  }
	if(pid == 0)
	  {
	    close(stdout_pipes[0]);
	    close(stdin_pipes[1]);
	    dup2(stdout_pipes[1], STDOUT_FILENO);
	    dup2(stdin_pipes[0], STDIN_FILENO);

	    if(merge_stderr_with_stdout) dup2(stdout_pipes[1], STDERR_FILENO);
	    else if(cmd_err != NULL) 
	      {
		close(error_pipe[0]);
		dup2(error_pipe[1], STDERR_FILENO);
	      }


	    uid_t euid = geteuid();
	    gid_t egid = getegid();
	    seteuid( 0 );
	    setgroups( 1, &egid );
	    setgid( egid );
	    setuid( euid );
	    
	    install_sig_handler(SIGPIPE, SIG_DFL);
	    sigset_t sigs;
	    sigfillset(&sigs);
	    sigprocmask(SIG_UNBLOCK, &sigs, NULL);


	    MyString cmd = args_array[0];

	    if ( use_privsep ) {
	    
	      ArgList al;
	      psforkexec.in_child(cmd, al);
          deleteStringArray( args_array );
	      args_array = al.GetStringArray();
	    }


	    execvp(cmd.Value(), args_array);
	    vmprintf(D_ALWAYS, "Could not execute %s: %s\n", args_array[0], strerror(errno));
	    exit(-1);
	  }
	close(stdin_pipes[0]);
	close(stdout_pipes[1]);
	fp_for_stdin = fdopen(stdin_pipes[1], "w");
	fp = fdopen(stdout_pipes[0], "r");
	if(cmd_err != NULL)
	  {
	    close(error_pipe[1]);
	    childerr = fdopen(error_pipe[0],"r");
	    if(childerr == 0)
	      {
		vmprintf(D_ALWAYS, "Could not open pipe for reading child error output: %s\n", strerror(errno));
		close(error_pipe[0]);
		close(stdin_pipes[1]);
		close(stdout_pipes[0]);
	    fclose(fp);
		fclose(fp_for_stdin);
		deleteStringArray( args_array );
		return -1;
	      }
	  }

	if ( use_privsep ) {
	  FILE* _fp = psforkexec.parent_begin();
	  privsep_exec_set_uid(_fp, job_user_uid);
	  privsep_exec_set_path(_fp, args_array[0]);
	  privsep_exec_set_args(_fp, args);
	  Env env;
	  env.MergeFrom(environ);
	  privsep_exec_set_env(_fp, env);
	  privsep_exec_set_iwd(_fp, ".");

	  privsep_exec_set_inherit_fd(_fp, 1);
	  privsep_exec_set_inherit_fd(_fp, 2);
	  privsep_exec_set_inherit_fd(_fp, 0);
	
	  if (!psforkexec.parent_end()) {
	    vmprintf(D_ALWAYS,
		     "my_popenv failure on %s\n",
		     args_array[0]);
	    fclose(fp);
		fclose(fp_for_stdin);
		if (childerr) {
			fclose(childerr);
		}
		deleteStringArray( args_array );
	    return -1;
	  }
	}

	deleteStringArray( args_array );
#endif
	set_priv( prev );
	if ( fp == NULL ) {
		MyString args_string;
		args.GetArgsStringForDisplay( &args_string, 0 );
		vmprintf( D_ALWAYS, "Failed to execute command: %s\n",
				  args_string.Value() );
		if (childerr)
			fclose(childerr);
		return -1;
	}

	if(cmd_in != NULL) {
	  cmd_in->rewind();
	  char * tmp;
	  while((tmp = cmd_in->next()) != NULL)
	    {
	      fprintf(fp_for_stdin, "%s\n", tmp);
	      fflush(fp_for_stdin);
	    }
	}
	if (fp_for_stdin) {
	  // So that we will not be waiting for output while the
	  // script waits for stdin to be closed.
	  fclose(fp_for_stdin);
	}

	if ( my_cmd_out == NULL ) {
		my_cmd_out = new StringList();
	}

	while ( fgets( buff, sizeof(buff), fp ) != NULL ) {
		line += buff;
		if ( line.chomp() ) {
			my_cmd_out->append( line.Value() );
			line = "";
		}
	}

	if(cmd_err != NULL)
	  {
	    while(fgets(buff, sizeof(buff), childerr) != NULL)
	      {
		line += buff;
		if(line.chomp())
		  {
		    cmd_err->append(line.Value());
		    line = "";
		  }
	      }
	    fclose(childerr);
	  }
#if defined(WIN32)
	result = my_pclose( fp );
#else
	// Why close first?  Just in case the child process is waiting
	// on a read, and we have nothing more to send it.  It will
	// now receive a SIGPIPE.
	fclose(fp);
	if(waitpid(pid, &result, 0) < 0)
	  {
	    vmprintf(D_ALWAYS, "Unable to wait: %s\n", strerror(errno));
		if ( cmd_out == NULL ) {
			delete my_cmd_out;
		}
	   
	    return -1;
	  }
#endif
	if( result != 0 ) {
		MyString args_string;
		args.GetArgsStringForDisplay(&args_string,0);
		vmprintf(D_ALWAYS,
		         "Command returned non-zero: %s\n",
		         args_string.Value());
		my_cmd_out->rewind();
		const char *next_line;
		while ( (next_line = my_cmd_out->next()) ) {
			vmprintf( D_ALWAYS, "  %s\n", next_line );
		}
	}
	if ( cmd_out == NULL ) {
		delete my_cmd_out;
	}
	return result;
}