void
check_jobs(std::string &alias, std::map<pid_t, std::string> & running_processes, const int max_concurrent_jobs)
{
airport::Mongo mongoDB;
mongo::DBClientConnection mongoConnection;
mongoDB.connect(&mongoConnection);
mongoConnection.ensureIndex(MONGO_DISPATCHER_COLLECTION, mongo::fromjson("{servername:1, started_datetime:1, completed_datetime:1, archived:1}"));
std::auto_ptr<mongo::DBClientCursor> cursor = mongoConnection.query( MONGO_DISPATCHER_COLLECTION, mongo::QUERY("servername"<<alias<<"started_datetime"<<0<<"completed_datetime"<<0<<"archived"<<false) );
while(cursor->more())
{
mongo::BSONObj b = cursor->next();
if (running_processes.size() >= max_concurrent_jobs)
continue;
std::string rulename = b.getStringField("rulename");
std::string rulecontent = b.getStringField("rule");
std::string rule_path = airport::DATA_PATH + "rules/" + rulename;
std::ofstream rulefile (rule_path.c_str());
rulefile << rulecontent;
rulefile.close();
pid_t pid = subprocess( rulename, alias );
if (pid > 0)
{
running_processes.insert(std::map<pid_t, std::string>::value_type(pid, rulename) );
mongo::BSONElement e;
bool getId = b.getObjectID (e);
if (getId)
{
mongo::OID oid = e.__oid();
mongoConnection.update( MONGO_DISPATCHER_COLLECTION, BSON("_id"<<oid) , BSON("$set"<<BSON("started_datetime"<<(long long int)time(NULL))) );
}
}
sleep(2);
}
}
// TODO(jmlvanre): Consider removing default argument for
// `parent_hooks` to force the caller to think about setting them.
inline Try<Subprocess> subprocess(
const std::string& command,
const Subprocess::IO& in = Subprocess::FD(STDIN_FILENO),
const Subprocess::IO& out = Subprocess::FD(STDOUT_FILENO),
const Subprocess::IO& err = Subprocess::FD(STDERR_FILENO),
const Option<std::map<std::string, std::string>>& environment = None(),
const Option<lambda::function<int()>>& setup = None(),
const Option<lambda::function<
pid_t(const lambda::function<int()>&)>>& clone = None(),
const std::vector<Subprocess::Hook>& parent_hooks =
Subprocess::Hook::None())
{
std::vector<std::string> argv = {"sh", "-c", command};
return subprocess(
"sh",
argv,
in,
out,
err,
None(),
environment,
setup,
clone,
parent_hooks);
}
Try<Nothing> MemoryTestHelper::spawn()
{
if (s.isSome()) {
return Error("A subprocess has been spawned already");
}
vector<string> argv;
argv.push_back("test-helper");
argv.push_back(MemoryTestHelper::NAME);
Try<Subprocess> process = subprocess(
getTestHelperPath("test-helper"),
argv,
Subprocess::PIPE(),
Subprocess::PIPE(),
Subprocess::FD(STDERR_FILENO));
if (process.isError()) {
return Error("Failed to spawn a subprocess: " + process.error());
}
s = process.get();
// Wait for the child to inform it has started before returning.
// Otherwise, the user might set the memory limit too earlier, and
// cause the child oom-killed because 'ld' could use a lot of
// memory.
Result<string> read = os::read(s->out().get(), sizeof(STARTED));
if (!read.isSome() || read.get() != string(sizeof(STARTED), STARTED)) {
cleanup();
return Error("Failed to sync with the subprocess");
}
return Nothing();
}
Try<Subprocess> run(
const string& _command,
const Option<string>& rootfs = None())
{
slave::MesosContainerizerLaunch::Flags launchFlags;
CommandInfo command;
command.set_value(_command);
launchFlags.command = JSON::Protobuf(command);
launchFlags.directory = "/tmp";
launchFlags.pipe_read = open("/dev/zero", O_RDONLY);
launchFlags.pipe_write = open("/dev/null", O_WRONLY);
launchFlags.rootfs = rootfs;
vector<string> argv(2);
argv[0] = "mesos-containerizer";
argv[1] = slave::MesosContainerizerLaunch::NAME;
Try<Subprocess> s = subprocess(
path::join(tests::flags.build_dir, "src", "mesos-containerizer"),
argv,
Subprocess::PATH("/dev/null"),
Subprocess::FD(STDOUT_FILENO),
Subprocess::FD(STDERR_FILENO),
launchFlags,
None(),
None(),
lambda::bind(&clone, lambda::_1));
close(launchFlags.pipe_read.get());
close(launchFlags.pipe_write.get());
return s;
}
static void expect2(const char *code, const char *type, const char *output)
{
int ret;
struct context context = { code, type, output };
ret = subprocess(&context);
if (ret != EXIT_SUCCESS)
exit(1);
}
Future<Nothing> untar(const string& file, const string& directory)
{
const vector<string> argv = {
"tar",
"-C",
directory,
"-x",
"-f",
file
};
Try<Subprocess> s = subprocess(
"tar",
argv,
Subprocess::PATH("/dev/null"),
Subprocess::PATH("/dev/null"),
Subprocess::PIPE());
if (s.isError()) {
return Failure("Failed to execute the subprocess: " + s.error());
}
return await(
s.get().status(),
process::io::read(s.get().err().get()))
.then([](const tuple<
Future<Option<int>>,
Future<string>>& t) -> Future<Nothing> {
Future<Option<int>> status = std::get<0>(t);
if (!status.isReady()) {
return Failure(
"Failed to get the exit status of the subprocess: " +
(status.isFailed() ? status.failure() : "discarded"));
}
Future<string> error = std::get<1>(t);
if (!error.isReady()) {
return Failure(
"Failed to read stderr from the subprocess: " +
(error.isFailed() ? error.failure() : "discarded"));
}
if (status->isNone()) {
return Failure("Failed to reap the subprocess");
}
if (status->get() != 0) {
return Failure(
"Unexpected result from the subprocess: " +
WSTRINGIFY(status->get()) +
", stderr='" + error.get() + "'");
}
return Nothing();
});
}
int main(int argc, char **argv) {
char buffer[BUFFER + 1], *cursor, *separator;
int sock;
ssize_t length;
struct sockaddr_nl addr;
progname = argv[0];
if (argc > 1)
subprocess(argv + 1);
memset(&addr, 0, sizeof(addr));
addr.nl_family = AF_NETLINK;
addr.nl_pid = getpid();
addr.nl_groups = 1;
if ((sock = socket(PF_NETLINK, SOCK_DGRAM, NETLINK_KOBJECT_UEVENT)) < 0)
error(EXIT_FAILURE, errno, "socket");
if (bind(sock, (struct sockaddr *) &addr, sizeof(addr)) < 0)
error(EXIT_FAILURE, errno, "bind");
while (1) {
if ((length = recv(sock, &buffer, sizeof(buffer) - 1, 0)) < 0) {
if (errno != EAGAIN && errno != EINTR)
error(EXIT_FAILURE, errno, "recv");
continue;
}
/* Null-terminate the uevent and replace stray newlines with spaces. */
buffer[length] = 0;
for (cursor = buffer; cursor < buffer + length; cursor++)
if (*cursor == '\n')
*cursor = ' ';
if (strlen(buffer) >= length - 1) {
/* No properties; fake a simple environment based on the header. */
if ((cursor = strchr(buffer, '@'))) {
*cursor++ = 0;
printf("ACTION %s\n", buffer);
printf("DEVPATH %s\n", cursor);
}
} else {
/* Ignore header as properties will include ACTION and DEVPATH. */
cursor = buffer;
while (cursor += strlen(cursor) + 1, cursor < buffer + length) {
if ((separator = strchr(cursor, '=')))
*separator = ' ';
puts(cursor);
}
}
putchar('\n');
fflush(stdout);
}
return EXIT_FAILURE;
}
int main (int argc, char* argv[])
{
int cmd_start_index;
int i;
pid_t pid, hardlimit_monitor;
init_default_limits ();
cmd_start_index = parse_args (argc, argv);
/* be safe on the timehard! */
if (limit_timehard < 1 + (int) ceil (limit_time))
limit_timehard = 1 + (int) ceil (limit_time);
/* close inherited file descriptors. Is there a better way? */
for (i = 3; i < (1<<16); i++)
close (i);
pid = fork();
if (pid==0) {
return subprocess (argc - cmd_start_index, argv + cmd_start_index);
}
hardlimit_monitor = fork ();
if (hardlimit_monitor == 0) {
sleep (6*limit_timehard);
/* if I reached here, then the main process is still running,
* upto some race condition possibilitiy */
//fprintf (stderr, "Severe hardlimit (%d) reached. Possibly \
malicious, or overloaded system.\n", 6*limit_timehard);
kill (pid, 9);
return 0;
}
int status;
struct rusage usage ;
/*
* Correctness: Pid dies on its own or, hardlimit_monitor process
* kills it. In both cases, this works, except perhaps the Pid
* process can be called for kill twice.
*/
wait4(pid,&status, 0, &usage); //Wait for child to terminate
kill (hardlimit_monitor, 9);
waitpid (hardlimit_monitor, NULL, 0);
// lets output the limits.
fflush(stderr) ; /* ordering of output of child and parent should be right */
double usertime = (float) (usage.ru_utime.tv_sec) + ((float) usage.ru_utime.tv_usec)/1000000 ;
//fprintf(stderr, "Usertime:
%lf\n", usertime);
Future<Nothing> CurlFetcherPlugin::fetch(
const URI& uri,
const string& directory)
{
// TODO(jieyu): Validate the given URI.
if (!uri.has_path()) {
return Failure("URI path is not specified");
}
if (!os::exists(directory)) {
Try<Nothing> mkdir = os::mkdir(directory);
if (mkdir.isError()) {
return Failure(
"Failed to create directory '" +
directory + "': " + mkdir.error());
}
}
// TODO(jieyu): Allow user to specify the name of the output file.
const string output = path::join(directory, Path(uri.path()).basename());
const vector<string> argv = {
"curl",
"-s", // Don’t show progress meter or error messages.
"-S", // Makes curl show an error message if it fails.
"-L", // Follow HTTP 3xx redirects.
"-w", "%{http_code}", // Display HTTP response code on stdout.
"-o", output, // Write output to the file.
strings::trim(stringify(uri))
};
Try<Subprocess> s = subprocess(
"curl",
argv,
Subprocess::PATH("/dev/null"),
Subprocess::PIPE(),
Subprocess::PIPE());
if (s.isError()) {
return Failure("Failed to exec the curl subprocess: " + s.error());
}
return await(
s.get().status(),
io::read(s.get().out().get()),
io::read(s.get().err().get()))
.then(_fetch);
}
int main(int argc, char** argv)
{
//
//pipe
int fd[2];
int result = pipe(fd);
if (result == -1) {
perror("pipe error!");
exit(1);
}
//fork
pid_t pid = fork();
if (pid == 0) {
//子进程
close(fd[1]);//关闭写通道
subprocess(fd[0]);//监督读取通道
} else if (pid > 0){
//主进程
close(fd[0]);//关闭读通道，使用写通道
} else {
//fork错误
char* errinfo = strerror(errno);
int length = strlen(errinfo)+7;
char* err = malloc(length);
snprintf(err, length, "FORK: %s", errinfo);
perror(err);
free(err);
exit(1);
}
char str[512];
int done = 1;
do {
scanf("%s", str);
if (!strcmp(str, "quit")) {
done = 0;
write(fd[1], str, strlen(str)+1);
wait(NULL);
} else {
//向子进程写入命令
write(fd[1], str, strlen(str)+1);
}
} while (done);
close(fd[1]);//关闭写通道
return 0;
}
/* Process one line of input from virtual channel */
static BOOL
lspci_process_line(const char *line, void *data)
{
char *lspci_command[5] = { "lspci", "-m", "-n", "-v", NULL };
if (!strcmp(line, "LSPCI"))
{
memset(¤t_device, 0, sizeof(current_device));
subprocess(lspci_command, handle_child_line, NULL);
/* Send single dot to indicate end of enumeration */
lspci_send(".\n");
}
else
{
error("lspci protocol error: Invalid line '%s'\n", line);
}
return True;
}
// Launch 'ping' using the given capabilities and user.
Try<Subprocess> ping(
const Set<Capability>& capabilities,
const Option<string>& user = None())
{
CapabilitiesTestHelper helper;
helper.flags.user = user;
helper.flags.capabilities = capabilities::convert(capabilities);
vector<string> argv = {
"test-helper",
CapabilitiesTestHelper::NAME
};
return subprocess(
getTestHelperPath("test-helper"),
argv,
Subprocess::FD(STDIN_FILENO),
Subprocess::FD(STDOUT_FILENO),
Subprocess::FD(STDERR_FILENO),
NO_SETSID,
&helper.flags);
}
Future<Nothing> untar(const string& file, const string& directory)
{
const vector<string> argv = {
"tar",
"-C",
directory,
"-x",
"-f",
file
};
Try<Subprocess> s = subprocess(
"tar",
argv,
Subprocess::PATH("/dev/null"),
Subprocess::PATH("/dev/null"),
Subprocess::PIPE());
if (s.isError()) {
return Failure(
"Failed to create untar subprocess for file '" +
file + "': " + s.error());
}
Owned<Promise<Nothing>> promise(new Promise<Nothing>());
s.get().status()
.onAny([s, file, promise](const Future<Option<int>>& future) {
if (!future.isReady()) {
promise->fail(
"Failed to launch untar subprocess for file '" + file
+ "': " +
(future.isFailed() ? future.failure() : "future discarded"));
return;
}
if (future.get().isNone()) {
promise->fail(
"Failed to get status for untar subprocess for file '" +
file + "'");
return;
}
int status = future.get().get();
if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) {
const string errorMessage =
"Failed to run ntar process for file '" + file + "' (exit code: " +
WSTRINGIFY(status) + ")";
// Read stderr from the process(if any) to add to the failure report.
process::io::read(s.get().err().get())
.onAny([file, promise, errorMessage](const Future<string>& future) {
if (!future.isReady()) {
LOG(WARNING) << "Failed to read stderr from untar process for"
<< "file: '" << file << "': "
<< (future.isFailed() ? future.failure()
: "future discarded");
promise->fail(errorMessage);
} else {
promise->fail(errorMessage + ": " + future.get());
}
});
return;
}
promise->set(Nothing());
});
return promise->future();
}
/*
* Checks whether key is allowed in output of command.
* returns 1 if the key is allowed or 0 otherwise.
*/
static int
user_key_command_allowed2(struct passwd *user_pw, Key *key)
{
FILE *f = NULL;
int r, ok, found_key = 0;
struct passwd *pw;
int i, uid_swapped = 0, ac = 0;
pid_t pid;
char *username = NULL, *key_fp = NULL, *keytext = NULL;
char *tmp, *command = NULL, **av = NULL;
void (*osigchld)(int);
if (options.authorized_keys_command == NULL)
return 0;
if (options.authorized_keys_command_user == NULL) {
error("No user for AuthorizedKeysCommand specified, skipping");
return 0;
}
/*
* NB. all returns later this function should go via "out" to
* ensure the original SIGCHLD handler is restored properly.
*/
osigchld = signal(SIGCHLD, SIG_DFL);
/* Prepare and verify the user for the command */
username = percent_expand(options.authorized_keys_command_user,
"u", user_pw->pw_name, (char *)NULL);
pw = getpwnam(username);
if (pw == NULL) {
error("AuthorizedKeysCommandUser \"%s\" not found: %s",
username, strerror(errno));
goto out;
}
/* Prepare AuthorizedKeysCommand */
if ((key_fp = sshkey_fingerprint(key, options.fingerprint_hash,
SSH_FP_DEFAULT)) == NULL) {
error("%s: sshkey_fingerprint failed", __func__);
goto out;
}
if ((r = sshkey_to_base64(key, &keytext)) != 0) {
error("%s: sshkey_to_base64 failed: %s", __func__, ssh_err(r));
goto out;
}
/* Turn the command into an argument vector */
if (split_argv(options.authorized_keys_command, &ac, &av) != 0) {
error("AuthorizedKeysCommand \"%s\" contains invalid quotes",
command);
goto out;
}
if (ac == 0) {
error("AuthorizedKeysCommand \"%s\" yielded no arguments",
command);
goto out;
}
for (i = 1; i < ac; i++) {
tmp = percent_expand(av[i],
"u", user_pw->pw_name,
"h", user_pw->pw_dir,
"t", sshkey_ssh_name(key),
"f", key_fp,
"k", keytext,
(char *)NULL);
if (tmp == NULL)
fatal("%s: percent_expand failed", __func__);
free(av[i]);
av[i] = tmp;
}
/* Prepare a printable command for logs, etc. */
command = assemble_argv(ac, av);
/*
* If AuthorizedKeysCommand was run without arguments
* then fall back to the old behaviour of passing the
* target username as a single argument.
*/
if (ac == 1) {
av = xreallocarray(av, ac + 2, sizeof(*av));
av[1] = xstrdup(user_pw->pw_name);
av[2] = NULL;
/* Fix up command too, since it is used in log messages */
free(command);
xasprintf(&command, "%s %s", av[0], av[1]);
}
if ((pid = subprocess("AuthorizedKeysCommand", pw, command,
ac, av, &f)) == 0)
goto out;
uid_swapped = 1;
temporarily_use_uid(pw);
ok = check_authkeys_file(f, options.authorized_keys_command, key, pw);
fclose(f);
f = NULL;
if (exited_cleanly(pid, "AuthorizedKeysCommand", command) != 0)
goto out;
/* Read completed successfully */
found_key = ok;
out:
if (f != NULL)
fclose(f);
signal(SIGCHLD, osigchld);
for (i = 0; i < ac; i++)
free(av[i]);
free(av);
if (uid_swapped)
restore_uid();
free(command);
free(username);
free(key_fp);
free(keytext);
return found_key;
}
/*
* Checks whether principal is allowed in output of command.
* returns 1 if the principal is allowed or 0 otherwise.
*/
static int
match_principals_command(struct passwd *user_pw, const struct sshkey *key)
{
const struct sshkey_cert *cert = key->cert;
FILE *f = NULL;
int r, ok, found_principal = 0;
struct passwd *pw;
int i, ac = 0, uid_swapped = 0;
pid_t pid;
char *tmp, *username = NULL, *command = NULL, **av = NULL;
char *ca_fp = NULL, *key_fp = NULL, *catext = NULL, *keytext = NULL;
char serial_s[16];
void (*osigchld)(int);
if (options.authorized_principals_command == NULL)
return 0;
if (options.authorized_principals_command_user == NULL) {
error("No user for AuthorizedPrincipalsCommand specified, "
"skipping");
return 0;
}
/*
* NB. all returns later this function should go via "out" to
* ensure the original SIGCHLD handler is restored properly.
*/
osigchld = signal(SIGCHLD, SIG_DFL);
/* Prepare and verify the user for the command */
username = percent_expand(options.authorized_principals_command_user,
"u", user_pw->pw_name, (char *)NULL);
pw = getpwnam(username);
if (pw == NULL) {
error("AuthorizedPrincipalsCommandUser \"%s\" not found: %s",
username, strerror(errno));
goto out;
}
/* Turn the command into an argument vector */
if (split_argv(options.authorized_principals_command, &ac, &av) != 0) {
error("AuthorizedPrincipalsCommand \"%s\" contains "
"invalid quotes", command);
goto out;
}
if (ac == 0) {
error("AuthorizedPrincipalsCommand \"%s\" yielded no arguments",
command);
goto out;
}
if ((ca_fp = sshkey_fingerprint(cert->signature_key,
options.fingerprint_hash, SSH_FP_DEFAULT)) == NULL) {
error("%s: sshkey_fingerprint failed", __func__);
goto out;
}
if ((key_fp = sshkey_fingerprint(key,
options.fingerprint_hash, SSH_FP_DEFAULT)) == NULL) {
error("%s: sshkey_fingerprint failed", __func__);
goto out;
}
if ((r = sshkey_to_base64(cert->signature_key, &catext)) != 0) {
error("%s: sshkey_to_base64 failed: %s", __func__, ssh_err(r));
goto out;
}
if ((r = sshkey_to_base64(key, &keytext)) != 0) {
error("%s: sshkey_to_base64 failed: %s", __func__, ssh_err(r));
goto out;
}
snprintf(serial_s, sizeof(serial_s), "%llu",
(unsigned long long)cert->serial);
for (i = 1; i < ac; i++) {
tmp = percent_expand(av[i],
"u", user_pw->pw_name,
"h", user_pw->pw_dir,
"t", sshkey_ssh_name(key),
"T", sshkey_ssh_name(cert->signature_key),
"f", key_fp,
"F", ca_fp,
"k", keytext,
"K", catext,
"i", cert->key_id,
"s", serial_s,
(char *)NULL);
if (tmp == NULL)
fatal("%s: percent_expand failed", __func__);
free(av[i]);
av[i] = tmp;
}
/* Prepare a printable command for logs, etc. */
command = assemble_argv(ac, av);
if ((pid = subprocess("AuthorizedPrincipalsCommand", pw, command,
ac, av, &f)) == 0)
goto out;
uid_swapped = 1;
temporarily_use_uid(pw);
ok = process_principals(f, NULL, pw, cert);
fclose(f);
f = NULL;
if (exited_cleanly(pid, "AuthorizedPrincipalsCommand", command) != 0)
goto out;
/* Read completed successfully */
found_principal = ok;
out:
if (f != NULL)
fclose(f);
signal(SIGCHLD, osigchld);
for (i = 0; i < ac; i++)
free(av[i]);
free(av);
if (uid_swapped)
restore_uid();
free(command);
free(username);
free(ca_fp);
free(key_fp);
free(catext);
free(keytext);
return found_principal;
}
pid_t launchTaskPosix(
const CommandInfo& command,
const string& launcherDir,
const Environment& environment,
const Option<string>& user,
const Option<string>& rootfs,
const Option<string>& sandboxDirectory,
const Option<string>& workingDirectory,
const Option<CapabilityInfo>& capabilities)
{
// Prepare the flags to pass to the launch process.
MesosContainerizerLaunch::Flags launchFlags;
ContainerLaunchInfo launchInfo;
launchInfo.mutable_command()->CopyFrom(command);
if (rootfs.isSome()) {
// The command executor is responsible for chrooting into the
// root filesystem and changing the user before exec-ing the
// user process.
#ifdef __linux__
if (geteuid() != 0) {
ABORT("The command executor requires root with rootfs");
}
// Ensure that mount namespace of the executor is not affected by
// changes in its task's namespace induced by calling `pivot_root`
// as part of the task setup in mesos-containerizer binary.
launchFlags.unshare_namespace_mnt = true;
#else
ABORT("Not expecting root volume with non-linux platform");
#endif // __linux__
launchInfo.set_rootfs(rootfs.get());
CHECK_SOME(sandboxDirectory);
launchInfo.set_working_directory(workingDirectory.isSome()
? workingDirectory.get()
: sandboxDirectory.get());
// TODO(jieyu): If the task has a rootfs, the executor itself will
// be running as root. Its sandbox is owned by root as well. In
// order for the task to be able to access to its sandbox, we need
// to make sure the owner of the sandbox is 'user'. However, this
// is still a workaround. The owner of the files downloaded by the
// fetcher is still not correct (i.e., root).
if (user.isSome()) {
// NOTE: We only chown the sandbox directory (non-recursively).
Try<Nothing> chown = os::chown(user.get(), os::getcwd(), false);
if (chown.isError()) {
ABORT("Failed to chown sandbox to user " +
user.get() + ": " + chown.error());
}
}
}
launchInfo.mutable_environment()->CopyFrom(environment);
if (user.isSome()) {
launchInfo.set_user(user.get());
}
if (capabilities.isSome()) {
launchInfo.mutable_capabilities()->CopyFrom(capabilities.get());
}
launchFlags.launch_info = JSON::protobuf(launchInfo);
string commandString = strings::format(
"%s %s %s",
path::join(launcherDir, MESOS_CONTAINERIZER),
MesosContainerizerLaunch::NAME,
stringify(launchFlags)).get();
// Fork the child using launcher.
vector<string> argv(2);
argv[0] = MESOS_CONTAINERIZER;
argv[1] = MesosContainerizerLaunch::NAME;
Try<Subprocess> s = subprocess(
path::join(launcherDir, MESOS_CONTAINERIZER),
argv,
Subprocess::FD(STDIN_FILENO),
Subprocess::FD(STDOUT_FILENO),
Subprocess::FD(STDERR_FILENO),
&launchFlags,
None(),
None(),
{},
{Subprocess::ChildHook::SETSID()});
if (s.isError()) {
ABORT("Failed to launch '" + commandString + "': " + s.error());
}
cout << commandString << endl;
return s->pid();
}
/*
* Checks whether principal is allowed in output of command.
* returns 1 if the principal is allowed or 0 otherwise.
*/
static int
match_principals_command(struct passwd *user_pw, struct sshkey_cert *cert)
{
FILE *f = NULL;
int ok, found_principal = 0;
struct passwd *pw;
int i, ac = 0, uid_swapped = 0;
pid_t pid;
char *tmp, *username = NULL, *command = NULL, **av = NULL;
void (*osigchld)(int);
if (options.authorized_principals_command == NULL)
return 0;
if (options.authorized_principals_command_user == NULL) {
error("No user for AuthorizedPrincipalsCommand specified, "
"skipping");
return 0;
}
/*
* NB. all returns later this function should go via "out" to
* ensure the original SIGCHLD handler is restored properly.
*/
#ifndef WIN32_FIXME
// PRAGMA:TODO
osigchld = signal(SIGCHLD, SIG_DFL);
#endif
/* Prepare and verify the user for the command */
username = percent_expand(options.authorized_principals_command_user,
"u", user_pw->pw_name, (char *)NULL);
pw = getpwnam(username);
if (pw == NULL) {
error("AuthorizedPrincipalsCommandUser \"%s\" not found: %s",
username, strerror(errno));
goto out;
}
/* Turn the command into an argument vector */
if (split_argv(options.authorized_principals_command, &ac, &av) != 0) {
error("AuthorizedPrincipalsCommand \"%s\" contains "
"invalid quotes", command);
goto out;
}
if (ac == 0) {
error("AuthorizedPrincipalsCommand \"%s\" yielded no arguments",
command);
goto out;
}
for (i = 1; i < ac; i++) {
tmp = percent_expand(av[i],
"u", user_pw->pw_name,
"h", user_pw->pw_dir,
(char *)NULL);
if (tmp == NULL)
fatal("%s: percent_expand failed", __func__);
free(av[i]);
av[i] = tmp;
}
/* Prepare a printable command for logs, etc. */
command = assemble_argv(ac, av);
if ((pid = subprocess("AuthorizedPrincipalsCommand", pw, command,
ac, av, &f)) == 0)
goto out;
uid_swapped = 1;
temporarily_use_uid(pw);
ok = process_principals(f, NULL, pw, cert);
if (exited_cleanly(pid, "AuthorizedPrincipalsCommand", command) != 0)
goto out;
/* Read completed successfully */
found_principal = ok;
out:
if (f != NULL)
fclose(f);
signal(SIGCHLD, osigchld);
for (i = 0; i < ac; i++)
free(av[i]);
free(av);
if (uid_swapped)
restore_uid();
free(command);
free(username);
return found_principal;
}
Future<Nothing> CopyFetcherPlugin::fetch(
const URI& uri,
const string& directory) const
{
// TODO(jojy): Validate the given URI.
if (!uri.has_path()) {
return Failure("URI path is not specified");
}
// TODO(jojy): Verify that the path is a file.
Try<Nothing> mkdir = os::mkdir(directory);
if (mkdir.isError()) {
return Failure(
"Failed to create directory '" +
directory + "': " + mkdir.error());
}
VLOG(1) << "Copying '" << uri.path() << "' to '" << directory << "'";
const vector<string> argv = {"cp", "-a", uri.path(), directory};
Try<Subprocess> s = subprocess(
"cp",
argv,
Subprocess::PATH(os::DEV_NULL),
Subprocess::PIPE(),
Subprocess::PIPE());
if (s.isError()) {
return Failure("Failed to exec the copy subprocess: " + s.error());
}
return await(
s.get().status(),
io::read(s.get().out().get()),
io::read(s.get().err().get()))
.then([](const tuple<
Future<Option<int>>,
Future<string>,
Future<string>>& t) -> Future<Nothing> {
Future<Option<int>> status = std::get<0>(t);
if (!status.isReady()) {
return Failure(
"Failed to get the exit status of the copy subprocess: " +
(status.isFailed() ? status.failure() : "discarded"));
}
if (status->isNone()) {
return Failure("Failed to reap the copy subprocess");
}
if (status->get() != 0) {
Future<string> error = std::get<2>(t);
if (!error.isReady()) {
return Failure(
"Failed to perform 'copy'. Reading stderr failed: " +
(error.isFailed() ? error.failure() : "discarded"));
}
return Failure("Failed to perform 'copy': " + error.get());
}
return Nothing();
});
}
Future<Nothing> CurlFetcherPlugin::fetch(
const URI& uri,
const string& directory)
{
// TODO(jieyu): Validate the given URI.
if (!uri.has_path()) {
return Failure("URI path is not specified");
}
Try<Nothing> mkdir = os::mkdir(directory);
if (mkdir.isError()) {
return Failure(
"Failed to create directory '" +
directory + "': " + mkdir.error());
}
// TODO(jieyu): Allow user to specify the name of the output file.
const string output = path::join(directory, Path(uri.path()).basename());
const vector<string> argv = {
"curl",
"-s", // Don’t show progress meter or error messages.
"-S", // Makes curl show an error message if it fails.
"-L", // Follow HTTP 3xx redirects.
"-w", "%{http_code}", // Display HTTP response code on stdout.
"-o", output, // Write output to the file.
strings::trim(stringify(uri))
};
Try<Subprocess> s = subprocess(
"curl",
argv,
Subprocess::PATH("/dev/null"),
Subprocess::PIPE(),
Subprocess::PIPE());
if (s.isError()) {
return Failure("Failed to exec the curl subprocess: " + s.error());
}
return await(
s.get().status(),
io::read(s.get().out().get()),
io::read(s.get().err().get()))
.then([](const tuple<
Future<Option<int>>,
Future<string>,
Future<string>>& t) -> Future<Nothing> {
Future<Option<int>> status = std::get<0>(t);
if (!status.isReady()) {
return Failure(
"Failed to get the exit status of the curl subprocess: " +
(status.isFailed() ? status.failure() : "discarded"));
}
if (status->isNone()) {
return Failure("Failed to reap the curl subprocess");
}
if (status->get() != 0) {
Future<string> error = std::get<2>(t);
if (!error.isReady()) {
return Failure(
"Failed to perform 'curl'. Reading stderr failed: " +
(error.isFailed() ? error.failure() : "discarded"));
}
return Failure("Failed to perform 'curl': " + error.get());
}
Future<string> output = std::get<1>(t);
if (!output.isReady()) {
return Failure(
"Failed to read stdout from 'curl': " +
(output.isFailed() ? output.failure() : "discarded"));
}
// Parse the output and get the HTTP response code.
Try<int> code = numify<int>(output.get());
if (code.isError()) {
return Failure("Unexpected output from 'curl': " + output.get());
}
if (code.get() != http::Status::OK) {
return Failure(
"Unexpected HTTP response code: " +
http::Status::string(code.get()));
}
return Nothing();
});
}
int main(int argc, char** argv)
{
utils::mpi_world mpi_world(argc, argv);
const int mpi_rank = MPI::COMM_WORLD.Get_rank();
const int mpi_size = MPI::COMM_WORLD.Get_size();
try {
if (getoptions(argc, argv) != 0)
return 1;
if (command.empty())
throw std::runtime_error("no command?");
typedef MapReduce map_reduce_type;
typedef map_reduce_type::id_type id_type;
typedef map_reduce_type::value_type value_type;
typedef map_reduce_type::queue_type queue_type;
typedef map_reduce_type::queue_id_type queue_id_type;
typedef map_reduce_type::subprocess_type subprocess_type;
typedef Mapper mapper_type;
typedef Reducer reducer_type;
typedef Consumer consumer_type;
typedef Merger merger_type;
if (mpi_rank == 0) {
subprocess_type subprocess(command);
queue_type queue_is(mpi_size);
queue_type queue_send(1);
queue_id_type queue_id;
queue_type queue_recv;
const bool flush_output = (output_file == "-"
|| (boost::filesystem::exists(output_file)
&& ! boost::filesystem::is_regular_file(output_file)));
utils::compress_istream is(input_file, 1024 * 1024);
utils::compress_ostream os(output_file, 1024 * 1024 * (! flush_output));
boost::thread consumer(consumer_type(queue_is, is));
boost::thread merger(merger_type(queue_recv, os, mpi_size));
boost::thread mapper(mapper_type(queue_send, queue_id, subprocess));
boost::thread reducer(reducer_type(queue_recv, queue_id, subprocess));
typedef utils::mpi_ostream ostream_type;
typedef utils::mpi_istream_simple istream_type;
typedef boost::shared_ptr<ostream_type> ostream_ptr_type;
typedef boost::shared_ptr<istream_type> istream_ptr_type;
typedef std::vector<ostream_ptr_type, std::allocator<ostream_ptr_type> > ostream_ptr_set_type;
typedef std::vector<istream_ptr_type, std::allocator<istream_ptr_type> > istream_ptr_set_type;
ostream_ptr_set_type ostream(mpi_size);
istream_ptr_set_type istream(mpi_size);
for (int rank = 1; rank < mpi_size; ++ rank) {
ostream[rank].reset(new ostream_type(rank, line_tag, 4096));
istream[rank].reset(new istream_type(rank, line_tag, 4096));
}
std::string line;
value_type value(0, std::string());
value_type value_recv(0, std::string());
int non_found_iter = 0;
while (value.first != id_type(-1)) {
bool found = false;
for (int rank = 1; rank < mpi_size && value.first != id_type(-1); ++ rank)
if (ostream[rank]->test() && queue_is.pop(value, true) && value.first != id_type(-1)) {
ostream[rank]->write(utils::lexical_cast<std::string>(value.first) + ' ' + value.second);
found = true;
}
if (queue_send.empty() && queue_is.pop(value, true) && value.first != id_type(-1)) {
queue_send.push(value);
found = true;
}
// reduce...
for (int rank = 1; rank < mpi_size; ++ rank)
if (istream[rank] && istream[rank]->test()) {
if (istream[rank]->read(line)) {
tokenize(line, value_recv);
queue_recv.push_swap(value_recv);
} else {
queue_recv.push(std::make_pair(id_type(-1), std::string()));
istream[rank].reset();
}
found = true;
}
non_found_iter = loop_sleep(found, non_found_iter);
}
bool terminated = false;
for (;;) {
bool found = false;
if (! terminated && queue_send.push(std::make_pair(id_type(-1), std::string()), true)) {
terminated = true;
found = true;
}
// termination...
for (int rank = 1; rank < mpi_size; ++ rank)
if (ostream[rank] && ostream[rank]->test()) {
if (! ostream[rank]->terminated())
ostream[rank]->terminate();
else
ostream[rank].reset();
found = true;
}
// reduce...
for (int rank = 1; rank < mpi_size; ++ rank)
if (istream[rank] && istream[rank]->test()) {
if (istream[rank]->read(line)) {
tokenize(line, value_recv);
queue_recv.push_swap(value_recv);
} else {
queue_recv.push(std::make_pair(id_type(-1), std::string()));
istream[rank].reset();
}
found = true;
}
// termination condition!
if (std::count(istream.begin(), istream.end(), istream_ptr_type()) == mpi_size
&& std::count(ostream.begin(), ostream.end(), ostream_ptr_type()) == mpi_size
&& terminated) break;
non_found_iter = loop_sleep(found, non_found_iter);
}
mapper.join();
reducer.join();
consumer.join();
merger.join();
} else {
subprocess_type subprocess(command);
queue_type queue_send(1);
queue_id_type queue_id;
queue_type queue_recv;
boost::thread mapper(mapper_type(queue_send, queue_id, subprocess));
boost::thread reducer(reducer_type(queue_recv, queue_id, subprocess));
typedef utils::mpi_istream istream_type;
typedef utils::mpi_ostream_simple ostream_type;
boost::shared_ptr<istream_type> is(new istream_type(0, line_tag, 4096));
boost::shared_ptr<ostream_type> os(new ostream_type(0, line_tag, 4096));
std::string line;
value_type value;
bool terminated = false;
int non_found_iter = 0;
for (;;) {
bool found = false;
if (is && is->test() && queue_send.empty()) {
if (is->read(line))
tokenize(line, value);
else {
value.first = id_type(-1);
value.second = std::string();
is.reset();
}
queue_send.push_swap(value);
found = true;
}
if (! terminated) {
if (os && os->test() && queue_recv.pop_swap(value, true)) {
if (value.first == id_type(-1))
terminated = true;
else
os->write(utils::lexical_cast<std::string>(value.first) + ' ' + value.second);
found = true;
}
} else {
if (os && os->test()) {
if (! os->terminated())
os->terminate();
else
os.reset();
found = true;
}
}
if (! is && ! os) break;
non_found_iter = loop_sleep(found, non_found_iter);
}
mapper.join();
reducer.join();
}
// synchronize...
if (mpi_rank == 0) {
std::vector<MPI::Request, std::allocator<MPI::Request> > request_recv(mpi_size);
std::vector<MPI::Request, std::allocator<MPI::Request> > request_send(mpi_size);
std::vector<bool, std::allocator<bool> > terminated_recv(mpi_size, false);
std::vector<bool, std::allocator<bool> > terminated_send(mpi_size, false);
terminated_recv[0] = true;
terminated_send[0] = true;
for (int rank = 1; rank != mpi_size; ++ rank) {
request_recv[rank] = MPI::COMM_WORLD.Irecv(0, 0, MPI::INT, rank, notify_tag);
request_send[rank] = MPI::COMM_WORLD.Isend(0, 0, MPI::INT, rank, notify_tag);
}
int non_found_iter = 0;
for (;;) {
bool found = false;
for (int rank = 1; rank != mpi_size; ++ rank)
if (! terminated_recv[rank] && request_recv[rank].Test()) {
terminated_recv[rank] = true;
found = true;
}
for (int rank = 1; rank != mpi_size; ++ rank)
if (! terminated_send[rank] && request_send[rank].Test()) {
terminated_send[rank] = true;
found = true;
}
if (std::count(terminated_send.begin(), terminated_send.end(), true) == mpi_size
&& std::count(terminated_recv.begin(), terminated_recv.end(), true) == mpi_size) break;
non_found_iter = loop_sleep(found, non_found_iter);
}
} else {
MPI::Request request_send = MPI::COMM_WORLD.Isend(0, 0, MPI::INT, 0, notify_tag);
MPI::Request request_recv = MPI::COMM_WORLD.Irecv(0, 0, MPI::INT, 0, notify_tag);
bool terminated_send = false;
bool terminated_recv = false;
int non_found_iter = 0;
for (;;) {
bool found = false;
if (! terminated_send && request_send.Test()) {
terminated_send = true;
found = true;
}
if (! terminated_recv && request_recv.Test()) {
terminated_recv = true;
found = true;
}
if (terminated_send && terminated_recv) break;
non_found_iter = loop_sleep(found, non_found_iter);
}
}
}
catch (const std::exception& err) {
std::cerr << "error: " << argv[0] << " "<< err.what() << std::endl;
MPI::COMM_WORLD.Abort(1);
return 1;
}
return 0;
}
int main (int argc, char* argv[]) {
int cmd_start_index;
int i;
pid_t pid, hardlimit_monitor;
init_default_limits ();
cmd_start_index = parse_args (argc, argv);
if (limit_timehard < 1 + (int) (limit_time))
limit_timehard = 1 + (int) (limit_time);
/* close inherited file descriptors. Is there a better way? */
for (i = 3; i < (1<<16); i++)
close (i);
pid = fork();
if (pid==0) {
return subprocess (argc - cmd_start_index, argv + cmd_start_index);
}
hardlimit_monitor = fork ();
if (hardlimit_monitor == 0) {
sleep (6*limit_timehard);
kill (pid, 9);
return 0;
}
int status;
struct rusage usage ;
wait4(pid,&status, 0, &usage); //Wait for child to terminate
kill (hardlimit_monitor, 9);
waitpid (hardlimit_monitor, NULL, 0);
fflush(stderr) ; /* ordering of output of child and parent should be right */
double usertime = (float) (usage.ru_utime.tv_sec) + ((float) usage.ru_utime.tv_usec)/1000000 ;
double systime = (float) (usage.ru_stime.tv_sec) + ((float)usage.ru_stime.tv_usec)/1000000 ;
if(WIFSIGNALED(status)) {
int signal = WTERMSIG(status);
#define die(s) { fprintf(stderr,s) ; exit(0) ; }
if(signal==SIGXCPU) die("TLE Time limit exceeded (H)\n");
if(signal==SIGFPE) die("FPE Floating point exception\n");
if(signal==SIGILL) die("ILL Illegal instruction\n");
if(signal==SIGSEGV) die("SEG Segmentation fault\n");
if(signal==SIGABRT) die("ABRT Aborted (got SIGABRT)\n");
if(signal==SIGBUS) die("BUS Bus error (bad memory access)\n");
if(signal==SIGSYS) die("SYS Invalid system call\n");
if(signal==SIGXFSZ) die("XFSZ Output file too large\n");
if(signal==SIGKILL) die("KILL Your program was killed (probably because of excessive memory usage)\n");
die("UNK Unknown error, possibly your program does not return 0, or maybe its some fault of ours!");
}
if (usertime + systime > limit_time) die("TLE Time Limit exceeded\n") ;
if (!WIFEXITED(status)) {
fprintf (stderr,"EXIT Program exited abnormally. This could be due to excessive memory usage, or any runtime error that is impossible to determine.\n");
exit (0);
}
if (WEXITSTATUS(status) != 0) {
fprintf (stderr,"EXIT Program did not return 0\n", WEXITSTATUS(status));
exit (0);
}
return 0;
}
static Future<string> launch(
const string& path,
const vector<string>& argv)
{
Try<Subprocess> s = subprocess(
path,
argv,
Subprocess::PATH("/dev/null"),
Subprocess::PIPE(),
Subprocess::PIPE());
string command = strings::join(
", ",
path,
strings::join(", ", argv));
if (s.isError()) {
return Failure(
"Failed to execute the subprocess '" + command + "': " + s.error());
}
return await(
s.get().status(),
process::io::read(s.get().out().get()),
process::io::read(s.get().err().get()))
.then([command](const tuple<
Future<Option<int>>,
Future<string>,
Future<string>>& t) -> Future<string> {
Future<Option<int>> status = std::get<0>(t);
if (!status.isReady()) {
return Failure(
"Failed to get the exit status of the subprocess: " +
(status.isFailed() ? status.failure() : "discarded"));
}
if (status->isNone()) {
return Failure("Failed to reap the subprocess");
}
if (status->get() != 0) {
Future<string> error = std::get<2>(t);
if (!error.isReady()) {
return Failure(
"Unexpected result from the subprocess: " +
WSTRINGIFY(status->get()) + ", stderr='" +
error.get() + "'");
}
return Failure("Subprocess '" + command + "' failed: " + error.get());
}
Future<string> output = std::get<1>(t);
if (!output.isReady()) {
return Failure(
"Failed to read stdout from '" + command + "': " +
(output.isFailed() ? output.failure() : "discarded"));
}
return output;
});
}