int main(int argc, char **argv) {
open_log_files();
assert_valid_setup(argv[0]);
int operation;
int ret = validate_arguments(argc, argv, &operation);
if (ret != 0) {
flush_and_close_log_files();
return ret;
}
int exit_code = 0;
switch (operation) {
case CHECK_SETUP:
//we already did this
exit_code = 0;
break;
case MOUNT_CGROUPS:
exit_code = 0;
while (optind < argc && exit_code == 0) {
exit_code = mount_cgroup(argv[optind++], cmd_input.cgroups_hierarchy);
}
break;
case TRAFFIC_CONTROL_MODIFY_STATE:
exit_code = traffic_control_modify_state(cmd_input.traffic_control_command_file);
break;
case TRAFFIC_CONTROL_READ_STATE:
exit_code = traffic_control_read_state(cmd_input.traffic_control_command_file);
break;
case TRAFFIC_CONTROL_READ_STATS:
exit_code = traffic_control_read_stats(cmd_input.traffic_control_command_file);
break;
case RUN_DOCKER:
exit_code = run_docker(cmd_input.docker_command_file);
break;
case RUN_AS_USER_INITIALIZE_CONTAINER:
exit_code = set_user(cmd_input.run_as_user_name);
if (exit_code != 0) {
break;
}
exit_code = initialize_app(cmd_input.yarn_user_name,
cmd_input.app_id,
cmd_input.cred_file,
extract_values(cmd_input.local_dirs),
extract_values(cmd_input.log_dirs),
argv + optind);
break;
case RUN_AS_USER_LAUNCH_DOCKER_CONTAINER:
if (cmd_input.traffic_control_command_file != NULL) {
//apply tc rules before switching users and launching the container
exit_code = traffic_control_modify_state(cmd_input.traffic_control_command_file);
if( exit_code != 0) {
//failed to apply tc rules - break out before launching the container
break;
}
}
exit_code = set_user(cmd_input.run_as_user_name);
if (exit_code != 0) {
break;
}
exit_code = launch_docker_container_as_user(cmd_input.yarn_user_name,
cmd_input.app_id,
cmd_input.container_id,
cmd_input.current_dir,
cmd_input.script_file,
cmd_input.cred_file,
cmd_input.pid_file,
extract_values(cmd_input.local_dirs),
extract_values(cmd_input.log_dirs),
cmd_input.docker_command_file,
cmd_input.resources_key,
cmd_input.resources_values);
break;
case RUN_AS_USER_LAUNCH_CONTAINER:
if (cmd_input.traffic_control_command_file != NULL) {
//apply tc rules before switching users and launching the container
exit_code = traffic_control_modify_state(cmd_input.traffic_control_command_file);
if( exit_code != 0) {
//failed to apply tc rules - break out before launching the container
break;
}
}
exit_code = set_user(cmd_input.run_as_user_name);
if (exit_code != 0) {
break;
}
exit_code = launch_container_as_user(cmd_input.yarn_user_name,
cmd_input.app_id,
cmd_input.container_id,
cmd_input.current_dir,
cmd_input.script_file,
cmd_input.cred_file,
cmd_input.pid_file,
extract_values(cmd_input.local_dirs),
extract_values(cmd_input.log_dirs),
cmd_input.resources_key,
cmd_input.resources_values);
free(cmd_input.resources_key);
free(cmd_input.resources_value);
free(cmd_input.resources_values);
break;
case RUN_AS_USER_SIGNAL_CONTAINER:
exit_code = set_user(cmd_input.run_as_user_name);
if (exit_code != 0) {
break;
}
exit_code = signal_container_as_user(cmd_input.yarn_user_name,
cmd_input.container_pid,
cmd_input.signal);
break;
case RUN_AS_USER_DELETE:
exit_code = set_user(cmd_input.run_as_user_name);
if (exit_code != 0) {
break;
}
exit_code = delete_as_user(cmd_input.yarn_user_name,
cmd_input.dir_to_be_deleted,
argv + optind);
break;
}
flush_and_close_log_files();
return exit_code;
}
int main(int argc, char **argv) {
int invalid_args = 0;
int do_check_setup = 0;
int do_mount_cgroups = 0;
LOGFILE = stdout;
ERRORFILE = stderr;
if (argc > 1) {
if (strcmp("--mount-cgroups", argv[1]) == 0) {
do_mount_cgroups = 1;
}
}
// Minimum number of arguments required to run
// the std. container-executor commands is 4
// 4 args not needed for checksetup option
if (argc < 4 && !do_mount_cgroups) {
invalid_args = 1;
if (argc == 2) {
const char *arg1 = argv[1];
if (strcmp("--checksetup", arg1) == 0) {
invalid_args = 0;
do_check_setup = 1;
}
}
}
if (invalid_args != 0) {
display_usage(stdout);
return INVALID_ARGUMENT_NUMBER;
}
int command;
const char * app_id = NULL;
const char * container_id = NULL;
const char * cred_file = NULL;
const char * script_file = NULL;
const char * current_dir = NULL;
const char * pid_file = NULL;
int exit_code = 0;
char * dir_to_be_deleted = NULL;
char *executable_file = get_executable();
char *orig_conf_file = HADOOP_CONF_DIR "/" CONF_FILENAME;
char *conf_file = resolve_config_path(orig_conf_file, argv[0]);
char *local_dirs, *log_dirs;
char *resources, *resources_key, *resources_value;
if (conf_file == NULL) {
fprintf(ERRORFILE, "Configuration file %s not found.\n", orig_conf_file);
exit(INVALID_CONFIG_FILE);
}
if (check_configuration_permissions(conf_file) != 0) {
exit(INVALID_CONFIG_FILE);
}
read_config(conf_file);
free(conf_file);
// look up the node manager group in the config file
char *nm_group = get_value(NM_GROUP_KEY);
if (nm_group == NULL) {
fprintf(ERRORFILE, "Can't get configured value for %s.\n", NM_GROUP_KEY);
exit(INVALID_CONFIG_FILE);
}
struct group *group_info = getgrnam(nm_group);
if (group_info == NULL) {
fprintf(ERRORFILE, "Can't get group information for %s - %s.\n", nm_group,
strerror(errno));
fflush(LOGFILE);
exit(INVALID_CONFIG_FILE);
}
set_nm_uid(getuid(), group_info->gr_gid);
// if we are running from a setuid executable, make the real uid root
setuid(0);
// set the real and effective group id to the node manager group
setgid(group_info->gr_gid);
if (check_executor_permissions(executable_file) != 0) {
fprintf(ERRORFILE, "Invalid permissions on container-executor binary.\n");
return INVALID_CONTAINER_EXEC_PERMISSIONS;
}
if (do_check_setup != 0) {
// basic setup checks done
// verified configs available and valid
// verified executor permissions
return 0;
}
if (do_mount_cgroups) {
optind++;
char *hierarchy = argv[optind++];
int result = 0;
while (optind < argc && result == 0) {
result = mount_cgroup(argv[optind++], hierarchy);
}
return result;
}
//checks done for user name
if (argv[optind] == NULL) {
fprintf(ERRORFILE, "Invalid user name.\n");
return INVALID_USER_NAME;
}
int ret = set_user(argv[optind]);
if (ret != 0) {
return ret;
}
// this string is used for building pathnames, the
// process management is done based on the 'user_detail'
// global, which was set by 'set_user()' above
optind = optind + 1;
char *yarn_user_name = argv[optind];
if (yarn_user_name == NULL) {
fprintf(ERRORFILE, "Invalid yarn user name.\n");
return INVALID_USER_NAME;
}
optind = optind + 1;
command = atoi(argv[optind++]);
fprintf(LOGFILE, "main : command provided %d\n",command);
fprintf(LOGFILE, "main : user is %s\n", user_detail->pw_name);
fprintf(LOGFILE, "main : requested yarn user is %s\n", yarn_user_name);
fflush(LOGFILE);
switch (command) {
case INITIALIZE_CONTAINER:
if (argc < 9) {
fprintf(ERRORFILE, "Too few arguments (%d vs 9) for initialize container\n",
argc);
fflush(ERRORFILE);
return INVALID_ARGUMENT_NUMBER;
}
app_id = argv[optind++];
cred_file = argv[optind++];
local_dirs = argv[optind++];// good local dirs as a comma separated list
log_dirs = argv[optind++];// good log dirs as a comma separated list
exit_code = initialize_app(yarn_user_name, app_id, cred_file,
extract_values(local_dirs),
extract_values(log_dirs), argv + optind);
break;
case LAUNCH_CONTAINER:
if (argc != 13) {
fprintf(ERRORFILE, "Wrong number of arguments (%d vs 13) for launch container\n",
argc);
fflush(ERRORFILE);
return INVALID_ARGUMENT_NUMBER;
}
app_id = argv[optind++];
container_id = argv[optind++];
current_dir = argv[optind++];
script_file = argv[optind++];
cred_file = argv[optind++];
pid_file = argv[optind++];
local_dirs = argv[optind++];// good local dirs as a comma separated list
log_dirs = argv[optind++];// good log dirs as a comma separated list
resources = argv[optind++];// key,value pair describing resources
char *resources_key = malloc(strlen(resources));
char *resources_value = malloc(strlen(resources));
if (get_kv_key(resources, resources_key, strlen(resources)) < 0 ||
get_kv_value(resources, resources_value, strlen(resources)) < 0) {
fprintf(ERRORFILE, "Invalid arguments for cgroups resources: %s",
resources);
fflush(ERRORFILE);
free(resources_key);
free(resources_value);
return INVALID_ARGUMENT_NUMBER;
}
char** resources_values = extract_values(resources_value);
exit_code = launch_container_as_user(yarn_user_name, app_id,
container_id, current_dir, script_file, cred_file,
pid_file, extract_values(local_dirs),
extract_values(log_dirs), resources_key,
resources_values);
free(resources_key);
free(resources_value);
break;
case SIGNAL_CONTAINER:
if (argc != 6) {
fprintf(ERRORFILE, "Wrong number of arguments (%d vs 6) for " \
"signal container\n", argc);
fflush(ERRORFILE);
return INVALID_ARGUMENT_NUMBER;
} else {
char* end_ptr = NULL;
char* option = argv[optind++];
int container_pid = strtol(option, &end_ptr, 10);
if (option == end_ptr || *end_ptr != '\0') {
fprintf(ERRORFILE, "Illegal argument for container pid %s\n", option);
fflush(ERRORFILE);
return INVALID_ARGUMENT_NUMBER;
}
option = argv[optind++];
int signal = strtol(option, &end_ptr, 10);
if (option == end_ptr || *end_ptr != '\0') {
fprintf(ERRORFILE, "Illegal argument for signal %s\n", option);
fflush(ERRORFILE);
return INVALID_ARGUMENT_NUMBER;
}
exit_code = signal_container_as_user(yarn_user_name, container_pid, signal);
}
break;
case DELETE_AS_USER:
dir_to_be_deleted = argv[optind++];
exit_code= delete_as_user(yarn_user_name, dir_to_be_deleted,
argv + optind);
break;
default:
fprintf(ERRORFILE, "Invalid command %d not supported.",command);
fflush(ERRORFILE);
exit_code = INVALID_COMMAND_PROVIDED;
}
fclose(LOGFILE);
fclose(ERRORFILE);
return exit_code;
}
int main(int argc, char **argv) {
int invalid_args = 0;
int do_check_setup = 0;
LOGFILE = stdout;
ERRORFILE = stderr;
// Minimum number of arguments required to run
// the std. container-executor commands is 4
// 4 args not needed for checksetup option
if (argc < 4) {
invalid_args = 1;
if (argc == 2) {
const char *arg1 = argv[1];
if (strcmp("--checksetup", arg1) == 0) {
invalid_args = 0;
do_check_setup = 1;
}
}
}
if (invalid_args != 0) {
display_usage(stdout);
return INVALID_ARGUMENT_NUMBER;
}
int command;
const char * app_id = NULL;
const char * container_id = NULL;
const char * cred_file = NULL;
const char * script_file = NULL;
const char * current_dir = NULL;
const char * pid_file = NULL;
int exit_code = 0;
char * dir_to_be_deleted = NULL;
char *executable_file = get_executable();
char *orig_conf_file = STRINGIFY(HADOOP_CONF_DIR) "/" CONF_FILENAME;
char *conf_file = realpath(orig_conf_file, NULL);
if (conf_file == NULL) {
fprintf(ERRORFILE, "Configuration file %s not found.\n", orig_conf_file);
exit(INVALID_CONFIG_FILE);
}
if (check_configuration_permissions(conf_file) != 0) {
exit(INVALID_CONFIG_FILE);
}
read_config(conf_file);
free(conf_file);
// look up the node manager group in the config file
char *nm_group = get_value(NM_GROUP_KEY);
if (nm_group == NULL) {
fprintf(ERRORFILE, "Can't get configured value for %s.\n", NM_GROUP_KEY);
exit(INVALID_CONFIG_FILE);
}
struct group *group_info = getgrnam(nm_group);
if (group_info == NULL) {
fprintf(ERRORFILE, "Can't get group information for %s - %s.\n", nm_group,
strerror(errno));
fflush(LOGFILE);
exit(INVALID_CONFIG_FILE);
}
set_nm_uid(getuid(), group_info->gr_gid);
// if we are running from a setuid executable, make the real uid root
setuid(0);
// set the real and effective group id to the node manager group
setgid(group_info->gr_gid);
if (check_executor_permissions(executable_file) != 0) {
fprintf(ERRORFILE, "Invalid permissions on container-executor binary.\n");
return INVALID_CONTAINER_EXEC_PERMISSIONS;
}
if (do_check_setup != 0) {
// basic setup checks done
// verified configs available and valid
// verified executor permissions
return 0;
}
//checks done for user name
if (argv[optind] == NULL) {
fprintf(ERRORFILE, "Invalid user name.\n");
return INVALID_USER_NAME;
}
int ret = set_user(argv[optind]);
if (ret != 0) {
return ret;
}
optind = optind + 1;
command = atoi(argv[optind++]);
fprintf(LOGFILE, "main : command provided %d\n",command);
fprintf(LOGFILE, "main : user is %s\n", user_detail->pw_name);
fflush(LOGFILE);
switch (command) {
case INITIALIZE_CONTAINER:
if (argc < 6) {
fprintf(ERRORFILE, "Too few arguments (%d vs 6) for initialize container\n",
argc);
fflush(ERRORFILE);
return INVALID_ARGUMENT_NUMBER;
}
app_id = argv[optind++];
cred_file = argv[optind++];
exit_code = initialize_app(user_detail->pw_name, app_id, cred_file,
argv + optind);
break;
case LAUNCH_CONTAINER:
if (argc < 9) {
fprintf(ERRORFILE, "Too few arguments (%d vs 9) for launch container\n",
argc);
fflush(ERRORFILE);
return INVALID_ARGUMENT_NUMBER;
}
app_id = argv[optind++];
container_id = argv[optind++];
current_dir = argv[optind++];
script_file = argv[optind++];
cred_file = argv[optind++];
pid_file = argv[optind++];
exit_code = launch_container_as_user(user_detail->pw_name, app_id, container_id,
current_dir, script_file, cred_file, pid_file);
break;
case SIGNAL_CONTAINER:
if (argc < 5) {
fprintf(ERRORFILE, "Too few arguments (%d vs 5) for signal container\n",
argc);
fflush(ERRORFILE);
return INVALID_ARGUMENT_NUMBER;
} else {
char* end_ptr = NULL;
char* option = argv[optind++];
int container_pid = strtol(option, &end_ptr, 10);
if (option == end_ptr || *end_ptr != '\0') {
fprintf(ERRORFILE, "Illegal argument for container pid %s\n", option);
fflush(ERRORFILE);
return INVALID_ARGUMENT_NUMBER;
}
option = argv[optind++];
int signal = strtol(option, &end_ptr, 10);
if (option == end_ptr || *end_ptr != '\0') {
fprintf(ERRORFILE, "Illegal argument for signal %s\n", option);
fflush(ERRORFILE);
return INVALID_ARGUMENT_NUMBER;
}
exit_code = signal_container_as_user(user_detail->pw_name, container_pid, signal);
}
break;
case DELETE_AS_USER:
dir_to_be_deleted = argv[optind++];
exit_code= delete_as_user(user_detail->pw_name, dir_to_be_deleted,
argv + optind);
break;
default:
fprintf(ERRORFILE, "Invalid command %d not supported.",command);
fflush(ERRORFILE);
exit_code = INVALID_COMMAND_PROVIDED;
}
fclose(LOGFILE);
fclose(ERRORFILE);
return exit_code;
}
void test_run_container() {
printf("\nTesting run container\n");
if (seteuid(0) != 0) {
printf("FAIL: seteuid to root failed - %s\n", strerror(errno));
exit(1);
}
FILE* creds = fopen(TEST_ROOT "/creds.txt", "w");
if (creds == NULL) {
printf("FAIL: failed to create credentials file - %s\n", strerror(errno));
exit(1);
}
if (fprintf(creds, "secret key\n") < 0) {
printf("FAIL: fprintf failed - %s\n", strerror(errno));
exit(1);
}
if (fclose(creds) != 0) {
printf("FAIL: fclose failed - %s\n", strerror(errno));
exit(1);
}
char * cgroups_pids[] = { TEST_ROOT "/cgroups-pid1.txt", TEST_ROOT "/cgroups-pid2.txt", 0 };
close(creat(cgroups_pids[0], O_RDWR));
close(creat(cgroups_pids[1], O_RDWR));
const char* script_name = TEST_ROOT "/container-script";
FILE* script = fopen(script_name, "w");
if (script == NULL) {
printf("FAIL: failed to create script file - %s\n", strerror(errno));
exit(1);
}
if (seteuid(user_detail->pw_uid) != 0) {
printf("FAIL: failed to seteuid back to user - %s\n", strerror(errno));
exit(1);
}
if (fprintf(script, "#!/bin/bash\n"
"touch foobar\n"
"exit 0") < 0) {
printf("FAIL: fprintf failed - %s\n", strerror(errno));
exit(1);
}
if (fclose(script) != 0) {
printf("FAIL: fclose failed - %s\n", strerror(errno));
exit(1);
}
fflush(stdout);
fflush(stderr);
char* container_dir = get_container_work_directory(TEST_ROOT "/local-1",
yarn_username, "app_4", "container_1");
const char * pid_file = TEST_ROOT "/pid.txt";
pid_t child = fork();
if (child == -1) {
printf("FAIL: failed to fork process for init_app - %s\n",
strerror(errno));
exit(1);
} else if (child == 0) {
if (launch_container_as_user(yarn_username, "app_4", "container_1",
container_dir, script_name, TEST_ROOT "/creds.txt", pid_file,
local_dirs, log_dirs,
"cgroups", cgroups_pids) != 0) {
printf("FAIL: failed in child\n");
exit(42);
}
// should never return
exit(1);
}
int status = 0;
if (waitpid(child, &status, 0) <= 0) {
printf("FAIL: failed waiting for process %" PRId64 " - %s\n", (int64_t)child,
strerror(errno));
exit(1);
}
if (access(TEST_ROOT "/logs/userlogs/app_4/container_1", R_OK) != 0) {
printf("FAIL: failed to create container log directory\n");
exit(1);
}
if (access(container_dir, R_OK) != 0) {
printf("FAIL: failed to create container directory %s\n", container_dir);
exit(1);
}
char buffer[100000];
sprintf(buffer, "%s/foobar", container_dir);
if (access(buffer, R_OK) != 0) {
printf("FAIL: failed to create touch file %s\n", buffer);
exit(1);
}
free(container_dir);
container_dir = get_app_log_directory(TEST_ROOT "/logs/userlogs", "app_4/container_1");
if (access(container_dir, R_OK) != 0) {
printf("FAIL: failed to create app log directory %s\n", container_dir);
exit(1);
}
free(container_dir);
if (seteuid(0) != 0) {
printf("FAIL: seteuid to root failed - %s\n", strerror(errno));
exit(1);
}
check_pid_file(pid_file, child);
check_pid_file(cgroups_pids[0], child);
check_pid_file(cgroups_pids[1], child);
}
void test_run_container() {
printf("\nTesting run container\n");
if (seteuid(0) != 0) {
printf("FAIL: seteuid to root failed - %s\n", strerror(errno));
exit(1);
}
FILE* creds = fopen(TEST_ROOT "/creds.txt", "w");
if (creds == NULL) {
printf("FAIL: failed to create credentials file - %s\n", strerror(errno));
exit(1);
}
if (fprintf(creds, "secret key\n") < 0) {
printf("FAIL: fprintf failed - %s\n", strerror(errno));
exit(1);
}
if (fclose(creds) != 0) {
printf("FAIL: fclose failed - %s\n", strerror(errno));
exit(1);
}
const char* script_name = TEST_ROOT "/container-script";
FILE* script = fopen(script_name, "w");
if (script == NULL) {
printf("FAIL: failed to create script file - %s\n", strerror(errno));
exit(1);
}
if (seteuid(user_detail->pw_uid) != 0) {
printf("FAIL: failed to seteuid back to user - %s\n", strerror(errno));
exit(1);
}
if (fprintf(script, "#!/bin/bash\n"
"touch foobar\n"
"exit 0") < 0) {
printf("FAIL: fprintf failed - %s\n", strerror(errno));
exit(1);
}
if (fclose(script) != 0) {
printf("FAIL: fclose failed - %s\n", strerror(errno));
exit(1);
}
fflush(stdout);
fflush(stderr);
char* container_dir = get_container_work_directory(TEST_ROOT "/local-1",
username, "app_4", "container_1");
const char * pid_file = TEST_ROOT "/pid.txt";
pid_t child = fork();
if (child == -1) {
printf("FAIL: failed to fork process for init_app - %s\n",
strerror(errno));
exit(1);
} else if (child == 0) {
if (launch_container_as_user(username, "app_4", "container_1",
container_dir, script_name, TEST_ROOT "/creds.txt", pid_file) != 0) {
printf("FAIL: failed in child\n");
exit(42);
}
// should never return
exit(1);
}
int status = 0;
if (waitpid(child, &status, 0) <= 0) {
printf("FAIL: failed waiting for process %d - %s\n", child,
strerror(errno));
exit(1);
}
if (access(TEST_ROOT "/logs/userlogs/app_4/container_1", R_OK) != 0) {
printf("FAIL: failed to create container log directory\n");
exit(1);
}
if (access(container_dir, R_OK) != 0) {
printf("FAIL: failed to create container directory %s\n", container_dir);
exit(1);
}
char buffer[100000];
sprintf(buffer, "%s/foobar", container_dir);
if (access(buffer, R_OK) != 0) {
printf("FAIL: failed to create touch file %s\n", buffer);
exit(1);
}
free(container_dir);
container_dir = get_app_log_directory("logs", "app_4/container_1");
if (access(container_dir, R_OK) != 0) {
printf("FAIL: failed to create app log directory %s\n", container_dir);
exit(1);
}
free(container_dir);
if(access(pid_file, R_OK) != 0) {
printf("FAIL: failed to create pid file %s\n", pid_file);
exit(1);
}
int pidfd = open(pid_file, O_RDONLY);
if (pidfd == -1) {
printf("FAIL: failed to open pid file %s - %s\n", pid_file, strerror(errno));
exit(1);
}
char pidBuf[100];
ssize_t bytes = read(pidfd, pidBuf, 100);
if (bytes == -1) {
printf("FAIL: failed to read from pid file %s - %s\n", pid_file, strerror(errno));
exit(1);
}
pid_t mypid = child;
char myPidBuf[33];
snprintf(myPidBuf, 33, "%d", mypid);
if (strncmp(pidBuf, myPidBuf, strlen(myPidBuf)) != 0) {
printf("FAIL: failed to find matching pid in pid file\n");
printf("FAIL: Expected pid %d : Got %.*s", mypid, (int)bytes, pidBuf);
exit(1);
}
}