int create_log_dirs(const char *app_id, char * const * log_dirs) { char* const* log_root; char *any_one_app_log_dir = NULL; for(log_root=log_dirs; *log_root != NULL; ++log_root) { char *app_log_dir = get_app_log_directory(*log_root, app_id); if (app_log_dir == NULL) { // try the next one } else if (create_directory_for_user(app_log_dir) != 0) { free(app_log_dir); return -1; } else if (any_one_app_log_dir == NULL) { any_one_app_log_dir = app_log_dir; } else { free(app_log_dir); } } if (any_one_app_log_dir == NULL) { fprintf(LOGFILE, "Did not create any app-log directories\n"); return -1; } free(any_one_app_log_dir); return 0; }
/** * Function to initialize the user directories of a user. */ int initialize_user(const char *user) { char **local_dir = get_values(TT_SYS_DIR_KEY); if (local_dir == NULL) { fprintf(LOGFILE, "%s is not configured.\n", TT_SYS_DIR_KEY); return INVALID_TT_ROOT; } char *user_dir; char **local_dir_ptr = local_dir; int failed = 0; for(local_dir_ptr = local_dir; *local_dir_ptr != 0; ++local_dir_ptr) { user_dir = get_user_directory(*local_dir_ptr, user); if (user_dir == NULL) { fprintf(LOGFILE, "Couldn't get userdir directory for %s.\n", user); failed = 1; break; } if (create_directory_for_user(user_dir) != 0) { failed = 1; } free(user_dir); } free_values(local_dir); return failed ? INITIALIZE_USER_FAILED : 0; }
/** * Function to initialize the user directories of a user. */ int initialize_user(const char *user, const char * good_local_dirs) { char **local_dir = get_mapred_local_dirs(good_local_dirs); if (local_dir == NULL) { fprintf(LOGFILE, "Good mapred local directories could ot be obtained.\n"); return INVALID_TT_ROOT; } char *user_dir; char **local_dir_ptr = local_dir; int failed = 0; for(local_dir_ptr = local_dir; *local_dir_ptr != 0; ++local_dir_ptr) { user_dir = get_user_directory(*local_dir_ptr, user); if (user_dir == NULL) { fprintf(LOGFILE, "Couldn't get userdir directory for %s.\n", user); failed = 1; break; } if (create_directory_for_user(user_dir) != 0) { failed = 1; } free(user_dir); } free_values(local_dir); return failed ? INITIALIZE_USER_FAILED : 0; }
void test_create_log_directory() { printf("\nTesting test_create_log_directory\n"); create_userlogs_dir(); char *job_log_dir = get_job_log_directory("job_7"); if (job_log_dir == NULL) { exit(1); } if (create_directory_for_user(job_log_dir) != 0) { exit(1); } free(job_log_dir); char* good_local_dirs = get_value("mapred.local.dir"); if (good_local_dirs == NULL) { fprintf(LOGFILE, "Mapred local directories could not be obtained.\n"); exit(1); } create_attempt_directories(username, good_local_dirs, "job_7", "task_1"); //check if symlink got created struct stat file; int status; char actualpath [PATH_MAX+1]; char *res; char *filepath = TEST_ROOT "/logs/userlogs/job_7/task_1"; status = lstat(filepath, &file); if (!S_ISLNK(file.st_mode)) { fprintf(LOGFILE, "Symlink creation failed\n"); exit(1); } //Check if symlink path exists res = realpath(filepath, actualpath); if(!res) { fprintf(LOGFILE, "Failed to get target for the symlink\n"); exit(1); } char local_job_dir[PATH_MAX+1]; int i; bool found = false; for(i=1; i<5; i++) { sprintf(local_job_dir, TEST_ROOT "/local-%d/userlogs/job_7/task_1", i); if (strcmp(local_job_dir, actualpath) == 0) { found = true; break; } } if(!found) { printf("FAIL: symlink path and target path mismatch\n"); exit(1); } free(good_local_dirs); }
void test_delete_log_directory() { printf("\nTesting delete_log_directory\n"); char *job_log_dir = get_job_log_directory("job_1"); if (job_log_dir == NULL) { exit(1); } if (create_directory_for_user(job_log_dir) != 0) { exit(1); } free(job_log_dir); char *task_log_dir = get_job_log_directory("job_1/task_2"); if (task_log_dir == NULL) { exit(1); } if (mkdirs(task_log_dir, 0700) != 0) { exit(1); } if (access(TEST_ROOT "/logs/userlogs/job_1/task_2", R_OK) != 0) { printf("FAIL: can't access task directory - %s\n", strerror(errno)); exit(1); } if (delete_log_directory("job_1/task_2") != 0) { printf("FAIL: can't delete task directory\n"); exit(1); } if (access(TEST_ROOT "/logs/userlogs/job_1/task_2", R_OK) == 0) { printf("FAIL: task directory not deleted\n"); exit(1); } if (access(TEST_ROOT "/logs/userlogs/job_1", R_OK) != 0) { printf("FAIL: job directory not deleted - %s\n", strerror(errno)); exit(1); } if (delete_log_directory("job_1") != 0) { printf("FAIL: can't delete task directory\n"); exit(1); } if (access(TEST_ROOT "/logs/userlogs/job_1", R_OK) == 0) { printf("FAIL: job directory not deleted\n"); exit(1); } free(task_log_dir); }
/** * Function to initialize the user directories of a user. */ int initialize_user(const char *user, char* const* local_dirs) { char *user_dir; char* const* local_dir_ptr; int failed = 0; for(local_dir_ptr = local_dirs; *local_dir_ptr != 0; ++local_dir_ptr) { user_dir = get_user_directory(*local_dir_ptr, user); if (user_dir == NULL) { fprintf(LOGFILE, "Couldn't get userdir directory for %s.\n", user); failed = 1; break; } if (create_directory_for_user(user_dir) != 0) { failed = 1; } free(user_dir); } return failed ? INITIALIZE_USER_FAILED : 0; }
/** * Function to prepare the job directories for the task JVM. */ int initialize_job(const char *user, const char *jobid, const char* credentials, const char* job_xml, char* const* args) { if (jobid == NULL || user == NULL) { fprintf(LOGFILE, "Either jobid is null or the user passed is null.\n"); return INVALID_ARGUMENT_NUMBER; } // create the user directory int result = initialize_user(user); if (result != 0) { return result; } // create the log directory for the job char *job_log_dir = get_job_log_directory(jobid); if (job_log_dir == NULL) { return -1; } result = create_directory_for_user(job_log_dir); free(job_log_dir); if (result != 0) { return -1; } // open up the credentials file int cred_file = open_file_as_task_tracker(credentials); if (cred_file == -1) { return -1; } int job_file = open_file_as_task_tracker(job_xml); if (job_file == -1) { return -1; } // give up root privs if (change_user(user_detail->pw_uid, user_detail->pw_gid) != 0) { return -1; } // 750 mode_t permissions = S_IRWXU | S_IRGRP | S_IXGRP; char **tt_roots = get_values(TT_SYS_DIR_KEY); if (tt_roots == NULL) { return INVALID_CONFIG_FILE; } char **tt_root; char *primary_job_dir = NULL; for(tt_root=tt_roots; *tt_root != NULL; ++tt_root) { char *job_dir = get_job_directory(*tt_root, user, jobid); if (job_dir == NULL) { // try the next one } else if (mkdirs(job_dir, permissions) != 0) { free(job_dir); } else if (primary_job_dir == NULL) { primary_job_dir = job_dir; } else { free(job_dir); } } free_values(tt_roots); if (primary_job_dir == NULL) { fprintf(LOGFILE, "Did not create any job directories\n"); return -1; } char *cred_file_name = concatenate("%s/%s", "cred file", 2, primary_job_dir, CREDENTIALS_FILENAME); if (cred_file_name == NULL) { return -1; } if (copy_file(cred_file, credentials, cred_file_name, S_IRUSR|S_IWUSR) != 0){ return -1; } char *job_file_name = concatenate("%s/%s", "job file", 2, primary_job_dir, JOB_FILENAME); if (job_file_name == NULL) { return -1; } if (copy_file(job_file, job_xml, job_file_name, S_IRUSR|S_IWUSR|S_IRGRP) != 0) { return -1; } fclose(stdin); fflush(LOGFILE); if (LOGFILE != stdout) { fclose(stdout); } fclose(stderr); if (chdir(primary_job_dir)) { fprintf(LOGFILE, "Failure to chdir to job dir - %s\n", strerror(errno)); return -1; } execvp(args[0], args); fprintf(LOGFILE, "Failure to exec job initialization process - %s\n", strerror(errno)); return -1; }
/** * Function to prepare the application directories for the container. */ int initialize_app(const char *user, const char *app_id, const char* nmPrivate_credentials_file, char* const* local_dirs, char* const* log_roots, char* const* args) { if (app_id == NULL || user == NULL) { fprintf(LOGFILE, "Either app_id is null or the user passed is null.\n"); return INVALID_ARGUMENT_NUMBER; } // create the user directory on all disks int result = initialize_user(user, local_dirs); if (result != 0) { return result; } ////////////// create the log directories for the app on all disks char* const* log_root; char *any_one_app_log_dir = NULL; for(log_root=log_roots; *log_root != NULL; ++log_root) { char *app_log_dir = get_app_log_directory(*log_root, app_id); if (app_log_dir == NULL) { // try the next one } else if (create_directory_for_user(app_log_dir) != 0) { free(app_log_dir); return -1; } else if (any_one_app_log_dir == NULL) { any_one_app_log_dir = app_log_dir; } else { free(app_log_dir); } } if (any_one_app_log_dir == NULL) { fprintf(LOGFILE, "Did not create any app-log directories\n"); return -1; } free(any_one_app_log_dir); ////////////// End of creating the log directories for the app on all disks // open up the credentials file int cred_file = open_file_as_nm(nmPrivate_credentials_file); if (cred_file == -1) { return -1; } // give up root privs if (change_user(user_detail->pw_uid, user_detail->pw_gid) != 0) { return -1; } // 750 mode_t permissions = S_IRWXU | S_IRGRP | S_IXGRP; char* const* nm_root; char *primary_app_dir = NULL; for(nm_root=local_dirs; *nm_root != NULL; ++nm_root) { char *app_dir = get_app_directory(*nm_root, user, app_id); if (app_dir == NULL) { // try the next one } else if (mkdirs(app_dir, permissions) != 0) { free(app_dir); } else if (primary_app_dir == NULL) { primary_app_dir = app_dir; } else { free(app_dir); } } if (primary_app_dir == NULL) { fprintf(LOGFILE, "Did not create any app directories\n"); return -1; } char *nmPrivate_credentials_file_copy = strdup(nmPrivate_credentials_file); // TODO: FIXME. The user's copy of creds should go to a path selected by // localDirAllocatoir char *cred_file_name = concatenate("%s/%s", "cred file", 2, primary_app_dir, basename(nmPrivate_credentials_file_copy)); if (cred_file_name == NULL) { free(nmPrivate_credentials_file_copy); return -1; } if (copy_file(cred_file, nmPrivate_credentials_file, cred_file_name, S_IRUSR|S_IWUSR) != 0){ free(nmPrivate_credentials_file_copy); return -1; } free(nmPrivate_credentials_file_copy); fclose(stdin); fflush(LOGFILE); if (LOGFILE != stdout) { fclose(stdout); } if (ERRORFILE != stderr) { fclose(stderr); } if (chdir(primary_app_dir) != 0) { fprintf(LOGFILE, "Failed to chdir to app dir - %s\n", strerror(errno)); return -1; } execvp(args[0], args); fprintf(ERRORFILE, "Failure to exec app initialization process - %s\n", strerror(errno)); return -1; }
void test_delete_log_directory() { printf("\nTesting delete_log_directory\n"); char* local_dirs = get_value("mapred.local.dir"); char *job_log_dir = get_job_log_directory("job_1"); if (job_log_dir == NULL) { exit(1); } if (create_directory_for_user(job_log_dir) != 0) { exit(1); } free(job_log_dir); char *task_log_dir = get_job_log_directory("job_1/task_2"); if (task_log_dir == NULL) { exit(1); } if (mkdirs(task_log_dir, 0700) != 0) { exit(1); } if (access(TEST_ROOT "/logs/userlogs/job_1/task_2", R_OK) != 0) { printf("FAIL: can't access task directory - %s\n", strerror(errno)); exit(1); } if (delete_log_directory("job_1/task_2", local_dirs) != 0) { printf("FAIL: can't delete task directory\n"); exit(1); } if (access(TEST_ROOT "/logs/userlogs/job_1/task_2", R_OK) == 0) { printf("FAIL: task directory not deleted\n"); exit(1); } if (access(TEST_ROOT "/logs/userlogs/job_1", R_OK) != 0) { printf("FAIL: job directory not deleted - %s\n", strerror(errno)); exit(1); } if (delete_log_directory("job_1", local_dirs) != 0) { printf("FAIL: can't delete task directory\n"); exit(1); } if (access(TEST_ROOT "/logs/userlogs/job_1", R_OK) == 0) { printf("FAIL: job directory not deleted\n"); exit(1); } if (delete_log_directory("job_7", local_dirs) != 0) { printf("FAIL: can't delete job directory\n"); exit(1); } if (access(TEST_ROOT "/logs/userlogs/job_7", R_OK) == 0) { printf("FAIL: job log directory not deleted\n"); exit(1); } char local_job_dir[PATH_MAX+1]; int i; for(i=1; i<5; i++) { sprintf(local_job_dir, TEST_ROOT "/local-%d/userlogs/job_7", i); if (access(local_job_dir, R_OK) == 0) { printf("FAIL: job log directory in mapred local not deleted\n"); exit(1); } } free(task_log_dir); free(local_dirs); }
/** * Function to prepare the attempt directories for the task JVM. * It creates the task work and log directories. */ int create_attempt_directories(const char* user, const char * good_local_dirs, const char *job_id, const char *task_id) { // create dirs as 0750 const mode_t perms = S_IRWXU | S_IRGRP | S_IXGRP; if (job_id == NULL || task_id == NULL || user == NULL) { fprintf(LOGFILE, "Either task_id is null or the user passed is null.\n"); return -1; } int result = 0; char **local_dir = get_mapred_local_dirs(good_local_dirs); if (local_dir == NULL) { fprintf(LOGFILE, "Good mapred local directories could not be obtained.\n"); return INVALID_TT_ROOT; } char **local_dir_ptr; for(local_dir_ptr = local_dir; *local_dir_ptr != NULL; ++local_dir_ptr) { char *task_dir = get_attempt_work_directory(*local_dir_ptr, user, job_id, task_id); if (task_dir == NULL) { free_values(local_dir); return -1; } if (mkdirs(task_dir, perms) != 0) { // continue on to create other task directories free(task_dir); } else { free(task_dir); } } // also make the directory for the task logs char *job_task_name = malloc(strlen(job_id) + strlen(task_id) + 2); char *real_task_dir = NULL; // target of symlink char *real_job_dir = NULL; // parent dir of target of symlink char *random_local_dir = NULL; char *link_task_log_dir = NULL; // symlink if (job_task_name == NULL) { fprintf(LOGFILE, "Malloc of job task name failed\n"); result = -1; } else { sprintf(job_task_name, "%s/%s", job_id, task_id); link_task_log_dir = get_job_log_directory(job_task_name); random_local_dir = get_random_local_dir(local_dir); if(random_local_dir == NULL) { result = -1; goto cleanup; } real_job_dir = malloc(strlen(random_local_dir) + strlen(USERLOGS) + strlen(job_id) + 3); if (real_job_dir == NULL) { fprintf(LOGFILE, "Malloc of real job directory failed\n"); result = -1; goto cleanup; } real_task_dir = malloc(strlen(random_local_dir) + strlen(USERLOGS) + strlen(job_id) + strlen(task_id) + 4); if (real_task_dir == NULL) { fprintf(LOGFILE, "Malloc of real task directory failed\n"); result = -1; goto cleanup; } sprintf(real_job_dir, "%s/userlogs/%s", random_local_dir, job_id); result = create_directory_for_user(real_job_dir); if( result != 0) { result = -1; goto cleanup; } sprintf(real_task_dir, "%s/userlogs/%s/%s", random_local_dir, job_id, task_id); result = mkdirs(real_task_dir, perms); if( result != 0) { result = -1; goto cleanup; } result = symlink(real_task_dir, link_task_log_dir); if( result != 0) { fprintf(LOGFILE, "Failed to create symlink %s to %s - %s\n", link_task_log_dir, real_task_dir, strerror(errno)); result = -1; } } cleanup: free(random_local_dir); free(job_task_name); free(link_task_log_dir); free(real_job_dir); free(real_task_dir); free_values(local_dir); return result; }