virtual int copy_file_to_file(source_file_t& source, const entry_t& target) {
const char_t* chars = 0;
size_t length = 0;
int err = 1;
if ((chars = target.path(length)) && (0 < length)) {
target_file_t target_file;
EV_LOG_MESSAGE_INFO("open target file \"" << chars << "\"...");
if ((target_file.open
(chars, (write_overwrite != write_)
?(target_file.mode_append_binary())
:(target_file.mode_write_binary())))) {
EV_LOG_MESSAGE_INFO("...opened target file \"" << chars << "\"");
err = copy_file_to_file(source, target_file);
EV_LOG_MESSAGE_INFO("close target file \"" << chars << "\"...");
if ((target_file.close())) {
EV_LOG_MESSAGE_INFO("...closed target file \"" << chars << "\"");
} else {
EV_LOG_MESSAGE_INFO("...failed to close target file \"" << chars << "\"");
}
} else {
EV_LOG_MESSAGE_ERROR("...failed to open target file \"" << chars << "\"");
}
}
return err;
}
int
main(int argc, char *argv[]) {
FILE *in, *out;
if (argc != 3) {
fprintf(stderr, "cp <src-file> <destination-file>\n");
return 1;
}
in = fopen(argv[1], "r");
if (in == NULL) {
fprintf(stderr, "cp: %s: ", argv[1]);
perror("");
return 1;
}
out = fopen(argv[2], "w");
if (out == NULL) {
fprintf(stderr, "cp: %s: ", argv[2]);
perror("");
return 1;
}
copy_file_to_file(in, out);
return 0;
}
virtual int copy_file_to_file(const entry_t& source, const entry_t& target) {
const char_t* chars = 0;
size_t length = 0;
int err = 1;
if ((chars = source.path(length)) && (0 < length)) {
source_file_t source_file;
EV_LOG_MESSAGE_INFO("open source file \"" << chars << "\"...");
if ((source_file.open(chars, source_file.mode_read_binary()))) {
EV_LOG_MESSAGE_INFO("...opened source file \"" << chars << "\"");
err = copy_file_to_file(source_file, target);
EV_LOG_MESSAGE_INFO("close source file \"" << chars << "\"...");
if ((source_file.close())) {
EV_LOG_MESSAGE_INFO("...closed source file \"" << chars << "\"");
} else {
EV_LOG_MESSAGE_INFO("...failed to close source file \"" << chars << "\"");
}
} else {
EV_LOG_MESSAGE_ERROR("...failed to open source file \"" << chars << "\"");
}
}
return err;
}
int copy_file_to_dir(const char *src, const char *dst)
{
char buf[PATH_MAX];
char *slash;
if ((slash = rindex(src, '/')) == NULL)
snprintf(buf, sizeof(buf), "%s/%s", dst, src);
else
snprintf(buf, sizeof(buf), "%s/%s", dst, slash);
return copy_file_to_file(src, buf);
}
void copy_file_to_dir(char* from, char* to){
char to_full[FNAME] = {"\0"};
char from_file[FNAME] = {"\0"};
get_file_name(from,from_file);
sprintf(to_full,"%s/%s",to,from_file);
if(g_argoption.hardlink){
copy_file_to_link(from,to_full);
}else{
copy_file_to_file(from,to_full);
}
}
virtual int copy_file(const entry_t& source, const path_t& target) {
entry_t& entry = target_entry_;
const char_t* chars = 0;
int err = 1;
if ((entry.exists(chars = target.chars()))) {
if ((write_overwrite != write_) && (write_append != write_)) {
errf("target file \"%s\" already exists\n", chars);
} else {
fs::entry_type type = fs::entry_type_none;
switch (type = entry.type()) {
case fs::entry_type_file:
err = copy_file_to_file(source, entry);
break;
default:
break;
}
}
} else {
if (!(err = make_directory(entry, target))) {
entry.set_path(chars);
err = copy_file_to_file(source, entry);
} else {
errf("failed to make directory \"%s\"\n", target.directory().chars());
}
}
if (!(err) && (!(to_same != to_) || !(target_modified_))) {
if ((entry.set_times_to_set(source))) {
if ((entry.set_times_set())) {
} else {
}
}
}
return err;
}
/**
* copy src to dst (recursively)
* @param src and dst are file or dir
* @return -1 if failed
*/
int copy_file(const char *src, const char *dst)
{
struct stat st;
if (stat(dst, &st) == 0 && S_ISDIR(st.st_mode)) {
if (stat(src, &st) < 0)
return -1;
if (S_ISDIR(st.st_mode))
return copy_dir_to_dir(src, dst);
else if (S_ISREG(st.st_mode))
return copy_file_to_dir(src, dst);
return -1;
}
else if (stat(src, &st) == 0 && S_ISDIR(st.st_mode))
return copy_dir_to_dir(src, dst);
return copy_file_to_file(src, dst);
}
static int dag_check( void * instance_struct, struct dag *d){
struct archive_instance *a = (struct archive_instance*)instance_struct;
unsigned char digest[SHA1_DIGEST_LENGTH];
// Takes hash of filename and stores it in digest
sha1_file(d->filename, digest);
// Makes a c string copy of your hash address
a->source_makeflow = xxstrdup(sha1_string(digest));
// If a is in write mode using the -w flag
if (a->write) {
// Formats archive file directory
char *source_makeflow_file_dir = string_format("%s/files/%.2s", a->dir, a->source_makeflow);
// If the directory was not able to be created and directory already exists
if (!create_dir(source_makeflow_file_dir, 0777) && errno != EEXIST){
// Prints out a debug error
debug(D_ERROR|D_MAKEFLOW_HOOK, "could not create makeflow archiving directory %s: %d %s\n",
source_makeflow_file_dir, errno, strerror(errno));
// Frees memory for source_makeflow_file_dir
free(source_makeflow_file_dir);
return MAKEFLOW_HOOK_FAILURE;
}
// Gets the path of the archive file
char *source_makeflow_file_path = string_format("%s/%s", source_makeflow_file_dir, a->source_makeflow);
// Frees memory
free(source_makeflow_file_dir);
// If the file was unable to be copied to the new path
if (!copy_file_to_file(d->filename, source_makeflow_file_path)){
// Prints out debug error
debug(D_ERROR|D_MAKEFLOW_HOOK, "Could not archive source makeflow file %s\n", source_makeflow_file_path);
free(source_makeflow_file_path);
return MAKEFLOW_HOOK_FAILURE;
} else {
// Print out where it is stored at
debug(D_MAKEFLOW_HOOK, "Source makeflow %s stored at %s\n", d->filename, source_makeflow_file_path);
}
free(source_makeflow_file_path);
}
return MAKEFLOW_HOOK_SUCCESS;
}
int
main(int argc, char *argv[]) {
if (argc != 3) {
fprintf(stderr, "cp <src-file> <destination-file>\n");
return 1;
}
int in_fd = open(argv[1], O_RDONLY);
if (in_fd < 0) {
fprintf(stderr, "cp: %s: ", argv[1]);
perror("");
return 1;
}
int out_fd = open(argv[2], O_WRONLY|O_CREAT|O_TRUNC, S_IRWXU);
if (out_fd <= 0) {
fprintf(stderr, "cp: %s: ", argv[2]);
perror("");
return 1;
}
copy_file_to_file(in_fd, out_fd);
return 0;
}
char *resource_monitor_copy_to_wd(char *path_from_cmdline)
{
char *mon_unique;
char *monitor_org;
monitor_org = resource_monitor_locate(path_from_cmdline);
if(!monitor_org)
fatal("Monitor program could not be found.\n");
mon_unique = string_format("monitor-%d", getpid());
debug(D_RMON,"copying monitor %s to %s.\n", monitor_org, mon_unique);
if(copy_file_to_file(monitor_org, mon_unique) == 0)
fatal("Could not copy monitor %s to %s in local directory: %s\n",
monitor_org, mon_unique, strerror(errno));
atexit(resource_monitor_delete_exe);
chmod(mon_unique, 0777);
monitor_exe = mon_unique;
return mon_unique;
}
static int bindfile (sqlite3 *db, chirp_jobid_t id, const char *subject, const char *sandbox, struct url_binding **urlsp, const char *task_path, const char *serv_path, const char *binding, const char *type, enum BINDSTATE mode)
{
static const char SQL[] =
"UPDATE JobFile"
" SET serv_path = ?, size = ?"
" WHERE id = ? AND task_path = ? AND type = 'OUTPUT'";
int rc;
sqlite3_stmt *stmt = NULL;
const char *current = SQL;
char task_path_resolved[CHIRP_PATH_MAX] = "";
char serv_path_interpolated[CHIRP_PATH_MAX] = "";
char serv_path_resolved[CHIRP_PATH_MAX] = "";
char task_path_dir[CHIRP_PATH_MAX] = ""; /* directory containing task_path_resolved */
CATCHUNIX(snprintf(task_path_resolved, sizeof(task_path_resolved), "%s/%s", sandbox, task_path));
if ((size_t)rc >= sizeof(task_path_resolved))
CATCH(ENAMETOOLONG);
path_dirname(task_path_resolved, task_path_dir);
if (strcmp(binding, "URL") == 0) {
if (strcmp(type, "INPUT") != 0)
CATCH(EINVAL);
if (mode == BOOTSTRAP) {
debug(D_DEBUG, "binding `%s' for future URL fetch `%s'", task_path, serv_path);
CATCHUNIX(create_dir(task_path_dir, S_IRWXU) ? 0 : -1);
struct url_binding *url = malloc(sizeof(struct url_binding));
CATCHUNIX(url ? 0 : -1);
memset(url, 0, sizeof(struct url_binding));
strncpy(url->path, "", sizeof(url->path));
url->url = NULL;
url->next = *urlsp;
*urlsp = url;
CATCHUNIX(snprintf(url->path, sizeof(url->path), "%s", task_path));
CATCHUNIX((url->url = strdup(serv_path)) ? 0 : -1);
}
rc = 0;
goto out;
}
strncpy(serv_path_interpolated, serv_path, sizeof(serv_path_interpolated)-1);
if (mode == STRAPBOOT && strcmp(type, "OUTPUT") == 0)
CATCH(interpolate(id, task_path_resolved, serv_path_interpolated));
serv_path = serv_path_interpolated;
CATCH(chirp_fs_local_resolve(serv_path, serv_path_resolved));
if (mode == BOOTSTRAP) {
debug(D_DEBUG, "binding `%s' as `%s'", task_path, serv_path);
CATCHUNIX(create_dir(task_path_dir, S_IRWXU) ? 0 : -1);
if (strcmp(type, "INPUT") == 0) {
if (strcmp(binding, "LINK") == 0) {
CATCHUNIX(link(serv_path_resolved, task_path_resolved));
} else if (strcmp(binding, "COPY") == 0) {
CATCHUNIX(copy_file_to_file(serv_path_resolved, task_path_resolved));
} else assert(0);
CATCHUNIX(chmod(serv_path_resolved, S_IRWXU));
}
} else if (mode == STRAPBOOT) {
if (strcmp(type, "OUTPUT") == 0) {
struct stat info;
debug(D_DEBUG, "binding output file `%s' as `%s'", task_path, serv_path);
if (strcmp(binding, "LINK") == 0) {
CATCHUNIXIGNORE(unlink(serv_path_resolved), ENOENT); /* ignore error/success */
CATCHUNIX(link(task_path_resolved, serv_path_resolved));
} else if (strcmp(binding, "COPY") == 0) {
CATCHUNIXIGNORE(unlink(serv_path_resolved), ENOENT);
CATCHUNIX(copy_file_to_file(task_path_resolved, serv_path_resolved));
}
if (stat(serv_path_resolved, &info) == 0) {
sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t));
sqlcatch(sqlite3_bind_text(stmt, 1, serv_path, -1, SQLITE_STATIC));
sqlcatch(sqlite3_bind_int64(stmt, 2, info.st_size));
sqlcatch(sqlite3_bind_int64(stmt, 3, (sqlite3_int64)id));
sqlcatch(sqlite3_bind_text(stmt, 4, task_path, -1, SQLITE_STATIC));
sqlcatchcode(sqlite3_step(stmt), SQLITE_DONE);
sqlcatch(sqlite3_finalize(stmt); stmt = NULL);
}
}
} else assert(0);
int makeflow_archive_copy_preserved_files(struct archive_instance *a, struct batch_task *t, char *task_path ) {
struct batch_file *f;
struct stat buf;
struct list_cursor *cur = list_cursor_create(t->output_files);
// Iterate through output files
for(list_seek(cur, 0); list_get(cur, (void**)&f); list_next(cur)) {
char *file_name = xxstrdup(f->outer_name);
debug(D_MAKEFLOW_HOOK,"Trying to copy file to %s",file_name);
char *file_to_check = xxstrdup(file_name);
//Check to see if the directory was copied as an empty file/incorrectly
stat(dirname(file_to_check),&buf);
if(S_ISREG(buf.st_mode)){
debug(D_MAKEFLOW,"Removing empty file in the place of directory name %s",file_to_check);
char *dirEmpty = string_format("rm -rf %s",file_to_check);
system(dirEmpty);
free(dirEmpty);
}
free(file_to_check);
// Gets path of output file
char *output_file_path = string_format("%s/output_files/%s",task_path,basename(file_name));
char *directory_name = xxstrdup(file_name);
debug(D_MAKEFLOW_HOOK,"Creating directory %s",dirname(directory_name));
if(strcmp(directory_name,file_name) != 0){
//Create the upper level directory to copy the output files into if necessary
if (!create_dir(directory_name, 0777) && errno != EEXIST){
debug(D_ERROR|D_MAKEFLOW_HOOK,"Failed to create directory %s",directory_name);
free(directory_name);
free(output_file_path);
free(file_name);
return 1;
}
}
free(directory_name);
// Copy output file or directory over to specified location
if(path_is_dir(output_file_path) != 1){
int success = copy_file_to_file(output_file_path, file_name);
free(output_file_path);
free(file_name);
if (!success) {
list_cursor_destroy(cur);
debug(D_ERROR|D_MAKEFLOW_HOOK,"Failed to copy output file %s to %s\n", output_file_path, file_name);
return 1;
}
}
else{
if(copy_dir(output_file_path, file_name) != 0){
list_cursor_destroy(cur);
debug(D_ERROR|D_MAKEFLOW_HOOK,"Failed to copy output file %s to %s\n", output_file_path, file_name);
free(output_file_path);
free(file_name);
return 1;
}
free(output_file_path);
free(file_name);
}
}
list_cursor_destroy(cur);
return 0;
}
/* Archive the specified file.
* This includes several steps:
* 1. Generate the id
* 2. Copy file to id if non-existent
* 3. Link back to creating task
*
@return 0 if successfully archived, 1 if failed at any point.
*/
static int makeflow_archive_file(struct archive_instance *a, struct batch_file *f, char *job_file_archive_path) {
/* Generate the file archive id (content based) if does not exist. */
char * id;
if(path_is_dir(f->inner_name) == 1){
f->hash = batch_file_generate_id_dir(f->inner_name);
id = xxstrdup(f->hash);
}
else{
id = batch_file_generate_id(f);
}
struct stat buf;
int rv = 0;
char * file_archive_dir = string_format("%s/files/%.2s", a->dir, id);
char * file_archive_path = string_format("%s/%s", file_archive_dir, id);
char * job_file_archive_dir = NULL;
/* Create the archive path with 2 character prefix. */
if (!create_dir(file_archive_dir, 0777) && errno != EEXIST){
debug(D_ERROR|D_MAKEFLOW_HOOK, "could not create file archiving directory %s: %d %s\n",
file_archive_dir, errno, strerror(errno));
rv = 1;
goto FAIL;
}
/* Check if file is already archived */
if(stat(file_archive_path, &buf) >= 0) {
debug(D_MAKEFLOW_HOOK, "file %s already archived at %s", f->outer_name, file_archive_path);
/* File did not already exist, store in general file area */
} else {
if(path_is_dir(f->outer_name) != 1){
if (!copy_file_to_file(f->outer_name, file_archive_path)){
debug(D_ERROR|D_MAKEFLOW_HOOK, "could not archive output file %s at %s: %d %s\n",
f->outer_name, file_archive_path, errno, strerror(errno));
rv = 1;
goto FAIL;
}
}
else{
debug(D_MAKEFLOW,"COPYING %s to the archive",f->outer_name);
if(copy_dir(f->outer_name,file_archive_path) != 0){
debug(D_ERROR|D_MAKEFLOW_HOOK, "could not archive output file %s at %s: %d %s\n",
f->outer_name, file_archive_path, errno, strerror(errno));
rv = 1;
goto FAIL;
}
}
}
/* Create the directory structure for job_file_archive. */
job_file_archive_dir = xxstrdup(job_file_archive_path);
path_dirname(job_file_archive_path, job_file_archive_dir);
if (!create_dir(job_file_archive_dir, 0777) && errno != EEXIST){
debug(D_ERROR|D_MAKEFLOW_HOOK, "could not create job file directory %s: %d %s\n",
file_archive_dir, errno, strerror(errno));
rv = 1;
goto FAIL;
}
if(a->s3){
int result = 1;
// Check to see if file already exists in the s3 bucket
if(a->s3_check){
if(!in_s3_archive(a,id)){
result = makeflow_archive_s3_file(a,id,file_archive_path);
}
}
else
result = makeflow_archive_s3_file(a,id,file_archive_path);
/* Copy file to the s3 bucket*/
if(!result){
debug(D_ERROR|D_MAKEFLOW_HOOK, "could not copy file %s to s3 bucket: %d %s\n", id, errno, strerror(errno));
rv = 1;
goto FAIL;
}
}
free(file_archive_path);
file_archive_path = string_format("../../../../files/%.2s/%s", id, id);
/* Create a symlink to task that used/created this file. */
int symlink_failure = symlink(file_archive_path, job_file_archive_path);
if (symlink_failure && errno != EEXIST) {
debug(D_ERROR|D_MAKEFLOW_HOOK, "could not create symlink %s pointing to %s: %d %s\n",
job_file_archive_path, file_archive_path, errno, strerror(errno));
rv = 1;
goto FAIL;
}
FAIL:
free(id);
free(file_archive_dir);
free(file_archive_path);
free(job_file_archive_dir);
return rv;
}
