static bool archive_directory (char file_name[], size_t file_name_size, int file_fd, int archive_fd, bool *write_error) { size_t dir_len; bool success = true; dir_len = strlen (file_name); if (dir_len + 1 + READDIR_MAX_LEN + 1 > file_name_size) { printf ("%s: file name too long\n", file_name); return false; } if (!write_header (file_name, USTAR_DIRECTORY, 0, archive_fd, write_error)) return false; file_name[dir_len] = '/'; while (readdir (file_fd, &file_name[dir_len + 1])) if (!archive_file (file_name, file_name_size, archive_fd, write_error)) success = false; file_name[dir_len] = '\0'; return success; }
static void archive_dir(const std::string& path, const std::string& dirname, config& cfg, std::pair<std::vector<std::string>, std::vector<std::string> >& ignore_patterns) { cfg["name"] = dirname; const std::string dir = path + '/' + dirname; std::vector<std::string> files, dirs; get_files_in_dir(dir,&files,&dirs); for(std::vector<std::string>::const_iterator i = files.begin(); i != files.end(); ++i) { bool valid = !looks_like_pbl(*i); for(std::vector<std::string>::const_iterator p = ignore_patterns.first.begin(); p != ignore_patterns.first.end(); ++p) { if (utils::wildcard_string_match(*i, *p)) { valid = false; break; } } if (valid) { archive_file(dir,*i,cfg.add_child("file")); } } for(std::vector<std::string>::const_iterator j = dirs.begin(); j != dirs.end(); ++j) { bool valid = true; for(std::vector<std::string>::const_iterator p = ignore_patterns.second.begin(); p != ignore_patterns.second.end(); ++p) { if (utils::wildcard_string_match(*j, *p)) { valid = false; break; } } if (valid) { archive_dir(dir,*j,cfg.add_child("dir"),ignore_patterns); } } }
//}}} //{{{ void save_options(void) { FILE *configfile; #ifdef REALCOMPUTER char full_filename[256]; #endif #ifdef IS_CALCULATOR EM_moveSymFromExtMem(SYMSTR_CONST(OPTIONSFILENAME), HS_NULL); #endif #ifdef REALCOMPUTER snprintf(full_filename, 256, "%s" OPTIONSFILENAME, file_prefix); configfile = fopen(full_filename, "wb"); #else configfile = fopen(OPTIONSFILENAME, "wb"); #endif if(!configfile) { message("Couldn't open configuration file to save options."); return; } fwrite(&num_options_set, sizeof(ulong), 1, configfile); fwrite(options_set, sizeof(option_value), num_options_set, configfile); fclose(configfile); #ifdef IS_CALCULATOR if(get_option(OPTION_ARCHIVEOPTIONS) == OPTION_ARCHIVE_YES) archive_file(OPTIONSFILENAME, sizeof(configfile)); #endif }
void archive_addon(const std::string& addon_name, config& cfg) { const std::string parentd = get_addon_campaigns_dir(); std::pair<std::vector<std::string>, std::vector<std::string> > ignore_patterns; // External .cfg may not exist; newer campaigns have a _main.cfg const std::string external_cfg = addon_name + ".cfg"; if (file_exists(parentd + "/" + external_cfg)) { archive_file(parentd, external_cfg, cfg.add_child("file")); } ignore_patterns = read_ignore_patterns(addon_name); archive_dir(parentd, addon_name, cfg.add_child("dir"), ignore_patterns); }
static bool make_tar_archive (const char *archive_name, char *files[], size_t file_cnt) { static const char zeros[512]; int archive_fd; bool success = true; bool write_error = false; size_t i; if (!create (archive_name, 0)) { printf ("%s: create failed\n", archive_name); return false; } archive_fd = open (archive_name); if (archive_fd < 0) { printf ("%s: open failed\n", archive_name); return false; } for (i = 0; i < file_cnt; i++) { char file_name[128]; strlcpy (file_name, files[i], sizeof file_name); if (!archive_file (file_name, sizeof file_name, archive_fd, &write_error)) success = false; } if (!do_write (archive_fd, zeros, 512, &write_error) || !do_write (archive_fd, zeros, 512, &write_error)) success = false; close (archive_fd); return success; }
/*$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ main() $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$*/ int main(int argc, char *argv[]) { #ifdef _WITH_BURST_2 int cb2_ret; #endif int exec_done, exit_status = TRANSFER_SUCCESS, ii, k, last_k, length, length_start, mask_file_name, #ifdef HAVE_SETPRIORITY sched_priority, #endif ret; time_t last_update_time, now, *p_file_mtime_buffer; char command_str[MAX_PATH_LENGTH + MAX_RECIPIENT_LENGTH], file_name[MAX_FILENAME_LENGTH], file_path[MAX_PATH_LENGTH], *fnp, *insert_list[MAX_EXEC_FILE_SUBSTITUTION], job_str[4], *p_command, *p_source_file, *p_file_name_buffer, *return_str = NULL, source_file[MAX_PATH_LENGTH], tmp_char, tmp_option[MAX_PATH_LENGTH + MAX_RECIPIENT_LENGTH]; struct job *p_db; #ifdef SA_FULLDUMP struct sigaction sact; #endif #ifdef WITH_FAST_MOVE nlink_t nlink; #endif #ifdef _OUTPUT_LOG clock_t end_time = 0, start_time = 0; struct tms tmsdummy; #endif CHECK_FOR_VERSION(argc, argv); #ifdef SA_FULLDUMP /* * When dumping core sure we do a FULL core dump! */ sact.sa_handler = SIG_DFL; sact.sa_flags = SA_FULLDUMP; sigemptyset(&sact.sa_mask); if (sigaction(SIGSEGV, &sact, NULL) == -1) { system_log(ERROR_SIGN, __FILE__, __LINE__, "sigaction() error : %s", strerror(errno)); exit(INCORRECT); } #endif /* Do some cleanups when we exit. */ if (atexit(sf_exec_exit) != 0) { system_log(ERROR_SIGN, __FILE__, __LINE__, "Could not register exit function : %s", strerror(errno)); exit(INCORRECT); } /* Initialise variables. */ local_file_counter = 0; files_to_send = init_sf(argc, argv, file_path, EXEC_FLAG); p_db = &db; if ((signal(SIGINT, sig_kill) == SIG_ERR) || (signal(SIGQUIT, sig_exit) == SIG_ERR) || (signal(SIGTERM, sig_exit) == SIG_ERR) || (signal(SIGSEGV, sig_segv) == SIG_ERR) || (signal(SIGBUS, sig_bus) == SIG_ERR) || (signal(SIGHUP, SIG_IGN) == SIG_ERR)) { system_log(ERROR_SIGN, __FILE__, __LINE__, "Failed to set signal handlers : %s", strerror(errno)); exit(INCORRECT); } /* Inform FSA that we have are ready to copy the files. */ if (gsf_check_fsa(p_db) != NEITHER) { fsa->job_status[(int)db.job_no].connect_status = EXEC_ACTIVE; fsa->job_status[(int)db.job_no].no_of_files = files_to_send; } /* Init job_str for exec_cmd(). */ job_str[0] = '['; job_str[1] = db.job_no + '0'; job_str[2] = ']'; job_str[3] = '\0'; #ifdef _WITH_BURST_2 do { if (burst_2_counter > 0) { if (fsa->debug > NORMAL_MODE) { trans_db_log(INFO_SIGN, __FILE__, __LINE__, NULL, "Bursting."); } } #endif /* _WITH_BURST_2 */ /* Prepare pointers and directory name. */ (void)strcpy(source_file, file_path); p_source_file = source_file + strlen(source_file); *p_source_file++ = '/'; #ifdef HAVE_SETPRIORITY if (db.exec_base_priority != NO_PRIORITY) { sched_priority = db.exec_base_priority; if (db.add_afd_priority == YES) { sched_priority += (int)(fsa->job_status[(int)db.job_no].unique_name[MAX_MSG_NAME_LENGTH - 1]); if (sched_priority > db.min_sched_priority) { sched_priority = db.min_sched_priority; } else if (sched_priority < db.max_sched_priority) { sched_priority = db.max_sched_priority; } } if ((sched_priority == db.current_priority) || ((db.current_priority > sched_priority) && (geteuid() != 0))) { sched_priority = NO_PRIORITY; } } else { sched_priority = NO_PRIORITY; } #endif p_command = db.exec_cmd; while ((*p_command == ' ') || (*p_command == '\t')) { p_command++; } if ((db.special_flag & EXEC_ONCE_ONLY) == 0) { char *p_end; /* Prepare insert list. */ p_end = p_command; k = 0; ii = 0; while ((*p_end != '\0') && (ii < MAX_EXEC_FILE_SUBSTITUTION)) { if ((*p_end == '%') && (*(p_end + 1) == 's')) { tmp_option[k] = *p_end; tmp_option[k + 1] = *(p_end + 1); insert_list[ii] = &tmp_option[k]; ii++; p_end += 2; k += 2; } else { tmp_option[k] = *p_end; p_end++; k++; } } if (ii >= MAX_EXEC_FILE_SUBSTITUTION) { trans_log(WARN_SIGN, __FILE__, __LINE__, NULL, NULL, "To many %%s in pexec option. Can oly handle %d.", MAX_EXEC_FILE_SUBSTITUTION); } tmp_option[k] = '\0'; p_command = tmp_option; last_k = k; } p_file_name_buffer = file_name_buffer; p_file_size_buffer = file_size_buffer; p_file_mtime_buffer = file_mtime_buffer; last_update_time = time(NULL); local_file_size = 0; exec_done = NO; for (files_send = 0; files_send < files_to_send; files_send++) { (void)strcpy(p_source_file, p_file_name_buffer); if (db.special_flag & EXEC_ONCE_ONLY) { if (exec_done == NO) { #ifdef _OUTPUT_LOG if (db.output_log == YES) { start_time = times(&tmsdummy); } #endif (void)sprintf(command_str, "cd %s && %s", file_path, p_command); if (fsa->debug > NORMAL_MODE) { trans_db_log(INFO_SIGN, __FILE__, __LINE__, NULL, "Executing command `%s' to send file `%s'", command_str, p_file_name_buffer); } if ((ret = exec_cmd(command_str, &return_str, transfer_log_fd, fsa->host_dsp_name, MAX_HOSTNAME_LENGTH, #ifdef HAVE_SETPRIORITY sched_priority, #endif job_str, (fsa->protocol_options & TIMEOUT_TRANSFER) ? (time_t)transfer_timeout : 0L, YES, YES)) != 0) { trans_log(ERROR_SIGN, __FILE__, __LINE__, NULL, NULL, "Failed to execute command %s [Return code = %d]", command_str, ret); if ((return_str != NULL) && (return_str[0] != '\0')) { char *end_ptr = return_str, *start_ptr; do { start_ptr = end_ptr; while ((*end_ptr != '\n') && (*end_ptr != '\0')) { end_ptr++; } if (*end_ptr == '\n') { *end_ptr = '\0'; end_ptr++; } trans_log(ERROR_SIGN, __FILE__, __LINE__, NULL, NULL, "%s", start_ptr); } while (*end_ptr != '\0'); } exit(EXEC_ERROR); } #ifdef _OUTPUT_LOG if (db.output_log == YES) { end_time = times(&tmsdummy); } #endif free(return_str); return_str = NULL; exec_done = YES; } } else { /* Write status to FSA? */ if (gsf_check_fsa(p_db) != NEITHER) { fsa->job_status[(int)db.job_no].file_size_in_use = *p_file_size_buffer; (void)strcpy(fsa->job_status[(int)db.job_no].file_name_in_use, p_file_name_buffer); } #ifdef _OUTPUT_LOG if (db.output_log == YES) { start_time = times(&tmsdummy); } #endif insert_list[ii] = &tmp_option[last_k]; tmp_char = *insert_list[0]; *insert_list[0] = '\0'; length_start = sprintf(command_str, "cd %s && %s", file_path, p_command); *insert_list[0] = tmp_char; fnp = p_file_name_buffer; mask_file_name = NO; do { if ((*fnp == ';') || (*fnp == ' ')) { mask_file_name = YES; break; } fnp++; } while (*fnp != '\0'); /* Generate command string with file name(s). */ length = 0; for (k = 1; k < (ii + 1); k++) { tmp_char = *insert_list[k]; *insert_list[k] = '\0'; if (mask_file_name == YES) { length += sprintf(command_str + length_start + length, "\"%s\"%s", p_file_name_buffer, insert_list[k - 1] + 2); } else { length += sprintf(command_str + length_start + length, "%s%s", p_file_name_buffer, insert_list[k - 1] + 2); } *insert_list[k] = tmp_char; } if ((ret = exec_cmd(command_str, &return_str, transfer_log_fd, fsa->host_dsp_name, MAX_HOSTNAME_LENGTH, #ifdef HAVE_SETPRIORITY sched_priority, #endif job_str, (fsa->protocol_options & TIMEOUT_TRANSFER) ? (time_t)transfer_timeout : 0L, YES, YES)) != 0) /* ie != SUCCESS */ { trans_log(ERROR_SIGN, __FILE__, __LINE__, NULL, NULL, "Failed to execute command %s [Return code = %d]", command_str, ret); if ((return_str != NULL) && (return_str[0] != '\0')) { char *end_ptr = return_str, *start_ptr; do { start_ptr = end_ptr; while ((*end_ptr != '\n') && (*end_ptr != '\0')) { end_ptr++; } if (*end_ptr == '\n') { *end_ptr = '\0'; end_ptr++; } trans_log(ERROR_SIGN, __FILE__, __LINE__, NULL, NULL, "%s", start_ptr); } while (*end_ptr != '\0'); } exit(EXEC_ERROR); } free(return_str); return_str = NULL; #ifdef _OUTPUT_LOG if (db.output_log == YES) { end_time = times(&tmsdummy); } #endif } /* Tell FSA we have done a file. */ if (gsf_check_fsa(p_db) != NEITHER) { fsa->job_status[(int)db.job_no].file_name_in_use[0] = '\0'; fsa->job_status[(int)db.job_no].no_of_files_done++; fsa->job_status[(int)db.job_no].file_size_in_use = 0; fsa->job_status[(int)db.job_no].file_size_in_use_done = 0; fsa->job_status[(int)db.job_no].file_size_done += *p_file_size_buffer; fsa->job_status[(int)db.job_no].bytes_send += *p_file_size_buffer; local_file_size += *p_file_size_buffer; local_file_counter += 1; now = time(NULL); if (now >= (last_update_time + LOCK_INTERVAL_TIME)) { last_update_time = now; update_tfc(local_file_counter, local_file_size, p_file_size_buffer, files_to_send, files_send); local_file_size = 0; local_file_counter = 0; } } #ifdef _WITH_TRANS_EXEC if (db.special_flag & TRANS_EXEC) { trans_exec(file_path, source_file, p_file_name_buffer); } #endif #ifdef _OUTPUT_LOG if (db.output_log == YES) { if (ol_fd == -2) { # ifdef WITHOUT_FIFO_RW_SUPPORT output_log_fd(&ol_fd, &ol_readfd); # else output_log_fd(&ol_fd); # endif } if ((ol_fd > -1) && (ol_data == NULL)) { output_log_ptrs(&ol_retries, &ol_job_number, &ol_data, &ol_file_name, &ol_file_name_length, &ol_archive_name_length, &ol_file_size, &ol_unl, &ol_size, &ol_transfer_time, &ol_output_type, db.host_alias, 0, EXEC); } } #endif /* Now archive file if necessary. */ if ((db.archive_time > 0) && (p_db->archive_dir[0] != FAILED_TO_CREATE_ARCHIVE_DIR)) { /* * By telling the function archive_file() that this * is the first time to archive a file for this job * (in struct p_db) it does not always have to check * whether the directory has been created or not. And * we ensure that we do not create duplicate names * when adding db.archive_time to msg_name. */ if (archive_file(file_path, p_file_name_buffer, p_db) < 0) { trans_log(ERROR_SIGN, __FILE__, __LINE__, NULL, NULL, "Failed to archive file `%s'", file_name); /* * NOTE: We _MUST_ delete the file we just send, * else the file directory will run full! */ if (unlink(source_file) == -1) { system_log(ERROR_SIGN, __FILE__, __LINE__, "Could not unlink() local file `%s' after copying it successfully : %s", source_file, strerror(errno)); } #ifdef _OUTPUT_LOG if (db.output_log == YES) { (void)memcpy(ol_file_name, db.p_unique_name, db.unl); if (db.trans_rename_rule[0] != '\0') { *ol_file_name_length = (unsigned short)sprintf(ol_file_name + db.unl, "%s%c/%s", p_file_name_buffer, SEPARATOR_CHAR, p_file_name_buffer) + db.unl; } else { (void)strcpy(ol_file_name + db.unl, p_file_name_buffer); *ol_file_name_length = (unsigned short)strlen(ol_file_name); ol_file_name[*ol_file_name_length] = SEPARATOR_CHAR; ol_file_name[*ol_file_name_length + 1] = '\0'; (*ol_file_name_length)++; } *ol_file_size = *p_file_size_buffer; *ol_job_number = fsa->job_status[(int)db.job_no].job_id; *ol_retries = db.retries; *ol_unl = db.unl; *ol_transfer_time = end_time - start_time; *ol_archive_name_length = 0; *ol_output_type = OT_NORMAL_DELIVERED + '0'; ol_real_size = *ol_file_name_length + ol_size; if (write(ol_fd, ol_data, ol_real_size) != ol_real_size) { system_log(ERROR_SIGN, __FILE__, __LINE__, "write() error : %s", strerror(errno)); } } #endif } else { if (fsa->debug > NORMAL_MODE) { trans_db_log(INFO_SIGN, __FILE__, __LINE__, NULL, "Archived file `%s'.", file_name); } #ifdef _OUTPUT_LOG if (db.output_log == YES) { (void)memcpy(ol_file_name, db.p_unique_name, db.unl); if (db.trans_rename_rule[0] != '\0') { *ol_file_name_length = (unsigned short)sprintf(ol_file_name + db.unl, "%s%c/%s", p_file_name_buffer, SEPARATOR_CHAR, p_file_name_buffer) + db.unl; } else { (void)strcpy(ol_file_name + db.unl, p_file_name_buffer); *ol_file_name_length = (unsigned short)strlen(ol_file_name); ol_file_name[*ol_file_name_length] = SEPARATOR_CHAR; ol_file_name[*ol_file_name_length + 1] = '\0'; (*ol_file_name_length)++; } (void)strcpy(&ol_file_name[*ol_file_name_length + 1], &db.archive_dir[db.archive_offset]); *ol_file_size = *p_file_size_buffer; *ol_job_number = fsa->job_status[(int)db.job_no].job_id; *ol_retries = db.retries; *ol_unl = db.unl; *ol_transfer_time = end_time - start_time; *ol_archive_name_length = (unsigned short)strlen(&ol_file_name[*ol_file_name_length + 1]); *ol_output_type = OT_NORMAL_DELIVERED + '0'; ol_real_size = *ol_file_name_length + *ol_archive_name_length + 1 + ol_size; if (write(ol_fd, ol_data, ol_real_size) != ol_real_size) { system_log(ERROR_SIGN, __FILE__, __LINE__, "write() error : %s", strerror(errno)); } } #endif } } else { #ifdef WITH_UNLINK_DELAY int unlink_loops = 0; try_again_unlink: #endif /* Delete the file we just have executed. */ if (unlink(source_file) == -1) { #ifdef WITH_UNLINK_DELAY if ((errno == EBUSY) && (unlink_loops < 20)) { (void)my_usleep(100000L); unlink_loops++; goto try_again_unlink; } #endif system_log(ERROR_SIGN, __FILE__, __LINE__, "Could not unlink() local file %s after copying it successfully : %s", source_file, strerror(errno)); } #ifdef _OUTPUT_LOG if (db.output_log == YES) { (void)memcpy(ol_file_name, db.p_unique_name, db.unl); if (db.trans_rename_rule[0] != '\0') { *ol_file_name_length = (unsigned short)sprintf(ol_file_name + db.unl, "%s%c/%s", p_file_name_buffer, SEPARATOR_CHAR, p_file_name_buffer) + db.unl; } else { (void)strcpy(ol_file_name + db.unl, p_file_name_buffer); *ol_file_name_length = (unsigned short)strlen(ol_file_name); ol_file_name[*ol_file_name_length] = SEPARATOR_CHAR; ol_file_name[*ol_file_name_length + 1] = '\0'; (*ol_file_name_length)++; } *ol_file_size = *p_file_size_buffer; *ol_job_number = fsa->job_status[(int)db.job_no].job_id; *ol_retries = db.retries; *ol_unl = db.unl; *ol_transfer_time = end_time - start_time; *ol_archive_name_length = 0; *ol_output_type = OT_NORMAL_DELIVERED + '0'; ol_real_size = *ol_file_name_length + ol_size; if (write(ol_fd, ol_data, ol_real_size) != ol_real_size) { system_log(ERROR_SIGN, __FILE__, __LINE__, "write() error : %s", strerror(errno)); } } #endif } /* * After each successful transfer set error counter to zero, * so that other jobs can be started. */ if (fsa->error_counter > 0) { int fd, #ifdef WITHOUT_FIFO_RW_SUPPORT readfd, #endif j; char fd_wake_up_fifo[MAX_PATH_LENGTH]; #ifdef LOCK_DEBUG lock_region_w(fsa_fd, db.lock_offset + LOCK_EC, __FILE__, __LINE__); #else lock_region_w(fsa_fd, db.lock_offset + LOCK_EC); #endif fsa->error_counter = 0; /* * Wake up FD! */ (void)sprintf(fd_wake_up_fifo, "%s%s%s", p_work_dir, FIFO_DIR, FD_WAKE_UP_FIFO); #ifdef WITHOUT_FIFO_RW_SUPPORT if (open_fifo_rw(fd_wake_up_fifo, &readfd, &fd) == -1) #else if ((fd = open(fd_wake_up_fifo, O_RDWR)) == -1) #endif { system_log(WARN_SIGN, __FILE__, __LINE__, "Failed to open() FIFO %s : %s", fd_wake_up_fifo, strerror(errno)); } else { if (write(fd, "", 1) != 1) { system_log(WARN_SIGN, __FILE__, __LINE__, "Failed to write() to FIFO %s : %s", fd_wake_up_fifo, strerror(errno)); } #ifdef WITHOUT_FIFO_RW_SUPPORT if (close(readfd) == -1) { system_log(DEBUG_SIGN, __FILE__, __LINE__, "Failed to close() FIFO %s : %s", fd_wake_up_fifo, strerror(errno)); } #endif if (close(fd) == -1) { system_log(DEBUG_SIGN, __FILE__, __LINE__, "Failed to close() FIFO %s : %s", fd_wake_up_fifo, strerror(errno)); } } /* * Remove the error condition (NOT_WORKING) from all jobs * of this host. */ for (j = 0; j < fsa->allowed_transfers; j++) { if ((j != db.job_no) && (fsa->job_status[j].connect_status == NOT_WORKING)) { fsa->job_status[j].connect_status = DISCONNECT; } } fsa->error_history[0] = 0; fsa->error_history[1] = 0; #ifdef LOCK_DEBUG unlock_region(fsa_fd, db.lock_offset + LOCK_EC, __FILE__, __LINE__); #else unlock_region(fsa_fd, db.lock_offset + LOCK_EC); #endif /* * Since we have successfully transmitted a file, no need to * have the queue stopped anymore. */ if (fsa->host_status & AUTO_PAUSE_QUEUE_STAT) { char *sign; #ifdef LOCK_DEBUG lock_region_w(fsa_fd, db.lock_offset + LOCK_HS, __FILE__, __LINE__); #else lock_region_w(fsa_fd, db.lock_offset + LOCK_HS); #endif fsa->host_status &= ~AUTO_PAUSE_QUEUE_STAT; if (fsa->host_status & HOST_ERROR_EA_STATIC) { fsa->host_status &= ~EVENT_STATUS_STATIC_FLAGS; } else { fsa->host_status &= ~EVENT_STATUS_FLAGS; } fsa->host_status &= ~PENDING_ERRORS; #ifdef LOCK_DEBUG unlock_region(fsa_fd, db.lock_offset + LOCK_HS, __FILE__, __LINE__); #else unlock_region(fsa_fd, db.lock_offset + LOCK_HS); #endif error_action(fsa->host_alias, "stop", HOST_ERROR_ACTION); event_log(0L, EC_HOST, ET_EXT, EA_ERROR_END, "%s", fsa->host_alias); if ((fsa->host_status & HOST_ERROR_OFFLINE_STATIC) || (fsa->host_status & HOST_ERROR_OFFLINE) || (fsa->host_status & HOST_ERROR_OFFLINE_T)) { sign = OFFLINE_SIGN; } else { sign = INFO_SIGN; } trans_log(sign, __FILE__, __LINE__, NULL, NULL, "Starting input queue that was stopped by init_afd."); event_log(0L, EC_HOST, ET_AUTO, EA_START_QUEUE, "%s", fsa->host_alias); } } /* if (fsa->error_counter > 0) */ #ifdef WITH_ERROR_QUEUE if (fsa->host_status & ERROR_QUEUE_SET) { remove_from_error_queue(db.job_id, fsa, db.fsa_pos, fsa_fd); } #endif if (fsa->host_status & HOST_ACTION_SUCCESS) { error_action(fsa->host_alias, "start", HOST_SUCCESS_ACTION); } p_file_name_buffer += MAX_FILENAME_LENGTH; p_file_size_buffer++; if (file_mtime_buffer != NULL) { p_file_mtime_buffer++; } } /* for (files_send = 0; files_send < files_to_send; files_send++) */ if (local_file_counter) { if (gsf_check_fsa(p_db) != NEITHER) { update_tfc(local_file_counter, local_file_size, p_file_size_buffer, files_to_send, files_send); local_file_size = 0; local_file_counter = 0; } } /* Do not forget to remove lock file if we have created one. */ if ((db.lock == LOCKFILE) && (fsa->active_transfers == 1)) { if (unlink(db.lock_file_name) == -1) { trans_log(ERROR_SIGN, __FILE__, __LINE__, NULL, NULL, "Failed to unlink() lock file `%s' : %s", db.lock_file_name, strerror(errno)); exit(REMOVE_LOCKFILE_ERROR); } else { if (fsa->debug > NORMAL_MODE) { trans_db_log(INFO_SIGN, __FILE__, __LINE__, NULL, "Removed lock file `%s'.", db.lock_file_name); } } } /* * Remove file directory with everything in it. */ if (rec_rmdir(file_path) == INCORRECT) { system_log(ERROR_SIGN, __FILE__, __LINE__, "Failed to rec_rmdir() `%s' : %s", file_path, strerror(errno)); exit_status = STILL_FILES_TO_SEND; } #ifdef _WITH_BURST_2 burst_2_counter++; } while ((cb2_ret = check_burst_2(file_path, &files_to_send, 0, # ifdef _WITH_INTERRUPT_JOB 0, # endif # ifdef _OUTPUT_LOG &ol_fd, # endif # ifndef AFDBENCH_CONFIG NULL, # endif NULL)) == YES); burst_2_counter--; if (cb2_ret == NEITHER) { exit_status = STILL_FILES_TO_SEND; } #endif /* _WITH_BURST_2 */ exitflag = 0; exit(exit_status); }
bool pkg_install(const fs::file& pkg_f, const std::string& dir, atomic_t<double>& sync, const std::string& pkg_filepath) { const std::size_t BUF_SIZE = 8192 * 1024; // 8 MB std::vector<fs::file> filelist; u32 cur_file=0; fs::file original_file(pkg_filepath); original_file.seek(pkg_f.pos()); filelist.push_back(std::move(original_file)); // Save current file offset (probably zero) const u64 start_offset = pkg_f.pos(); u64 cur_offset = start_offset; u64 cur_file_offset = start_offset; // Get basic PKG information PKGHeader header; auto archive_seek = [&](const s64 new_offset, const fs::seek_mode damode = fs::seek_set) { if(damode == fs::seek_set) cur_offset = new_offset; else if (damode == fs::seek_cur) cur_offset += new_offset; u64 _offset = 0; for (u32 i = 0; i < filelist.size(); i++) { if (cur_offset < (_offset + filelist[i].size())) { cur_file = i; cur_file_offset = cur_offset - _offset; filelist[i].seek(cur_file_offset); break; } _offset += filelist[i].size(); } }; auto archive_read = [&](const void *data_ptr, const u64 num_bytes) { u64 num_bytes_left = filelist[cur_file].size() - cur_file_offset; //check if it continues in another file if (num_bytes > num_bytes_left) { filelist[cur_file].read((u8 *)data_ptr, num_bytes_left); if ((cur_file + 1) < filelist.size()) cur_file++; else { cur_offset += num_bytes_left; cur_file_offset = filelist[cur_file].size(); return num_bytes_left; } u64 num_read = filelist[cur_file].read((u8 *)data_ptr + num_bytes_left, num_bytes - num_bytes_left); cur_offset += (num_read + num_bytes_left); cur_file_offset = num_read; return (num_read+num_bytes_left); } u64 num_read = filelist[cur_file].read((u8 *)data_ptr, num_bytes); cur_offset += num_read; cur_file_offset += num_read; return num_read; }; if (archive_read(&header, sizeof(header)) != sizeof(header)) { LOG_ERROR(LOADER, "PKG file is too short!"); return false; } if (header.pkg_magic != "\x7FPKG"_u32) { LOG_ERROR(LOADER, "Not a PKG file!"); return false; } switch (const u16 type = header.pkg_type) { case PKG_RELEASE_TYPE_DEBUG: break; case PKG_RELEASE_TYPE_RELEASE: break; default: { LOG_ERROR(LOADER, "Unknown PKG type (0x%x)", type); return false; } } switch (const u16 platform = header.pkg_platform) { case PKG_PLATFORM_TYPE_PS3: break; case PKG_PLATFORM_TYPE_PSP: break; default: { LOG_ERROR(LOADER, "Unknown PKG platform (0x%x)", platform); return false; } } if (header.pkg_size > pkg_f.size()) { //Check if multi-files pkg if (pkg_filepath.length() < 7 || pkg_filepath.substr(pkg_filepath.length() - 7).compare("_00.pkg") != 0) { LOG_ERROR(LOADER, "PKG file size mismatch (pkg_size=0x%llx)", header.pkg_size); return false; } std::string name_wo_number = pkg_filepath.substr(0, pkg_filepath.length() - 7); u64 cursize = pkg_f.size(); while (cursize != header.pkg_size) { std::string archive_filename = fmt::format("%s_%2d.pkg", name_wo_number, filelist.size()); fs::file archive_file(archive_filename); if (!archive_file) { LOG_ERROR(LOADER, "Missing part of the multi-files pkg: %s", archive_filename); return false; } cursize += archive_file.size(); filelist.push_back(std::move(archive_file)); } } if (header.data_size + header.data_offset > header.pkg_size) { LOG_ERROR(LOADER, "PKG data size mismatch (data_size=0x%llx, data_offset=0x%llx, file_size=0x%llx)", header.data_size, header.data_offset, header.pkg_size); return false; } be_t<u32> drm_type{0}; be_t<u32> content_type{0}; archive_seek(header.pkg_info_off); for (u32 i = 0; i < header.pkg_info_num; i++) { struct packet_T { be_t<u32> id; be_t<u32> size; } packet; archive_read(&packet, sizeof(packet)); // TODO switch (+packet.id) { case 0x1: { if (packet.size == sizeof(drm_type)) { archive_read(&drm_type, sizeof(drm_type)); continue; } break; } case 0x2: { if (packet.size == sizeof(content_type)) { archive_read(&content_type, sizeof(content_type)); continue; } break; } } archive_seek(packet.size, fs::seek_cur); } // Allocate buffer with BUF_SIZE size or more if required const std::unique_ptr<u128[]> buf(new u128[std::max<u64>(BUF_SIZE, sizeof(PKGEntry) * header.file_count) / sizeof(u128)]); // Define decryption subfunction (`psp` arg selects the key for specific block) auto decrypt = [&](u64 offset, u64 size, const uchar* key) -> u64 { archive_seek(start_offset + header.data_offset + offset); // Read the data and set available size const u64 read = archive_read(buf.get(), size); // Get block count const u64 blocks = (read + 15) / 16; if (header.pkg_type == PKG_RELEASE_TYPE_DEBUG) { // Debug key be_t<u64> input[8] = { header.qa_digest[0], header.qa_digest[0], header.qa_digest[1], header.qa_digest[1], }; for (u64 i = 0; i < blocks; i++) { // Initialize stream cipher for current position input[7] = offset / 16 + i; union sha1_hash { u8 data[20]; u128 _v128; } hash; sha1(reinterpret_cast<const u8*>(input), sizeof(input), hash.data); buf[i] ^= hash._v128; } } if (header.pkg_type == PKG_RELEASE_TYPE_RELEASE) { aes_context ctx; // Set encryption key for stream cipher aes_setkey_enc(&ctx, key, 128); // Initialize stream cipher for start position be_t<u128> input = header.klicensee.value() + offset / 16; // Increment stream position for every block for (u64 i = 0; i < blocks; i++, input++) { u128 key; aes_crypt_ecb(&ctx, AES_ENCRYPT, reinterpret_cast<const u8*>(&input), reinterpret_cast<u8*>(&key)); buf[i] ^= key; } } // Return the amount of data written in buf return read; }; std::array<uchar, 16> dec_key; if (header.pkg_platform == PKG_PLATFORM_TYPE_PSP && content_type >= 0x15 && content_type <= 0x17) { const uchar psp2t1[] = {0xE3, 0x1A, 0x70, 0xC9, 0xCE, 0x1D, 0xD7, 0x2B, 0xF3, 0xC0, 0x62, 0x29, 0x63, 0xF2, 0xEC, 0xCB}; const uchar psp2t2[] = {0x42, 0x3A, 0xCA, 0x3A, 0x2B, 0xD5, 0x64, 0x9F, 0x96, 0x86, 0xAB, 0xAD, 0x6F, 0xD8, 0x80, 0x1F}; const uchar psp2t3[] = {0xAF, 0x07, 0xFD, 0x59, 0x65, 0x25, 0x27, 0xBA, 0xF1, 0x33, 0x89, 0x66, 0x8B, 0x17, 0xD9, 0xEA}; aes_context ctx; aes_setkey_enc(&ctx, content_type == 0x15 ? psp2t1 : content_type == 0x16 ? psp2t2 : psp2t3, 128); aes_crypt_ecb(&ctx, AES_ENCRYPT, reinterpret_cast<const uchar*>(&header.klicensee), dec_key.data()); decrypt(0, header.file_count * sizeof(PKGEntry), dec_key.data()); } else { std::memcpy(dec_key.data(), PKG_AES_KEY, dec_key.size()); decrypt(0, header.file_count * sizeof(PKGEntry), header.pkg_platform == PKG_PLATFORM_TYPE_PSP ? PKG_AES_KEY2 : dec_key.data()); } std::vector<PKGEntry> entries(header.file_count); std::memcpy(entries.data(), buf.get(), entries.size() * sizeof(PKGEntry)); for (const auto& entry : entries) { const bool is_psp = (entry.type & PKG_FILE_ENTRY_PSP) != 0; if (entry.name_size > 256) { LOG_ERROR(LOADER, "PKG name size is too big (0x%x)", entry.name_size); continue; } decrypt(entry.name_offset, entry.name_size, is_psp ? PKG_AES_KEY2 : dec_key.data()); const std::string name(reinterpret_cast<char*>(buf.get()), entry.name_size); LOG_NOTICE(LOADER, "Entry 0x%08x: %s", entry.type, name); switch (entry.type & 0xff) { case PKG_FILE_ENTRY_NPDRM: case PKG_FILE_ENTRY_NPDRMEDAT: case PKG_FILE_ENTRY_SDAT: case PKG_FILE_ENTRY_REGULAR: case PKG_FILE_ENTRY_UNK1: case 0xe: case 0x10: case 0x11: case 0x13: case 0x15: case 0x16: { const std::string path = dir + name; const bool did_overwrite = fs::is_file(path); if (did_overwrite && (entry.type&PKG_FILE_ENTRY_OVERWRITE) == 0) { LOG_NOTICE(LOADER, "Didn't overwrite %s", name); break; } if (fs::file out{ path, fs::rewrite }) { for (u64 pos = 0; pos < entry.file_size; pos += BUF_SIZE) { const u64 block_size = std::min<u64>(BUF_SIZE, entry.file_size - pos); if (decrypt(entry.file_offset + pos, block_size, is_psp ? PKG_AES_KEY2 : dec_key.data()) != block_size) { LOG_ERROR(LOADER, "Failed to extract file %s", path); break; } if (out.write(buf.get(), block_size) != block_size) { LOG_ERROR(LOADER, "Failed to write file %s", path); break; } if (sync.fetch_add((block_size + 0.0) / header.data_size) < 0.) { LOG_ERROR(LOADER, "Package installation cancelled: %s", dir); return false; } } if (did_overwrite) { LOG_WARNING(LOADER, "Overwritten file %s", name); } else { LOG_NOTICE(LOADER, "Created file %s", name); } } else { LOG_ERROR(LOADER, "Failed to create file %s", path); } break; } case PKG_FILE_ENTRY_FOLDER: case 0x12: { const std::string path = dir + name; if (fs::create_dir(path)) { LOG_NOTICE(LOADER, "Created directory %s", name); } else if (fs::is_dir(path)) { LOG_WARNING(LOADER, "Reused existing directory %s", name); } else { LOG_ERROR(LOADER, "Failed to create directory %s", path); } break; } default: { LOG_ERROR(LOADER, "Unknown PKG entry type (0x%x) %s", entry.type, name); } } } LOG_SUCCESS(LOADER, "Package successfully installed to %s", dir); return true; }
/*$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ main() $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$*/ int main(int argc, char *argv[]) { #ifdef _WITH_BURST_2 int cb2_ret; #endif int current_toggle, exit_status = TRANSFER_SUCCESS, j, fd, status, loops, rest, blocksize, *wmo_counter, wmo_counter_fd = -1; #ifdef WITH_ARCHIVE_COPY_INFO unsigned int archived_copied = 0; #endif off_t no_of_bytes; time_t connected, #ifdef _WITH_BURST_2 diff_time, #endif end_transfer_time_file, start_transfer_time_file = 0, last_update_time, now; #ifdef _OUTPUT_LOG clock_t end_time = 0, start_time = 0; struct tms tmsdummy; #endif char *p_file_name_buffer, *buffer, fullname[MAX_PATH_LENGTH + 1], file_path[MAX_PATH_LENGTH]; clock_t clktck; struct stat stat_buf; struct job *p_db; #ifdef SA_FULLDUMP struct sigaction sact; #endif CHECK_FOR_VERSION(argc, argv); #ifdef SA_FULLDUMP /* * When dumping core sure we do a FULL core dump! */ sact.sa_handler = SIG_DFL; sact.sa_flags = SA_FULLDUMP; sigemptyset(&sact.sa_mask); if (sigaction(SIGSEGV, &sact, NULL) == -1) { system_log(ERROR_SIGN, __FILE__, __LINE__, "sigaction() error : %s", strerror(errno)); exit(INCORRECT); } #endif /* Do some cleanups when we exit. */ if (atexit(sf_wmo_exit) != 0) { system_log(ERROR_SIGN, __FILE__, __LINE__, "Could not register exit function : %s", strerror(errno)); exit(INCORRECT); } /* Initialise variables. */ local_file_counter = 0; files_to_send = init_sf(argc, argv, file_path, WMO_FLAG); p_db = &db; if ((clktck = sysconf(_SC_CLK_TCK)) <= 0) { system_log(ERROR_SIGN, __FILE__, __LINE__, "Could not get clock ticks per second : %s", strerror(errno)); exit(INCORRECT); } if (fsa->trl_per_process > 0) { if (fsa->trl_per_process < fsa->block_size) { blocksize = fsa->trl_per_process; } else { blocksize = fsa->block_size; } } else { blocksize = fsa->block_size; } if ((signal(SIGINT, sig_kill) == SIG_ERR) || (signal(SIGQUIT, sig_exit) == SIG_ERR) || (signal(SIGTERM, SIG_IGN) == SIG_ERR) || (signal(SIGSEGV, sig_segv) == SIG_ERR) || (signal(SIGBUS, sig_bus) == SIG_ERR) || (signal(SIGHUP, SIG_IGN) == SIG_ERR) || (signal(SIGPIPE, SIG_IGN) == SIG_ERR)) { system_log(ERROR_SIGN, __FILE__, __LINE__, "signal() error : %s", strerror(errno)); exit(INCORRECT); } /* Now determine the real hostname. */ if (db.toggle_host == YES) { if (fsa->host_toggle == HOST_ONE) { (void)strcpy(db.hostname, fsa->real_hostname[HOST_TWO - 1]); current_toggle = HOST_TWO; } else { (void)strcpy(db.hostname, fsa->real_hostname[HOST_ONE - 1]); current_toggle = HOST_ONE; } } else { (void)strcpy(db.hostname, fsa->real_hostname[(int)(fsa->host_toggle - 1)]); current_toggle = (int)fsa->host_toggle; } /* Connect to remote WMO-server. */ #ifdef FTP_CTRL_KEEP_ALIVE_INTERVAL if (fsa->protocol_options & AFD_TCP_KEEPALIVE) { timeout_flag = transfer_timeout - 5; if (timeout_flag < MIN_KEEP_ALIVE_INTERVAL) { timeout_flag = MIN_KEEP_ALIVE_INTERVAL; } } #else timeout_flag = OFF; #endif if ((status = wmo_connect(db.hostname, db.port, db.sndbuf_size)) != SUCCESS) { trans_log(ERROR_SIGN, __FILE__, __LINE__, NULL, NULL, "WMO connection to <%s> at port %d failed (%d).", db.hostname, db.port, status); exit(eval_timeout(CONNECT_ERROR)); } else { if (fsa->debug > NORMAL_MODE) { trans_db_log(INFO_SIGN, __FILE__, __LINE__, NULL, "Connected to port %d.", db.port); } } connected = time(NULL); /* Inform FSA that we have finished connecting and */ /* will now start to transfer data. */ if (gsf_check_fsa(p_db) != NEITHER) { #ifdef LOCK_DEBUG lock_region_w(fsa_fd, db.lock_offset + LOCK_CON, __FILE__, __LINE__); #else lock_region_w(fsa_fd, db.lock_offset + LOCK_CON); #endif fsa->job_status[(int)db.job_no].connect_status = WMO_ACTIVE; fsa->job_status[(int)db.job_no].no_of_files = files_to_send; fsa->connections += 1; #ifdef LOCK_DEBUG unlock_region(fsa_fd, db.lock_offset + LOCK_CON, __FILE__, __LINE__); #else unlock_region(fsa_fd, db.lock_offset + LOCK_CON); #endif } /* Allocate buffer to read data from the source file. */ if ((buffer = malloc(blocksize + 1 + 4 /* For bulletin end. */)) == NULL) { system_log(ERROR_SIGN, __FILE__, __LINE__, "malloc() error : %s", strerror(errno)); exit(ALLOC_ERROR); } if (db.special_flag & WITH_SEQUENCE_NUMBER) { char counter_file_name[MAX_FILENAME_LENGTH]; (void)snprintf(counter_file_name, MAX_FILENAME_LENGTH, "/%s.%d", db.host_alias, db.port); if ((wmo_counter_fd = open_counter_file(counter_file_name, &wmo_counter)) < 0) { system_log(ERROR_SIGN, __FILE__, __LINE__, "Failed to open counter file `%s'.", counter_file_name); } } #ifdef _WITH_BURST_2 do { if (burst_2_counter > 0) { if (fsa->debug > NORMAL_MODE) { trans_db_log(INFO_SIGN, __FILE__, __LINE__, NULL, "WMO Bursting."); } } #endif /* Send all files. */ p_file_name_buffer = file_name_buffer; p_file_size_buffer = file_size_buffer; last_update_time = time(NULL); local_file_size = 0; for (files_send = 0; files_send < files_to_send; files_send++) { (void)snprintf(fullname, MAX_PATH_LENGTH + 1, "%s/%s", file_path, p_file_name_buffer); if (*p_file_size_buffer > 0) { int end_length = 0, header_length = 0, length_type_indicator = 10; if (gsf_check_fsa(p_db) != NEITHER) { fsa->job_status[(int)db.job_no].file_size_in_use = *p_file_size_buffer; (void)strcpy(fsa->job_status[(int)db.job_no].file_name_in_use, p_file_name_buffer); } /* Open local file. */ #ifdef O_LARGEFILE if ((fd = open(fullname, O_RDONLY | O_LARGEFILE)) == -1) #else if ((fd = open(fullname, O_RDONLY)) == -1) #endif { trans_log(ERROR_SIGN, __FILE__, __LINE__, NULL, NULL, "Failed to open local file `%s' : %s", fullname, strerror(errno)); wmo_quit(); exit(OPEN_LOCAL_ERROR); } if (fsa->debug > NORMAL_MODE) { trans_db_log(INFO_SIGN, __FILE__, __LINE__, NULL, "Open local file `%s'", fullname); } #ifdef _OUTPUT_LOG if (db.output_log == YES) { start_time = times(&tmsdummy); } #endif /* * When the contents does not contain a bulletin header * it must be stored in the file name. */ if (db.special_flag & FILE_NAME_IS_HEADER) { int space_count; char *ptr = p_file_name_buffer; buffer[length_type_indicator] = 1; /* SOH */ buffer[length_type_indicator + 1] = '\015'; /* CR */ buffer[length_type_indicator + 2] = '\015'; /* CR */ buffer[length_type_indicator + 3] = '\012'; /* LF */ header_length = 4; space_count = 0; if (wmo_counter_fd > 0) { if (next_counter(wmo_counter_fd, wmo_counter, MAX_WMO_COUNTER) < 0) { close_counter_file(wmo_counter_fd, &wmo_counter); wmo_counter_fd = -1; wmo_counter = NULL; system_log(ERROR_SIGN, __FILE__, __LINE__, "Failed to get next WMO counter."); } else { if (*wmo_counter < 10) { buffer[length_type_indicator + header_length] = '0'; buffer[length_type_indicator + header_length + 1] = '0'; buffer[length_type_indicator + header_length + 2] = *wmo_counter + '0'; } else if (*wmo_counter < 100) { buffer[length_type_indicator + header_length] = '0'; buffer[length_type_indicator + header_length + 1] = (*wmo_counter / 10) + '0'; buffer[length_type_indicator + header_length + 2] = (*wmo_counter % 10) + '0'; } else if (*wmo_counter < 1000) { buffer[length_type_indicator + header_length] = ((*wmo_counter / 100) % 10) + '0'; buffer[length_type_indicator + header_length + 1] = ((*wmo_counter / 10) % 10) + '0'; buffer[length_type_indicator + header_length + 2] = (*wmo_counter % 10) + '0'; } buffer[length_type_indicator + header_length + 3] = '\015'; /* CR */ buffer[length_type_indicator + header_length + 4] = '\015'; /* CR */ buffer[length_type_indicator + header_length + 5] = '\012'; /* LF */ header_length += 6; } } /* if (wmo_counter_fd > 0) */ for (;;) { while ((*ptr != '_') && (*ptr != '-') && (*ptr != ' ') && (*ptr != '\0') && (*ptr != '.') && (*ptr != ';')) { buffer[length_type_indicator + header_length] = *ptr; header_length++; ptr++; } if ((*ptr == '\0') || (*ptr == '.') || (*ptr == ';')) { break; } else { if (space_count == 2) { if ((isalpha((int)(*(ptr + 1)))) && (isalpha((int)(*(ptr + 2)))) && (isalpha((int)(*(ptr + 3))))) { buffer[length_type_indicator + header_length] = ' '; buffer[length_type_indicator + header_length + 1] = *(ptr + 1); buffer[length_type_indicator + header_length + 2] = *(ptr + 2); buffer[length_type_indicator + header_length + 3] = *(ptr + 3); header_length += 4; } break; } else { buffer[length_type_indicator + header_length] = ' '; header_length++; ptr++; space_count++; } } } /* for (;;) */ buffer[length_type_indicator + header_length] = '\015'; /* CR */ buffer[length_type_indicator + header_length + 1] = '\015'; /* CR */ buffer[length_type_indicator + header_length + 2] = '\012'; /* LF */ header_length += 3; end_length = 4; } /* Read (local) and write (remote) file. */ no_of_bytes = 0; loops = (length_type_indicator + header_length + *p_file_size_buffer) / blocksize; rest = (length_type_indicator + header_length + *p_file_size_buffer) % blocksize; if ((db.special_flag & FILE_NAME_IS_HEADER) && (rest == 0)) { loops--; rest = blocksize; } /* Write length and type indicator. */ (void)snprintf(buffer, 9, "%08lu", (unsigned long)(*p_file_size_buffer + header_length + end_length)); if (db.transfer_mode == 'I') { buffer[length_type_indicator - 2] = 'B'; buffer[length_type_indicator - 1] = 'I'; } else if (db.transfer_mode == 'A') { buffer[length_type_indicator - 2] = 'A'; buffer[length_type_indicator - 1] = 'N'; } else { buffer[length_type_indicator - 2] = 'F'; buffer[length_type_indicator - 1] = 'X'; } if (fsa->trl_per_process > 0) { init_limit_transfer_rate(); } if (fsa->protocol_options & TIMEOUT_TRANSFER) { start_transfer_time_file = time(NULL); } for (;;) { for (j = 0; j < loops; j++) { #ifdef _SIMULATE_SLOW_TRANSFER (void)sleep(_SIMULATE_SLOW_TRANSFER); #endif if ((status = read(fd, (buffer + length_type_indicator + header_length), (blocksize - length_type_indicator - header_length))) != (blocksize - length_type_indicator - header_length)) { trans_log(ERROR_SIGN, __FILE__, __LINE__, NULL, NULL, "Could not read() local file `%s' : %s", fullname, strerror(errno)); wmo_quit(); exit(READ_LOCAL_ERROR); } if ((status = wmo_write(buffer, blocksize)) != SUCCESS) { trans_log(ERROR_SIGN, __FILE__, __LINE__, NULL, NULL, "Failed to write block from file `%s' to remote port %d [%d].", p_file_name_buffer, db.port, status); wmo_quit(); exit(eval_timeout(WRITE_REMOTE_ERROR)); } if (fsa->trl_per_process > 0) { limit_transfer_rate(blocksize, fsa->trl_per_process, clktck); } no_of_bytes += blocksize; if (gsf_check_fsa(p_db) != NEITHER) { fsa->job_status[(int)db.job_no].file_size_in_use_done = no_of_bytes; fsa->job_status[(int)db.job_no].file_size_done += blocksize; fsa->job_status[(int)db.job_no].bytes_send += blocksize; if (fsa->protocol_options & TIMEOUT_TRANSFER) { end_transfer_time_file = time(NULL); if (end_transfer_time_file < start_transfer_time_file) { start_transfer_time_file = end_transfer_time_file; } else { if ((end_transfer_time_file - start_transfer_time_file) > transfer_timeout) { trans_log(INFO_SIGN, __FILE__, __LINE__, NULL, NULL, #if SIZEOF_TIME_T == 4 "Transfer timeout reached for `%s' after %ld seconds.", #else "Transfer timeout reached for `%s' after %lld seconds.", #endif fsa->job_status[(int)db.job_no].file_name_in_use, (pri_time_t)(end_transfer_time_file - start_transfer_time_file)); wmo_quit(); exitflag = 0; exit(STILL_FILES_TO_SEND); } } } } if (length_type_indicator > 0) { length_type_indicator = 0; header_length = 0; } } /* for (j = 0; j < loops; j++) */ if (rest > 0) { if ((status = read(fd, (buffer + length_type_indicator + header_length), (rest - length_type_indicator - header_length))) != (rest - length_type_indicator - header_length)) { trans_log(ERROR_SIGN, __FILE__, __LINE__, NULL, NULL, "Could not read() local file `%s' : %s", fullname, strerror(errno)); wmo_quit(); exit(READ_LOCAL_ERROR); } if (end_length == 4) { buffer[rest] = '\015'; buffer[rest + 1] = '\015'; buffer[rest + 2] = '\012'; buffer[rest + 3] = 3; /* ETX */ } if ((status = wmo_write(buffer, rest + end_length)) != SUCCESS) { trans_log(ERROR_SIGN, __FILE__, __LINE__, NULL, NULL, "Failed to write rest of file to remote port %d [%d].", p_file_name_buffer, db.port, status); wmo_quit(); exit(eval_timeout(WRITE_REMOTE_ERROR)); } if (fsa->trl_per_process > 0) { limit_transfer_rate(rest + end_length, fsa->trl_per_process, clktck); } no_of_bytes += rest + end_length; if (gsf_check_fsa(p_db) != NEITHER) { fsa->job_status[(int)db.job_no].file_size_in_use_done = no_of_bytes; fsa->job_status[(int)db.job_no].file_size_done += rest; fsa->job_status[(int)db.job_no].bytes_send += rest; } } /* * Since there are always some users sending files to the * AFD not in dot notation, lets check here if this is really * the EOF. * If not lets continue so long until we hopefully have reached * the EOF. * NOTE: This is NOT a fool proof way. There must be a better * way! */ if (fstat(fd, &stat_buf) == -1) { (void)rec(transfer_log_fd, DEBUG_SIGN, "Hmmm. Failed to stat() `%s' : %s (%s %d)\n", fullname, strerror(errno), __FILE__, __LINE__); break; } else { if (stat_buf.st_size > *p_file_size_buffer) { char sign[LOG_SIGN_LENGTH]; if (db.special_flag & SILENT_NOT_LOCKED_FILE) { (void)memcpy(sign, DEBUG_SIGN, LOG_SIGN_LENGTH); } else { (void)memcpy(sign, WARN_SIGN, LOG_SIGN_LENGTH); } loops = (stat_buf.st_size - *p_file_size_buffer) / blocksize; rest = (stat_buf.st_size - *p_file_size_buffer) % blocksize; *p_file_size_buffer = stat_buf.st_size; /* * Give a warning in the receive log, so some action * can be taken against the originator. */ receive_log(sign, __FILE__, __LINE__, 0L, db.id.job, "File `%s' for host %s was DEFINITELY send without any locking. #%x", p_file_name_buffer, fsa->host_dsp_name, db.id.job); } else { break; } } } /* for (;;) */ if (db.special_flag & WMO_CHECK_ACKNOWLEDGE) { int ret; if ((ret = wmo_check_reply()) == INCORRECT) { trans_log(ERROR_SIGN, __FILE__, __LINE__, NULL, NULL, "Failed to receive reply from port %d for file %s.", db.port, p_file_name_buffer); wmo_quit(); exit(eval_timeout(CHECK_REPLY_ERROR)); } else if (ret == NEGATIV_ACKNOWLEDGE) { trans_log(ERROR_SIGN, __FILE__, __LINE__, NULL, NULL, "Received negative acknowledge from remote port %d for file %s.", db.port, p_file_name_buffer); } } #ifdef _OUTPUT_LOG if (db.output_log == YES) { end_time = times(&tmsdummy); } #endif /* Close local file. */ if (close(fd) == -1) { (void)rec(transfer_log_fd, WARN_SIGN, "%-*s[%d]: Failed to close() local file %s : %s (%s %d)\n", MAX_HOSTNAME_LENGTH, tr_hostname, (int)db.job_no, p_file_name_buffer, strerror(errno), __FILE__, __LINE__); /* * Since we usually do not send more then 100 files and * sf_wmo() will exit(), there is no point in stopping * the transmission. */ } } else { trans_log(INFO_SIGN, __FILE__, __LINE__, NULL, NULL, "File `%s' is of zero length, ignoring.", p_file_name_buffer); } /* Update FSA, one file transmitted. */ if (gsf_check_fsa(p_db) != NEITHER) { fsa->job_status[(int)db.job_no].file_name_in_use[0] = '\0'; fsa->job_status[(int)db.job_no].no_of_files_done++; fsa->job_status[(int)db.job_no].file_size_in_use = 0; fsa->job_status[(int)db.job_no].file_size_in_use_done = 0; local_file_size += *p_file_size_buffer; local_file_counter += 1; now = time(NULL); if (now >= (last_update_time + LOCK_INTERVAL_TIME)) { last_update_time = now; update_tfc(local_file_counter, local_file_size, p_file_size_buffer, files_to_send, files_send, now); local_file_size = 0; local_file_counter = 0; } } #ifdef _WITH_TRANS_EXEC if (db.special_flag & TRANS_EXEC) { trans_exec(file_path, fullname, p_file_name_buffer, clktck); } #endif #ifdef _OUTPUT_LOG if (db.output_log == YES) { if (ol_fd == -2) { # ifdef WITHOUT_FIFO_RW_SUPPORT output_log_fd(&ol_fd, &ol_readfd, &db.output_log); # else output_log_fd(&ol_fd, &db.output_log); # endif } if ((ol_fd > -1) && (ol_data == NULL)) { output_log_ptrs(&ol_retries, &ol_job_number, &ol_data, /* Pointer to buffer. */ &ol_file_name, &ol_file_name_length, &ol_archive_name_length, &ol_file_size, &ol_unl, &ol_size, &ol_transfer_time, &ol_output_type, db.host_alias, (current_toggle - 1), WMO, &db.output_log); } } #endif /* Now archive file if necessary. */ if ((db.archive_time > 0) && (p_db->archive_dir[0] != FAILED_TO_CREATE_ARCHIVE_DIR)) { #ifdef WITH_ARCHIVE_COPY_INFO int ret; #endif /* * By telling the function archive_file() that this * is the first time to archive a file for this job * (in struct p_db) it does not always have to check * whether the directory has been created or not. And * we ensure that we do not create duplicate names * when adding db.archive_time to msg_name. */ #ifdef WITH_ARCHIVE_COPY_INFO if ((ret = archive_file(file_path, p_file_name_buffer, p_db)) < 0) #else if (archive_file(file_path, p_file_name_buffer, p_db) < 0) #endif { if (fsa->debug > NORMAL_MODE) { trans_db_log(ERROR_SIGN, __FILE__, __LINE__, NULL, "Failed to archive file `%s'", p_file_name_buffer); } /* * NOTE: We _MUST_ delete the file we just send, * else the file directory will run full! */ if (unlink(fullname) == -1) { system_log(ERROR_SIGN, __FILE__, __LINE__, "Could not unlink() local file `%s' after sending it successfully : %s", fullname, strerror(errno)); } #ifdef _OUTPUT_LOG if (db.output_log == YES) { (void)memcpy(ol_file_name, db.p_unique_name, db.unl); (void)strcpy(ol_file_name + db.unl, p_file_name_buffer); *ol_file_name_length = (unsigned short)strlen(ol_file_name); ol_file_name[*ol_file_name_length] = SEPARATOR_CHAR; ol_file_name[*ol_file_name_length + 1] = '\0'; (*ol_file_name_length)++; *ol_file_size = *p_file_size_buffer; *ol_job_number = fsa->job_status[(int)db.job_no].job_id; *ol_retries = db.retries; *ol_unl = db.unl; *ol_transfer_time = end_time - start_time; *ol_archive_name_length = 0; *ol_output_type = OT_NORMAL_DELIVERED + '0'; ol_real_size = *ol_file_name_length + ol_size; if (write(ol_fd, ol_data, ol_real_size) != ol_real_size) { system_log(ERROR_SIGN, __FILE__, __LINE__, "write() error : %s", strerror(errno)); } } #endif } else { if (fsa->debug > NORMAL_MODE) { trans_db_log(INFO_SIGN, __FILE__, __LINE__, NULL, "Archived file `%s'", p_file_name_buffer); } #ifdef WITH_ARCHIVE_COPY_INFO if (ret == DATA_COPIED) { archived_copied++; } #endif #ifdef _OUTPUT_LOG if (db.output_log == YES) { (void)memcpy(ol_file_name, db.p_unique_name, db.unl); (void)strcpy(ol_file_name + db.unl, p_file_name_buffer); *ol_file_name_length = (unsigned short)strlen(ol_file_name); ol_file_name[*ol_file_name_length] = SEPARATOR_CHAR; ol_file_name[*ol_file_name_length + 1] = '\0'; (*ol_file_name_length)++; (void)strcpy(&ol_file_name[*ol_file_name_length + 1], &db.archive_dir[db.archive_offset]); *ol_file_size = *p_file_size_buffer; *ol_job_number = fsa->job_status[(int)db.job_no].job_id; *ol_retries = db.retries; *ol_unl = db.unl; *ol_transfer_time = end_time - start_time; *ol_archive_name_length = (unsigned short)strlen(&ol_file_name[*ol_file_name_length + 1]); *ol_output_type = OT_NORMAL_DELIVERED + '0'; ol_real_size = *ol_file_name_length + *ol_archive_name_length + 1 + ol_size; if (write(ol_fd, ol_data, ol_real_size) != ol_real_size) { system_log(ERROR_SIGN, __FILE__, __LINE__, "write() error : %s", strerror(errno)); } } #endif } } else { #ifdef WITH_UNLINK_DELAY int unlink_loops = 0; try_again_unlink: #endif /* Delete the file we just have send. */ if (unlink(fullname) == -1) { #ifdef WITH_UNLINK_DELAY if ((errno == EBUSY) && (unlink_loops < 20)) { (void)my_usleep(100000L); unlink_loops++; goto try_again_unlink; } #endif system_log(ERROR_SIGN, __FILE__, __LINE__, "Could not unlink() local file %s after sending it successfully : %s", fullname, strerror(errno)); } #ifdef _OUTPUT_LOG if (db.output_log == YES) { (void)memcpy(ol_file_name, db.p_unique_name, db.unl); (void)strcpy(ol_file_name + db.unl, p_file_name_buffer); *ol_file_name_length = (unsigned short)strlen(ol_file_name); ol_file_name[*ol_file_name_length] = SEPARATOR_CHAR; ol_file_name[*ol_file_name_length + 1] = '\0'; (*ol_file_name_length)++; *ol_file_size = *p_file_size_buffer; *ol_job_number = fsa->job_status[(int)db.job_no].job_id; *ol_retries = db.retries; *ol_unl = db.unl; *ol_transfer_time = end_time - start_time; *ol_archive_name_length = 0; *ol_output_type = OT_NORMAL_DELIVERED + '0'; ol_real_size = *ol_file_name_length + ol_size; if (write(ol_fd, ol_data, ol_real_size) != ol_real_size) { system_log(ERROR_SIGN, __FILE__, __LINE__, "write() error : %s", strerror(errno)); } } #endif } /* * After each successful transfer set error * counter to zero, so that other jobs can be * started. */ if (gsf_check_fsa(p_db) != NEITHER) { if ((*p_file_size_buffer > 0) && (fsa->error_counter > 0)) { int fd, #ifdef WITHOUT_FIFO_RW_SUPPORT readfd, #endif j; char fd_wake_up_fifo[MAX_PATH_LENGTH]; #ifdef LOCK_DEBUG lock_region_w(fsa_fd, db.lock_offset + LOCK_EC, __FILE__, __LINE__); #else lock_region_w(fsa_fd, db.lock_offset + LOCK_EC); #endif fsa->error_counter = 0; /* * Wake up FD! */ (void)snprintf(fd_wake_up_fifo, MAX_PATH_LENGTH, "%s%s%s", p_work_dir, FIFO_DIR, FD_WAKE_UP_FIFO); #ifdef WITHOUT_FIFO_RW_SUPPORT if (open_fifo_rw(fd_wake_up_fifo, &readfd, &fd) == -1) #else if ((fd = open(fd_wake_up_fifo, O_RDWR)) == -1) #endif { system_log(WARN_SIGN, __FILE__, __LINE__, "Failed to open() FIFO %s : %s", fd_wake_up_fifo, strerror(errno)); } else { if (write(fd, "", 1) != 1) { system_log(WARN_SIGN, __FILE__, __LINE__, "Failed to write() to FIFO %s : %s", fd_wake_up_fifo, strerror(errno)); } #ifdef WITHOUT_FIFO_RW_SUPPORT if (close(readfd) == -1) { system_log(DEBUG_SIGN, __FILE__, __LINE__, "Failed to close() FIFO %s (read) : %s", fd_wake_up_fifo, strerror(errno)); } #endif if (close(fd) == -1) { system_log(DEBUG_SIGN, __FILE__, __LINE__, "Failed to close() FIFO %s : %s", fd_wake_up_fifo, strerror(errno)); } } /* * Remove the error condition (NOT_WORKING) from all jobs * of this host. */ for (j = 0; j < fsa->allowed_transfers; j++) { if ((j != db.job_no) && (fsa->job_status[j].connect_status == NOT_WORKING)) { fsa->job_status[j].connect_status = DISCONNECT; } } fsa->error_history[0] = 0; fsa->error_history[1] = 0; #ifdef LOCK_DEBUG unlock_region(fsa_fd, db.lock_offset + LOCK_EC, __FILE__, __LINE__); #else unlock_region(fsa_fd, db.lock_offset + LOCK_EC); #endif #ifdef LOCK_DEBUG lock_region_w(fsa_fd, db.lock_offset + LOCK_HS, __FILE__, __LINE__); #else lock_region_w(fsa_fd, db.lock_offset + LOCK_HS); #endif now = time(NULL); if (now > fsa->end_event_handle) { fsa->host_status &= ~(EVENT_STATUS_FLAGS | AUTO_PAUSE_QUEUE_STAT); if (fsa->end_event_handle > 0L) { fsa->end_event_handle = 0L; } if (fsa->start_event_handle > 0L) { fsa->start_event_handle = 0L; } } else { fsa->host_status &= ~(EVENT_STATUS_STATIC_FLAGS | AUTO_PAUSE_QUEUE_STAT); } #ifdef LOCK_DEBUG unlock_region(fsa_fd, db.lock_offset + LOCK_HS, __FILE__, __LINE__); #else unlock_region(fsa_fd, db.lock_offset + LOCK_HS); #endif /* * Since we have successfully transmitted a file, no need to * have the queue stopped anymore. */ if (fsa->host_status & AUTO_PAUSE_QUEUE_STAT) { char sign[LOG_SIGN_LENGTH]; error_action(fsa->host_alias, "stop", HOST_ERROR_ACTION); event_log(0L, EC_HOST, ET_EXT, EA_ERROR_END, "%s", fsa->host_alias); if ((fsa->host_status & HOST_ERROR_OFFLINE_STATIC) || (fsa->host_status & HOST_ERROR_OFFLINE) || (fsa->host_status & HOST_ERROR_OFFLINE_T)) { (void)memcpy(sign, OFFLINE_SIGN, LOG_SIGN_LENGTH); } else { (void)memcpy(sign, INFO_SIGN, LOG_SIGN_LENGTH); } trans_log(sign, __FILE__, __LINE__, NULL, NULL, "Starting input queue that was stopped by init_afd."); event_log(0L, EC_HOST, ET_AUTO, EA_START_QUEUE, "%s", fsa->host_alias); } } /* if (fsa->error_counter > 0) */ #ifdef WITH_ERROR_QUEUE if (fsa->host_status & ERROR_QUEUE_SET) { remove_from_error_queue(db.id.job, fsa, db.fsa_pos, fsa_fd); } #endif if (fsa->host_status & HOST_ACTION_SUCCESS) { error_action(fsa->host_alias, "start", HOST_SUCCESS_ACTION); } } p_file_name_buffer += MAX_FILENAME_LENGTH; p_file_size_buffer++; } /* for (files_send = 0; files_send < files_to_send; files_send++) */ #ifdef WITH_ARCHIVE_COPY_INFO if (archived_copied > 0) { trans_log(DEBUG_SIGN, __FILE__, __LINE__, NULL, NULL, "Copied %u files to archive.", archived_copied); archived_copied = 0; } #endif if (local_file_counter) { if (gsf_check_fsa(p_db) != NEITHER) { update_tfc(local_file_counter, local_file_size, p_file_size_buffer, files_to_send, files_send, time(NULL)); local_file_size = 0; local_file_counter = 0; } } /* * Remove file directory, but only when all files have * been transmitted. */ if ((files_to_send == files_send) || (files_to_send < 1)) { if (rmdir(file_path) < 0) { system_log(ERROR_SIGN, __FILE__, __LINE__, "Failed to remove directory %s : %s", file_path, strerror(errno)); } } else { system_log(WARN_SIGN, __FILE__, __LINE__, "There are still %d files for %s. Will NOT remove this job!", files_to_send - files_send, file_path); exit_status = STILL_FILES_TO_SEND; } #ifdef _WITH_BURST_2 burst_2_counter++; diff_time = time(NULL) - connected; if (((fsa->protocol_options & KEEP_CONNECTED_DISCONNECT) && (db.keep_connected > 0) && (diff_time > db.keep_connected)) || ((db.disconnect > 0) && (diff_time > db.disconnect))) { cb2_ret = NO; break; } } while ((cb2_ret = check_burst_sf(file_path, &files_to_send, 0, # ifdef _WITH_INTERRUPT_JOB 0, # endif # ifdef _OUTPUT_LOG &ol_fd, # endif # ifndef AFDBENCH_CONFIG NULL, # endif NULL)) == YES); burst_2_counter--; if (cb2_ret == NEITHER) { exit_status = STILL_FILES_TO_SEND; } #endif /* _WITH_BURST_2 */ free(buffer); /* Disconnect from remote port. */ wmo_quit(); if ((fsa != NULL) && (fsa->debug > NORMAL_MODE)) { trans_db_log(INFO_SIGN, __FILE__, __LINE__, NULL, "Disconnected from port %d.", db.port); } if (wmo_counter_fd > 0) { close_counter_file(wmo_counter_fd, &wmo_counter); } exitflag = 0; exit(exit_status); }