/*########################### pop3_retrieve() ###########################*/ int pop3_retrieve(unsigned int msg_number, off_t *msg_size) { int reply; if ((reply = command(pop3_fd, "RETR %u", msg_number)) == SUCCESS) { if ((reply = get_reply()) == POP3_OK) { char *ptr; ptr = msg_str + 3; if (*ptr == ' ') { int i; char str_number[MAX_LONG_LENGTH + 1]; i = 0; ptr++; while (isdigit((int)(*(ptr + i))) && (i < MAX_LONG_LENGTH)) { str_number[i] = *(ptr + i); i++; } if (i <= MAX_LONG_LENGTH) { if (i > 0) { str_number[i] = '\0'; *msg_size = (off_t)str2offt(str_number, NULL, 8); } else { trans_log(WARN_SIGN, __FILE__, __LINE__, "pop3_retrieve", msg_str, _("Failed to get size from reply.")); *msg_size = 0; } } else { trans_log(WARN_SIGN, __FILE__, __LINE__, "pop3_retrieve", msg_str, _("Size in reply to large to store.")); *msg_size = 0; } } else { trans_log(WARN_SIGN, __FILE__, __LINE__, "pop3_retrieve", msg_str, _("Failed to get size from reply.")); *msg_size = 0; } } else { *msg_size = 0; } } return(reply); }
/*########################### scp_close_file() ##########################*/ int scp_close_file(void) { int status; if ((status = pipe_write(data_fd, "\0", 1)) != 1) { if (errno != 0) { trans_log(ERROR_SIGN, __FILE__, __LINE__, "scp_close_file", NULL, _("Failed to pipe_write() [close file] to pipe [%d] : %s"), status, strerror(errno)); } status = INCORRECT; } else { if (simulation_mode == YES) { status = SUCCESS; } else { status = get_ssh_reply(data_fd, YES); } } return(status); }
/*++++++++++++++++++++++++++++ sf_exec_exit() +++++++++++++++++++++++++++*/ static void sf_exec_exit(void) { if ((fsa != NULL) && (db.fsa_pos >= 0)) { int diff_no_of_files_done; u_off_t diff_file_size_done; if (local_file_counter) { if (gsf_check_fsa((struct job *)&db) != NEITHER) { update_tfc(local_file_counter, local_file_size, p_file_size_buffer, files_to_send, files_send); } } diff_no_of_files_done = fsa->job_status[(int)db.job_no].no_of_files_done - prev_no_of_files_done; diff_file_size_done = fsa->job_status[(int)db.job_no].file_size_done - prev_file_size_done; if ((diff_file_size_done > 0) || (diff_no_of_files_done > 0)) { int length; #ifdef _WITH_BURST_2 char buffer[MAX_INT_LENGTH + 5 + MAX_OFF_T_LENGTH + 24 + MAX_INT_LENGTH + 11 + MAX_INT_LENGTH + 1]; #else char buffer[MAX_INT_LENGTH + 5 + MAX_OFF_T_LENGTH + 24 + MAX_INT_LENGTH + 1]; #endif WHAT_DONE_BUFFER(length, buffer, "execed", diff_file_size_done, diff_no_of_files_done); #ifdef _WITH_BURST_2 if (burst_2_counter == 1) { (void)strcpy(&buffer[length], " [BURST]"); } else if (burst_2_counter > 1) { (void)sprintf(buffer + length, " [BURST * %u]", burst_2_counter); } #endif /* _WITH_BURST_2 */ trans_log(INFO_SIGN, NULL, 0, NULL, NULL, "%s #%x", buffer, db.job_id); } reset_fsa((struct job *)&db, exitflag, 0, 0); } free(file_name_buffer); free(file_size_buffer); send_proc_fin(NO); if (sys_log_fd != STDERR_FILENO) { (void)close(sys_log_fd); } return; }
/*$$$$$$$$$$$$$$$$$$$$$$$$$$ delete_wrapper() $$$$$$$$$$$$$$$$$$$$$$$$$$$*/ int delete_wrapper(char *file_name) { int ret; ret = sftp_dele(file_name); if (ret != SUCCESS) { trans_log(WARN_SIGN, __FILE__, __LINE__, NULL, msg_str, "Failed to delete remote file %s (%d).", file_name, ret); } return(ret); }
/*########################## scp_open_file() ############################*/ int scp_open_file(char *filename, off_t size, mode_t mode) { int status; size_t length; char cmd[MAX_PATH_LENGTH]; length = snprintf(cmd, MAX_PATH_LENGTH, #if SIZEOF_OFF_T == 4 "C%04o %ld %s\n", #else "C%04o %lld %s\n", #endif (mode & (S_ISUID|S_ISGID|S_IRWXU|S_IRWXG|S_IRWXO)), (pri_off_t)size, filename); if ((status = pipe_write(data_fd, cmd, length)) != length) { if (errno != 0) { cmd[length - 1] = '\0'; trans_log(ERROR_SIGN, __FILE__, __LINE__, "scp_open_file", NULL, _("Failed to pipe_write() `%s' to pipe [%d] : %s"), cmd, status, strerror(errno)); } status = INCORRECT; } else { if (simulation_mode == YES) { status = SUCCESS; } else { status = get_ssh_reply(data_fd, YES); } } return(status); }
/*+++++++++++++++++++++++++++++ check_list() ++++++++++++++++++++++++++++*/ static int check_list(char *file, off_t file_size, time_t file_mtime, int *files_to_retrieve, off_t *file_size_to_retrieve, int *more_files_in_list) { int i; if ((fra[db.fra_pos].stupid_mode == YES) || (fra[db.fra_pos].remove == YES)) { for (i = 0; i < no_of_listed_files; i++) { if (CHECK_STRCMP(rl[i].file_name, file) == 0) { rl[i].in_list = YES; if (((rl[i].assigned == 0) || (rl[i].retrieved == YES)) && (((db.special_flag & OLD_ERROR_JOB) == 0) || #ifdef LOCK_DEBUG (lock_region(rl_fd, (off_t)(LOCK_RETR_FILE + i), __FILE__, __LINE__) == LOCK_IS_NOT_SET) #else (lock_region(rl_fd, (off_t)(LOCK_RETR_FILE + i)) == LOCK_IS_NOT_SET) #endif )) { int ret; rl[i].file_mtime = file_mtime; rl[i].got_date = YES; rl[i].size = file_size; rl[i].prev_size = 0; if ((fra[db.fra_pos].ignore_size == -1) || ((fra[db.fra_pos].gt_lt_sign & ISIZE_EQUAL) && (fra[db.fra_pos].ignore_size == rl[i].size)) || ((fra[db.fra_pos].gt_lt_sign & ISIZE_LESS_THEN) && (fra[db.fra_pos].ignore_size < rl[i].size)) || ((fra[db.fra_pos].gt_lt_sign & ISIZE_GREATER_THEN) && (fra[db.fra_pos].ignore_size > rl[i].size))) { if (fra[db.fra_pos].ignore_file_time == 0) { *file_size_to_retrieve += rl[i].size; *files_to_retrieve += 1; #ifdef DO_NOT_PARALLELIZE_ALL_FETCH if ((fra[db.fra_pos].stupid_mode == YES) || (fra[db.fra_pos].remove == YES) || ((*files_to_retrieve < fra[db.fra_pos].max_copied_files) && (*file_size_to_retrieve < fra[db.fra_pos].max_copied_file_size))) #else if ((*files_to_retrieve < fra[db.fra_pos].max_copied_files) && (*file_size_to_retrieve < fra[db.fra_pos].max_copied_file_size)) #endif { rl[i].retrieved = NO; rl[i].assigned = (unsigned char)db.job_no + 1; } else { rl[i].assigned = 0; *file_size_to_retrieve -= rl[i].size; *files_to_retrieve -= 1; *more_files_in_list = YES; } ret = 0; } else { time_t diff_time; diff_time = current_time - rl[i].file_mtime; if (((fra[db.fra_pos].gt_lt_sign & IFTIME_EQUAL) && (fra[db.fra_pos].ignore_file_time == diff_time)) || ((fra[db.fra_pos].gt_lt_sign & IFTIME_LESS_THEN) && (fra[db.fra_pos].ignore_file_time < diff_time)) || ((fra[db.fra_pos].gt_lt_sign & IFTIME_GREATER_THEN) && (fra[db.fra_pos].ignore_file_time > diff_time))) { *file_size_to_retrieve += rl[i].size; *files_to_retrieve += 1; #ifdef DO_NOT_PARALLELIZE_ALL_FETCH if ((fra[db.fra_pos].stupid_mode == YES) || (fra[db.fra_pos].remove == YES) || ((*files_to_retrieve < fra[db.fra_pos].max_copied_files) && (*file_size_to_retrieve < fra[db.fra_pos].max_copied_file_size))) #else if ((*files_to_retrieve < fra[db.fra_pos].max_copied_files) && (*file_size_to_retrieve < fra[db.fra_pos].max_copied_file_size)) #endif { rl[i].retrieved = NO; rl[i].assigned = (unsigned char)db.job_no + 1; } else { rl[i].assigned = 0; *file_size_to_retrieve -= rl[i].size; *files_to_retrieve -= 1; *more_files_in_list = YES; } ret = 0; } else { ret = 1; } } #ifdef DEBUG_ASSIGNMENT trans_log(DEBUG_SIGN, __FILE__, __LINE__, NULL, NULL, # if SIZEOF_OFF_T == 4 "%s assigned %d: file_name=%s assigned=%d size=%ld", # else "%s assigned %d: file_name=%s assigned=%d size=%lld", # endif (fra[db.fra_pos].ls_data_alias[0] == '\0') ? fra[db.fra_pos].dir_alias : fra[db.fra_pos].ls_data_alias, i, rl[i].file_name, (int)rl[i].assigned, (pri_off_t)rl[i].size); #endif /* DEBUG_ASSIGNMENT */ } else { ret = 1; } if (db.special_flag & OLD_ERROR_JOB) { #ifdef LOCK_DEBUG unlock_region(rl_fd, (off_t)(LOCK_RETR_FILE + i), __FILE__, __LINE__); #else unlock_region(rl_fd, (off_t)(LOCK_RETR_FILE + i)); #endif } return(ret); } else { return(1); } } } /* for (i = 0; i < no_of_listed_files; i++) */ } else { /* Check if this file is in the list. */ for (i = 0; i < no_of_listed_files; i++) { if (CHECK_STRCMP(rl[i].file_name, file) == 0) { rl[i].in_list = YES; if ((rl[i].assigned != 0) || ((fra[db.fra_pos].stupid_mode == GET_ONCE_ONLY) && (rl[i].retrieved == YES))) { return(1); } if (((db.special_flag & OLD_ERROR_JOB) == 0) || #ifdef LOCK_DEBUG (lock_region(rl_fd, (off_t)(LOCK_RETR_FILE + i), __FILE__, __LINE__) == LOCK_IS_NOT_SET) #else (lock_region(rl_fd, (off_t)(LOCK_RETR_FILE + i)) == LOCK_IS_NOT_SET) #endif ) { int ret; off_t prev_size = 0; if (rl[i].file_mtime != file_mtime) { rl[i].file_mtime = file_mtime; rl[i].retrieved = NO; rl[i].assigned = 0; } rl[i].got_date = YES; if (rl[i].size != file_size) { prev_size = rl[i].size; rl[i].size = file_size; rl[i].retrieved = NO; rl[i].assigned = 0; } if (rl[i].retrieved == NO) { if ((fra[db.fra_pos].ignore_size == -1) || ((fra[db.fra_pos].gt_lt_sign & ISIZE_EQUAL) && (fra[db.fra_pos].ignore_size == rl[i].size)) || ((fra[db.fra_pos].gt_lt_sign & ISIZE_LESS_THEN) && (fra[db.fra_pos].ignore_size < rl[i].size)) || ((fra[db.fra_pos].gt_lt_sign & ISIZE_GREATER_THEN) && (fra[db.fra_pos].ignore_size > rl[i].size))) { off_t size_to_retrieve; if ((rl[i].got_date == NO) || (fra[db.fra_pos].ignore_file_time == 0)) { if ((fra[db.fra_pos].stupid_mode == APPEND_ONLY) && (rl[i].size > prev_size)) { size_to_retrieve = rl[i].size - prev_size; } else { size_to_retrieve = rl[i].size; } rl[i].prev_size = prev_size; #ifdef DO_NOT_PARALLELIZE_ALL_FETCH if ((fra[db.fra_pos].stupid_mode == YES) || (fra[db.fra_pos].remove == YES) || (((*files_to_retrieve + 1) < fra[db.fra_pos].max_copied_files) && ((*file_size_to_retrieve + size_to_retrieve) < fra[db.fra_pos].max_copied_file_size))) #else if (((*files_to_retrieve + 1) < fra[db.fra_pos].max_copied_files) && ((*file_size_to_retrieve + size_to_retrieve) < fra[db.fra_pos].max_copied_file_size)) #endif { rl[i].assigned = (unsigned char)db.job_no + 1; *file_size_to_retrieve += size_to_retrieve; *files_to_retrieve += 1; } else { rl[i].assigned = 0; *more_files_in_list = YES; } ret = 0; } else { time_t diff_time; diff_time = current_time - rl[i].file_mtime; if (((fra[db.fra_pos].gt_lt_sign & IFTIME_EQUAL) && (fra[db.fra_pos].ignore_file_time == diff_time)) || ((fra[db.fra_pos].gt_lt_sign & IFTIME_LESS_THEN) && (fra[db.fra_pos].ignore_file_time < diff_time)) || ((fra[db.fra_pos].gt_lt_sign & IFTIME_GREATER_THEN) && (fra[db.fra_pos].ignore_file_time > diff_time))) { if ((fra[db.fra_pos].stupid_mode == APPEND_ONLY) && (rl[i].size > prev_size)) { size_to_retrieve = rl[i].size - prev_size; } else { size_to_retrieve = rl[i].size; } rl[i].prev_size = prev_size; #ifdef DO_NOT_PARALLELIZE_ALL_FETCH if ((fra[db.fra_pos].stupid_mode == YES) || (fra[db.fra_pos].remove == YES) || (((*files_to_retrieve + 1) < fra[db.fra_pos].max_copied_files) && ((*file_size_to_retrieve + size_to_retrieve) < fra[db.fra_pos].max_copied_file_size))) #else if (((*files_to_retrieve + 1) < fra[db.fra_pos].max_copied_files) && ((*file_size_to_retrieve + size_to_retrieve) < fra[db.fra_pos].max_copied_file_size)) #endif { rl[i].assigned = (unsigned char)db.job_no + 1; *file_size_to_retrieve += size_to_retrieve; *files_to_retrieve += 1; } else { rl[i].assigned = 0; *more_files_in_list = YES; } ret = 0; } else { ret = 1; } } #ifdef DEBUG_ASSIGNMENT trans_log(DEBUG_SIGN, __FILE__, __LINE__, NULL, NULL, # if SIZEOF_OFF_T == 4 "%s assigned %d: file_name=%s assigned=%d size=%ld", # else "%s assigned %d: file_name=%s assigned=%d size=%lld", # endif (fra[db.fra_pos].ls_data_alias[0] == '\0') ? fra[db.fra_pos].dir_alias : fra[db.fra_pos].ls_data_alias, i, rl[i].file_name, (int)rl[i].assigned, (pri_off_t)rl[i].size); #endif /* DEBUG_ASSIGNMENT */ } else { ret = 1; } } else { ret = 1; } if (db.special_flag & OLD_ERROR_JOB) { #ifdef LOCK_DEBUG unlock_region(rl_fd, (off_t)(LOCK_RETR_FILE + i), __FILE__, __LINE__); #else unlock_region(rl_fd, (off_t)(LOCK_RETR_FILE + i)); #endif } return(ret); } else { return(1); } } } /* for (i = 0; i < no_of_listed_files; i++) */ } /* Add this file to the list. */ if ((no_of_listed_files != 0) && ((no_of_listed_files % RETRIEVE_LIST_STEP_SIZE) == 0)) { char *ptr; size_t new_size = (((no_of_listed_files / RETRIEVE_LIST_STEP_SIZE) + 1) * RETRIEVE_LIST_STEP_SIZE * sizeof(struct retrieve_list)) + AFD_WORD_OFFSET; ptr = (char *)rl - AFD_WORD_OFFSET; #ifdef DO_NOT_PARALLELIZE_ALL_FETCH if ((fra[db.fra_pos].stupid_mode == YES) || (fra[db.fra_pos].remove == YES)) { if ((ptr = realloc(ptr, new_size)) == NULL) { system_log(ERROR_SIGN, __FILE__, __LINE__, "realloc() error : %s", strerror(errno)); (void)ftp_quit(); exit(INCORRECT); } } else { #endif if ((ptr = mmap_resize(rl_fd, ptr, new_size)) == (caddr_t) -1) { system_log(ERROR_SIGN, __FILE__, __LINE__, "mmap_resize() error : %s", strerror(errno)); (void)ftp_quit(); exit(INCORRECT); } rl_size = new_size; #ifdef DO_NOT_PARALLELIZE_ALL_FETCH } #endif if (no_of_listed_files < 0) { system_log(DEBUG_SIGN, __FILE__, __LINE__, "Hmmm, no_of_listed_files = %d", no_of_listed_files); no_of_listed_files = 0; } *(int *)ptr = no_of_listed_files; ptr += AFD_WORD_OFFSET; rl = (struct retrieve_list *)ptr; } (void)strcpy(rl[no_of_listed_files].file_name, file); rl[no_of_listed_files].retrieved = NO; rl[no_of_listed_files].in_list = YES; rl[no_of_listed_files].size = file_size; rl[no_of_listed_files].prev_size = 0; rl[no_of_listed_files].file_mtime = file_mtime; rl[no_of_listed_files].got_date = YES; if ((fra[db.fra_pos].ignore_size == -1) || ((fra[db.fra_pos].gt_lt_sign & ISIZE_EQUAL) && (fra[db.fra_pos].ignore_size == rl[no_of_listed_files].size)) || ((fra[db.fra_pos].gt_lt_sign & ISIZE_LESS_THEN) && (fra[db.fra_pos].ignore_size < rl[no_of_listed_files].size)) || ((fra[db.fra_pos].gt_lt_sign & ISIZE_GREATER_THEN) && (fra[db.fra_pos].ignore_size > rl[no_of_listed_files].size))) { if ((rl[no_of_listed_files].got_date == NO) || (fra[db.fra_pos].ignore_file_time == 0)) { *file_size_to_retrieve += file_size; *files_to_retrieve += 1; no_of_listed_files++; } else { time_t diff_time; diff_time = current_time - rl[no_of_listed_files].file_mtime; if (((fra[db.fra_pos].gt_lt_sign & IFTIME_EQUAL) && (fra[db.fra_pos].ignore_file_time == diff_time)) || ((fra[db.fra_pos].gt_lt_sign & IFTIME_LESS_THEN) && (fra[db.fra_pos].ignore_file_time < diff_time)) || ((fra[db.fra_pos].gt_lt_sign & IFTIME_GREATER_THEN) && (fra[db.fra_pos].ignore_file_time > diff_time))) { *file_size_to_retrieve += file_size; *files_to_retrieve += 1; no_of_listed_files++; } else { return(1); } } #ifdef DO_NOT_PARALLELIZE_ALL_FETCH if ((fra[db.fra_pos].stupid_mode == YES) || (fra[db.fra_pos].remove == YES) || ((*files_to_retrieve < fra[db.fra_pos].max_copied_files) && (*file_size_to_retrieve < fra[db.fra_pos].max_copied_file_size))) #else if ((*files_to_retrieve < fra[db.fra_pos].max_copied_files) && (*file_size_to_retrieve < fra[db.fra_pos].max_copied_file_size)) #endif { rl[no_of_listed_files - 1].assigned = (unsigned char)db.job_no + 1; } else { rl[no_of_listed_files - 1].assigned = 0; *file_size_to_retrieve -= file_size; *files_to_retrieve -= 1; *more_files_in_list = YES; } *(int *)((char *)rl - AFD_WORD_OFFSET) = no_of_listed_files; #ifdef DEBUG_ASSIGNMENT trans_log(DEBUG_SIGN, __FILE__, __LINE__, NULL, NULL, # if SIZEOF_OFF_T == 4 "%s assigned %d: file_name=%s assigned=%d size=%ld", # else "%s assigned %d: file_name=%s assigned=%d size=%lld", # endif (fra[db.fra_pos].ls_data_alias[0] == '\0') ? fra[db.fra_pos].dir_alias : fra[db.fra_pos].ls_data_alias, i, rl[i].file_name, (int)rl[i].assigned, (pri_off_t)rl[i].size); #endif /* DEBUG_ASSIGNMENT */ return(0); } else { return(1); } }
/*$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ main() $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$*/ int main(int argc, char *argv[]) { #ifdef _WITH_BURST_2 int cb2_ret; #endif int exec_done, exit_status = TRANSFER_SUCCESS, ii, k, last_k, length, length_start, mask_file_name, #ifdef HAVE_SETPRIORITY sched_priority, #endif ret; time_t last_update_time, now, *p_file_mtime_buffer; char command_str[MAX_PATH_LENGTH + MAX_RECIPIENT_LENGTH], file_name[MAX_FILENAME_LENGTH], file_path[MAX_PATH_LENGTH], *fnp, *insert_list[MAX_EXEC_FILE_SUBSTITUTION], job_str[4], *p_command, *p_source_file, *p_file_name_buffer, *return_str = NULL, source_file[MAX_PATH_LENGTH], tmp_char, tmp_option[MAX_PATH_LENGTH + MAX_RECIPIENT_LENGTH]; struct job *p_db; #ifdef SA_FULLDUMP struct sigaction sact; #endif #ifdef WITH_FAST_MOVE nlink_t nlink; #endif #ifdef _OUTPUT_LOG clock_t end_time = 0, start_time = 0; struct tms tmsdummy; #endif CHECK_FOR_VERSION(argc, argv); #ifdef SA_FULLDUMP /* * When dumping core sure we do a FULL core dump! */ sact.sa_handler = SIG_DFL; sact.sa_flags = SA_FULLDUMP; sigemptyset(&sact.sa_mask); if (sigaction(SIGSEGV, &sact, NULL) == -1) { system_log(ERROR_SIGN, __FILE__, __LINE__, "sigaction() error : %s", strerror(errno)); exit(INCORRECT); } #endif /* Do some cleanups when we exit. */ if (atexit(sf_exec_exit) != 0) { system_log(ERROR_SIGN, __FILE__, __LINE__, "Could not register exit function : %s", strerror(errno)); exit(INCORRECT); } /* Initialise variables. */ local_file_counter = 0; files_to_send = init_sf(argc, argv, file_path, EXEC_FLAG); p_db = &db; if ((signal(SIGINT, sig_kill) == SIG_ERR) || (signal(SIGQUIT, sig_exit) == SIG_ERR) || (signal(SIGTERM, sig_exit) == SIG_ERR) || (signal(SIGSEGV, sig_segv) == SIG_ERR) || (signal(SIGBUS, sig_bus) == SIG_ERR) || (signal(SIGHUP, SIG_IGN) == SIG_ERR)) { system_log(ERROR_SIGN, __FILE__, __LINE__, "Failed to set signal handlers : %s", strerror(errno)); exit(INCORRECT); } /* Inform FSA that we have are ready to copy the files. */ if (gsf_check_fsa(p_db) != NEITHER) { fsa->job_status[(int)db.job_no].connect_status = EXEC_ACTIVE; fsa->job_status[(int)db.job_no].no_of_files = files_to_send; } /* Init job_str for exec_cmd(). */ job_str[0] = '['; job_str[1] = db.job_no + '0'; job_str[2] = ']'; job_str[3] = '\0'; #ifdef _WITH_BURST_2 do { if (burst_2_counter > 0) { if (fsa->debug > NORMAL_MODE) { trans_db_log(INFO_SIGN, __FILE__, __LINE__, NULL, "Bursting."); } } #endif /* _WITH_BURST_2 */ /* Prepare pointers and directory name. */ (void)strcpy(source_file, file_path); p_source_file = source_file + strlen(source_file); *p_source_file++ = '/'; #ifdef HAVE_SETPRIORITY if (db.exec_base_priority != NO_PRIORITY) { sched_priority = db.exec_base_priority; if (db.add_afd_priority == YES) { sched_priority += (int)(fsa->job_status[(int)db.job_no].unique_name[MAX_MSG_NAME_LENGTH - 1]); if (sched_priority > db.min_sched_priority) { sched_priority = db.min_sched_priority; } else if (sched_priority < db.max_sched_priority) { sched_priority = db.max_sched_priority; } } if ((sched_priority == db.current_priority) || ((db.current_priority > sched_priority) && (geteuid() != 0))) { sched_priority = NO_PRIORITY; } } else { sched_priority = NO_PRIORITY; } #endif p_command = db.exec_cmd; while ((*p_command == ' ') || (*p_command == '\t')) { p_command++; } if ((db.special_flag & EXEC_ONCE_ONLY) == 0) { char *p_end; /* Prepare insert list. */ p_end = p_command; k = 0; ii = 0; while ((*p_end != '\0') && (ii < MAX_EXEC_FILE_SUBSTITUTION)) { if ((*p_end == '%') && (*(p_end + 1) == 's')) { tmp_option[k] = *p_end; tmp_option[k + 1] = *(p_end + 1); insert_list[ii] = &tmp_option[k]; ii++; p_end += 2; k += 2; } else { tmp_option[k] = *p_end; p_end++; k++; } } if (ii >= MAX_EXEC_FILE_SUBSTITUTION) { trans_log(WARN_SIGN, __FILE__, __LINE__, NULL, NULL, "To many %%s in pexec option. Can oly handle %d.", MAX_EXEC_FILE_SUBSTITUTION); } tmp_option[k] = '\0'; p_command = tmp_option; last_k = k; } p_file_name_buffer = file_name_buffer; p_file_size_buffer = file_size_buffer; p_file_mtime_buffer = file_mtime_buffer; last_update_time = time(NULL); local_file_size = 0; exec_done = NO; for (files_send = 0; files_send < files_to_send; files_send++) { (void)strcpy(p_source_file, p_file_name_buffer); if (db.special_flag & EXEC_ONCE_ONLY) { if (exec_done == NO) { #ifdef _OUTPUT_LOG if (db.output_log == YES) { start_time = times(&tmsdummy); } #endif (void)sprintf(command_str, "cd %s && %s", file_path, p_command); if (fsa->debug > NORMAL_MODE) { trans_db_log(INFO_SIGN, __FILE__, __LINE__, NULL, "Executing command `%s' to send file `%s'", command_str, p_file_name_buffer); } if ((ret = exec_cmd(command_str, &return_str, transfer_log_fd, fsa->host_dsp_name, MAX_HOSTNAME_LENGTH, #ifdef HAVE_SETPRIORITY sched_priority, #endif job_str, (fsa->protocol_options & TIMEOUT_TRANSFER) ? (time_t)transfer_timeout : 0L, YES, YES)) != 0) { trans_log(ERROR_SIGN, __FILE__, __LINE__, NULL, NULL, "Failed to execute command %s [Return code = %d]", command_str, ret); if ((return_str != NULL) && (return_str[0] != '\0')) { char *end_ptr = return_str, *start_ptr; do { start_ptr = end_ptr; while ((*end_ptr != '\n') && (*end_ptr != '\0')) { end_ptr++; } if (*end_ptr == '\n') { *end_ptr = '\0'; end_ptr++; } trans_log(ERROR_SIGN, __FILE__, __LINE__, NULL, NULL, "%s", start_ptr); } while (*end_ptr != '\0'); } exit(EXEC_ERROR); } #ifdef _OUTPUT_LOG if (db.output_log == YES) { end_time = times(&tmsdummy); } #endif free(return_str); return_str = NULL; exec_done = YES; } } else { /* Write status to FSA? */ if (gsf_check_fsa(p_db) != NEITHER) { fsa->job_status[(int)db.job_no].file_size_in_use = *p_file_size_buffer; (void)strcpy(fsa->job_status[(int)db.job_no].file_name_in_use, p_file_name_buffer); } #ifdef _OUTPUT_LOG if (db.output_log == YES) { start_time = times(&tmsdummy); } #endif insert_list[ii] = &tmp_option[last_k]; tmp_char = *insert_list[0]; *insert_list[0] = '\0'; length_start = sprintf(command_str, "cd %s && %s", file_path, p_command); *insert_list[0] = tmp_char; fnp = p_file_name_buffer; mask_file_name = NO; do { if ((*fnp == ';') || (*fnp == ' ')) { mask_file_name = YES; break; } fnp++; } while (*fnp != '\0'); /* Generate command string with file name(s). */ length = 0; for (k = 1; k < (ii + 1); k++) { tmp_char = *insert_list[k]; *insert_list[k] = '\0'; if (mask_file_name == YES) { length += sprintf(command_str + length_start + length, "\"%s\"%s", p_file_name_buffer, insert_list[k - 1] + 2); } else { length += sprintf(command_str + length_start + length, "%s%s", p_file_name_buffer, insert_list[k - 1] + 2); } *insert_list[k] = tmp_char; } if ((ret = exec_cmd(command_str, &return_str, transfer_log_fd, fsa->host_dsp_name, MAX_HOSTNAME_LENGTH, #ifdef HAVE_SETPRIORITY sched_priority, #endif job_str, (fsa->protocol_options & TIMEOUT_TRANSFER) ? (time_t)transfer_timeout : 0L, YES, YES)) != 0) /* ie != SUCCESS */ { trans_log(ERROR_SIGN, __FILE__, __LINE__, NULL, NULL, "Failed to execute command %s [Return code = %d]", command_str, ret); if ((return_str != NULL) && (return_str[0] != '\0')) { char *end_ptr = return_str, *start_ptr; do { start_ptr = end_ptr; while ((*end_ptr != '\n') && (*end_ptr != '\0')) { end_ptr++; } if (*end_ptr == '\n') { *end_ptr = '\0'; end_ptr++; } trans_log(ERROR_SIGN, __FILE__, __LINE__, NULL, NULL, "%s", start_ptr); } while (*end_ptr != '\0'); } exit(EXEC_ERROR); } free(return_str); return_str = NULL; #ifdef _OUTPUT_LOG if (db.output_log == YES) { end_time = times(&tmsdummy); } #endif } /* Tell FSA we have done a file. */ if (gsf_check_fsa(p_db) != NEITHER) { fsa->job_status[(int)db.job_no].file_name_in_use[0] = '\0'; fsa->job_status[(int)db.job_no].no_of_files_done++; fsa->job_status[(int)db.job_no].file_size_in_use = 0; fsa->job_status[(int)db.job_no].file_size_in_use_done = 0; fsa->job_status[(int)db.job_no].file_size_done += *p_file_size_buffer; fsa->job_status[(int)db.job_no].bytes_send += *p_file_size_buffer; local_file_size += *p_file_size_buffer; local_file_counter += 1; now = time(NULL); if (now >= (last_update_time + LOCK_INTERVAL_TIME)) { last_update_time = now; update_tfc(local_file_counter, local_file_size, p_file_size_buffer, files_to_send, files_send); local_file_size = 0; local_file_counter = 0; } } #ifdef _WITH_TRANS_EXEC if (db.special_flag & TRANS_EXEC) { trans_exec(file_path, source_file, p_file_name_buffer); } #endif #ifdef _OUTPUT_LOG if (db.output_log == YES) { if (ol_fd == -2) { # ifdef WITHOUT_FIFO_RW_SUPPORT output_log_fd(&ol_fd, &ol_readfd); # else output_log_fd(&ol_fd); # endif } if ((ol_fd > -1) && (ol_data == NULL)) { output_log_ptrs(&ol_retries, &ol_job_number, &ol_data, &ol_file_name, &ol_file_name_length, &ol_archive_name_length, &ol_file_size, &ol_unl, &ol_size, &ol_transfer_time, &ol_output_type, db.host_alias, 0, EXEC); } } #endif /* Now archive file if necessary. */ if ((db.archive_time > 0) && (p_db->archive_dir[0] != FAILED_TO_CREATE_ARCHIVE_DIR)) { /* * By telling the function archive_file() that this * is the first time to archive a file for this job * (in struct p_db) it does not always have to check * whether the directory has been created or not. And * we ensure that we do not create duplicate names * when adding db.archive_time to msg_name. */ if (archive_file(file_path, p_file_name_buffer, p_db) < 0) { trans_log(ERROR_SIGN, __FILE__, __LINE__, NULL, NULL, "Failed to archive file `%s'", file_name); /* * NOTE: We _MUST_ delete the file we just send, * else the file directory will run full! */ if (unlink(source_file) == -1) { system_log(ERROR_SIGN, __FILE__, __LINE__, "Could not unlink() local file `%s' after copying it successfully : %s", source_file, strerror(errno)); } #ifdef _OUTPUT_LOG if (db.output_log == YES) { (void)memcpy(ol_file_name, db.p_unique_name, db.unl); if (db.trans_rename_rule[0] != '\0') { *ol_file_name_length = (unsigned short)sprintf(ol_file_name + db.unl, "%s%c/%s", p_file_name_buffer, SEPARATOR_CHAR, p_file_name_buffer) + db.unl; } else { (void)strcpy(ol_file_name + db.unl, p_file_name_buffer); *ol_file_name_length = (unsigned short)strlen(ol_file_name); ol_file_name[*ol_file_name_length] = SEPARATOR_CHAR; ol_file_name[*ol_file_name_length + 1] = '\0'; (*ol_file_name_length)++; } *ol_file_size = *p_file_size_buffer; *ol_job_number = fsa->job_status[(int)db.job_no].job_id; *ol_retries = db.retries; *ol_unl = db.unl; *ol_transfer_time = end_time - start_time; *ol_archive_name_length = 0; *ol_output_type = OT_NORMAL_DELIVERED + '0'; ol_real_size = *ol_file_name_length + ol_size; if (write(ol_fd, ol_data, ol_real_size) != ol_real_size) { system_log(ERROR_SIGN, __FILE__, __LINE__, "write() error : %s", strerror(errno)); } } #endif } else { if (fsa->debug > NORMAL_MODE) { trans_db_log(INFO_SIGN, __FILE__, __LINE__, NULL, "Archived file `%s'.", file_name); } #ifdef _OUTPUT_LOG if (db.output_log == YES) { (void)memcpy(ol_file_name, db.p_unique_name, db.unl); if (db.trans_rename_rule[0] != '\0') { *ol_file_name_length = (unsigned short)sprintf(ol_file_name + db.unl, "%s%c/%s", p_file_name_buffer, SEPARATOR_CHAR, p_file_name_buffer) + db.unl; } else { (void)strcpy(ol_file_name + db.unl, p_file_name_buffer); *ol_file_name_length = (unsigned short)strlen(ol_file_name); ol_file_name[*ol_file_name_length] = SEPARATOR_CHAR; ol_file_name[*ol_file_name_length + 1] = '\0'; (*ol_file_name_length)++; } (void)strcpy(&ol_file_name[*ol_file_name_length + 1], &db.archive_dir[db.archive_offset]); *ol_file_size = *p_file_size_buffer; *ol_job_number = fsa->job_status[(int)db.job_no].job_id; *ol_retries = db.retries; *ol_unl = db.unl; *ol_transfer_time = end_time - start_time; *ol_archive_name_length = (unsigned short)strlen(&ol_file_name[*ol_file_name_length + 1]); *ol_output_type = OT_NORMAL_DELIVERED + '0'; ol_real_size = *ol_file_name_length + *ol_archive_name_length + 1 + ol_size; if (write(ol_fd, ol_data, ol_real_size) != ol_real_size) { system_log(ERROR_SIGN, __FILE__, __LINE__, "write() error : %s", strerror(errno)); } } #endif } } else { #ifdef WITH_UNLINK_DELAY int unlink_loops = 0; try_again_unlink: #endif /* Delete the file we just have executed. */ if (unlink(source_file) == -1) { #ifdef WITH_UNLINK_DELAY if ((errno == EBUSY) && (unlink_loops < 20)) { (void)my_usleep(100000L); unlink_loops++; goto try_again_unlink; } #endif system_log(ERROR_SIGN, __FILE__, __LINE__, "Could not unlink() local file %s after copying it successfully : %s", source_file, strerror(errno)); } #ifdef _OUTPUT_LOG if (db.output_log == YES) { (void)memcpy(ol_file_name, db.p_unique_name, db.unl); if (db.trans_rename_rule[0] != '\0') { *ol_file_name_length = (unsigned short)sprintf(ol_file_name + db.unl, "%s%c/%s", p_file_name_buffer, SEPARATOR_CHAR, p_file_name_buffer) + db.unl; } else { (void)strcpy(ol_file_name + db.unl, p_file_name_buffer); *ol_file_name_length = (unsigned short)strlen(ol_file_name); ol_file_name[*ol_file_name_length] = SEPARATOR_CHAR; ol_file_name[*ol_file_name_length + 1] = '\0'; (*ol_file_name_length)++; } *ol_file_size = *p_file_size_buffer; *ol_job_number = fsa->job_status[(int)db.job_no].job_id; *ol_retries = db.retries; *ol_unl = db.unl; *ol_transfer_time = end_time - start_time; *ol_archive_name_length = 0; *ol_output_type = OT_NORMAL_DELIVERED + '0'; ol_real_size = *ol_file_name_length + ol_size; if (write(ol_fd, ol_data, ol_real_size) != ol_real_size) { system_log(ERROR_SIGN, __FILE__, __LINE__, "write() error : %s", strerror(errno)); } } #endif } /* * After each successful transfer set error counter to zero, * so that other jobs can be started. */ if (fsa->error_counter > 0) { int fd, #ifdef WITHOUT_FIFO_RW_SUPPORT readfd, #endif j; char fd_wake_up_fifo[MAX_PATH_LENGTH]; #ifdef LOCK_DEBUG lock_region_w(fsa_fd, db.lock_offset + LOCK_EC, __FILE__, __LINE__); #else lock_region_w(fsa_fd, db.lock_offset + LOCK_EC); #endif fsa->error_counter = 0; /* * Wake up FD! */ (void)sprintf(fd_wake_up_fifo, "%s%s%s", p_work_dir, FIFO_DIR, FD_WAKE_UP_FIFO); #ifdef WITHOUT_FIFO_RW_SUPPORT if (open_fifo_rw(fd_wake_up_fifo, &readfd, &fd) == -1) #else if ((fd = open(fd_wake_up_fifo, O_RDWR)) == -1) #endif { system_log(WARN_SIGN, __FILE__, __LINE__, "Failed to open() FIFO %s : %s", fd_wake_up_fifo, strerror(errno)); } else { if (write(fd, "", 1) != 1) { system_log(WARN_SIGN, __FILE__, __LINE__, "Failed to write() to FIFO %s : %s", fd_wake_up_fifo, strerror(errno)); } #ifdef WITHOUT_FIFO_RW_SUPPORT if (close(readfd) == -1) { system_log(DEBUG_SIGN, __FILE__, __LINE__, "Failed to close() FIFO %s : %s", fd_wake_up_fifo, strerror(errno)); } #endif if (close(fd) == -1) { system_log(DEBUG_SIGN, __FILE__, __LINE__, "Failed to close() FIFO %s : %s", fd_wake_up_fifo, strerror(errno)); } } /* * Remove the error condition (NOT_WORKING) from all jobs * of this host. */ for (j = 0; j < fsa->allowed_transfers; j++) { if ((j != db.job_no) && (fsa->job_status[j].connect_status == NOT_WORKING)) { fsa->job_status[j].connect_status = DISCONNECT; } } fsa->error_history[0] = 0; fsa->error_history[1] = 0; #ifdef LOCK_DEBUG unlock_region(fsa_fd, db.lock_offset + LOCK_EC, __FILE__, __LINE__); #else unlock_region(fsa_fd, db.lock_offset + LOCK_EC); #endif /* * Since we have successfully transmitted a file, no need to * have the queue stopped anymore. */ if (fsa->host_status & AUTO_PAUSE_QUEUE_STAT) { char *sign; #ifdef LOCK_DEBUG lock_region_w(fsa_fd, db.lock_offset + LOCK_HS, __FILE__, __LINE__); #else lock_region_w(fsa_fd, db.lock_offset + LOCK_HS); #endif fsa->host_status &= ~AUTO_PAUSE_QUEUE_STAT; if (fsa->host_status & HOST_ERROR_EA_STATIC) { fsa->host_status &= ~EVENT_STATUS_STATIC_FLAGS; } else { fsa->host_status &= ~EVENT_STATUS_FLAGS; } fsa->host_status &= ~PENDING_ERRORS; #ifdef LOCK_DEBUG unlock_region(fsa_fd, db.lock_offset + LOCK_HS, __FILE__, __LINE__); #else unlock_region(fsa_fd, db.lock_offset + LOCK_HS); #endif error_action(fsa->host_alias, "stop", HOST_ERROR_ACTION); event_log(0L, EC_HOST, ET_EXT, EA_ERROR_END, "%s", fsa->host_alias); if ((fsa->host_status & HOST_ERROR_OFFLINE_STATIC) || (fsa->host_status & HOST_ERROR_OFFLINE) || (fsa->host_status & HOST_ERROR_OFFLINE_T)) { sign = OFFLINE_SIGN; } else { sign = INFO_SIGN; } trans_log(sign, __FILE__, __LINE__, NULL, NULL, "Starting input queue that was stopped by init_afd."); event_log(0L, EC_HOST, ET_AUTO, EA_START_QUEUE, "%s", fsa->host_alias); } } /* if (fsa->error_counter > 0) */ #ifdef WITH_ERROR_QUEUE if (fsa->host_status & ERROR_QUEUE_SET) { remove_from_error_queue(db.job_id, fsa, db.fsa_pos, fsa_fd); } #endif if (fsa->host_status & HOST_ACTION_SUCCESS) { error_action(fsa->host_alias, "start", HOST_SUCCESS_ACTION); } p_file_name_buffer += MAX_FILENAME_LENGTH; p_file_size_buffer++; if (file_mtime_buffer != NULL) { p_file_mtime_buffer++; } } /* for (files_send = 0; files_send < files_to_send; files_send++) */ if (local_file_counter) { if (gsf_check_fsa(p_db) != NEITHER) { update_tfc(local_file_counter, local_file_size, p_file_size_buffer, files_to_send, files_send); local_file_size = 0; local_file_counter = 0; } } /* Do not forget to remove lock file if we have created one. */ if ((db.lock == LOCKFILE) && (fsa->active_transfers == 1)) { if (unlink(db.lock_file_name) == -1) { trans_log(ERROR_SIGN, __FILE__, __LINE__, NULL, NULL, "Failed to unlink() lock file `%s' : %s", db.lock_file_name, strerror(errno)); exit(REMOVE_LOCKFILE_ERROR); } else { if (fsa->debug > NORMAL_MODE) { trans_db_log(INFO_SIGN, __FILE__, __LINE__, NULL, "Removed lock file `%s'.", db.lock_file_name); } } } /* * Remove file directory with everything in it. */ if (rec_rmdir(file_path) == INCORRECT) { system_log(ERROR_SIGN, __FILE__, __LINE__, "Failed to rec_rmdir() `%s' : %s", file_path, strerror(errno)); exit_status = STILL_FILES_TO_SEND; } #ifdef _WITH_BURST_2 burst_2_counter++; } while ((cb2_ret = check_burst_2(file_path, &files_to_send, 0, # ifdef _WITH_INTERRUPT_JOB 0, # endif # ifdef _OUTPUT_LOG &ol_fd, # endif # ifndef AFDBENCH_CONFIG NULL, # endif NULL)) == YES); burst_2_counter--; if (cb2_ret == NEITHER) { exit_status = STILL_FILES_TO_SEND; } #endif /* _WITH_BURST_2 */ exitflag = 0; exit(exit_status); }
/*############################## scp_quit() #############################*/ void scp_quit(void) { /* Close pipe for read/write data connection. */ if (data_fd != -1) { if (close(data_fd) == -1) { trans_log(WARN_SIGN, __FILE__, __LINE__, "scp_quit", NULL, _("Failed to close() write pipe to ssh process : %s"), strerror(errno)); } data_fd = -1; } /* Remove ssh process for writing data. */ if (data_pid > 0) { int loop_counter, max_waitpid_loops, status; pid_t return_pid; loop_counter = 0; max_waitpid_loops = (transfer_timeout / 2) * 10; while (((return_pid = waitpid(data_pid, &status, WNOHANG)) != data_pid) && (return_pid != -1) && (loop_counter < max_waitpid_loops)) { my_usleep(100000L); loop_counter++; } if ((return_pid == -1) || (loop_counter >= max_waitpid_loops)) { msg_str[0] = '\0'; if (return_pid == -1) { trans_log(WARN_SIGN, __FILE__, __LINE__, "scp_quit", NULL, _("Failed to catch zombie of data ssh process : %s"), strerror(errno)); } if (data_pid > 0) { if (kill(data_pid, SIGKILL) == -1) { trans_log(ERROR_SIGN, __FILE__, __LINE__, "scp_quit", NULL, #if SIZEOF_PID_T == 4 _("Failed to kill() data ssh process %d : %s"), #else _("Failed to kill() data ssh process %lld : %s"), #endif (pri_pid_t)data_pid, strerror(errno)); } else { trans_log(WARN_SIGN, __FILE__, __LINE__, "scp_quit", NULL, #if SIZEOF_PID_T == 4 _("Killing hanging data ssh process %d."), #else _("Killing hanging data ssh process %lld."), #endif (pri_pid_t)data_pid); } } else { trans_log(DEBUG_SIGN, __FILE__, __LINE__, "scp_quit", NULL, #if SIZEOF_PID_T == 4 _("Hmm, pid is %d!!!"), (pri_pid_t)data_pid); #else _("Hmm, pid is %lld!!!"), (pri_pid_t)data_pid); #endif } }
/*############################## scp_write() ############################*/ int scp_write(char *block, int size) { int status; fd_set wset; struct timeval timeout; /* Initialise descriptor set. */ FD_ZERO(&wset); FD_SET(data_fd, &wset); timeout.tv_usec = 0L; timeout.tv_sec = transfer_timeout; /* Wait for message x seconds and then continue. */ status = select(data_fd + 1, NULL, &wset, NULL, &timeout); if (status == 0) { /* Timeout has arrived. */ timeout_flag = ON; return(INCORRECT); } else if (FD_ISSET(data_fd, &wset)) { int tmp_errno; /* In some cases, the write system call hangs. */ if (signal(SIGALRM, sig_handler) == SIG_ERR) { trans_log(ERROR_SIGN, __FILE__, __LINE__, "scp_write", NULL, _("Failed to set signal handler : %s"), strerror(errno)); return(INCORRECT); } if (sigsetjmp(env_alrm, 1) != 0) { trans_log(ERROR_SIGN, __FILE__, __LINE__, "scp_write", NULL, _("write() timeout (%ld)"), transfer_timeout); timeout_flag = ON; return(INCORRECT); } (void)alarm(transfer_timeout); status = write(data_fd, block, size); tmp_errno = errno; (void)alarm(0); if (status != size) { trans_log(ERROR_SIGN, __FILE__, __LINE__, "scp_write", NULL, _("write() error (%d) : %s"), status, strerror(tmp_errno)); return(tmp_errno); } #ifdef WITH_TRACE trace_log(NULL, 0, BIN_W_TRACE, block, size, NULL); #endif } else if (status < 0) { trans_log(ERROR_SIGN, __FILE__, __LINE__, "scp_write", NULL, _("select() error : %s"), strerror(errno)); return(INCORRECT); } else { trans_log(ERROR_SIGN, __FILE__, __LINE__, "scp_write", NULL, _("Unknown condition.")); return(INCORRECT); } return(SUCCESS); }
/*############################# pop3_stat() #############################*/ int pop3_stat(int *no_of_messages, off_t *msg_size) { int reply; if ((reply = command(pop3_fd, "STAT")) == SUCCESS) { if ((reply = get_reply()) == POP3_OK) { int i; char *ptr, str_number[MAX_LONG_LENGTH + 1]; ptr = msg_str + 4; i = 0; while (isdigit((int)(*(ptr + i))) && (i < MAX_INT_LENGTH)) { str_number[i] = *(ptr + i); i++; } if (i <= MAX_INT_LENGTH) { if (i > 0) { str_number[i] = '\0'; *no_of_messages = atoi(str_number); } else { *no_of_messages = 0; } } else { trans_log(WARN_SIGN, __FILE__, __LINE__, "pop3_stat", msg_str, _("Number of messages in reply to large to store.")); while ((*(ptr + i) != ' ') && (*(ptr + i) != '\r') && (*(ptr + i) != '\n') && (*(ptr + i) != '\0')) { i++; } *no_of_messages = 0; } ptr += i; i = 0; while (isdigit((int)(*(ptr + i))) && (i < MAX_LONG_LENGTH)) { str_number[i] = *(ptr + i); i++; } if (i <= MAX_LONG_LENGTH) { if (i > 0) { str_number[i] = '\0'; *msg_size = (off_t)str2offt(str_number, NULL, 8); } else { *msg_size = 0; } } else { trans_log(WARN_SIGN, __FILE__, __LINE__, "pop3_stat", msg_str, _("Size in reply to large to store.")); *msg_size = 0; } read_buffer[0] = '\0'; rb_offset = 0; } } return(reply); }
/*$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ main() $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$*/ int main(int argc, char *argv[]) { #ifdef _WITH_BURST_2 int cb2_ret; #endif int current_toggle, exit_status = TRANSFER_SUCCESS, j, fd, status, loops, rest, blocksize, *wmo_counter, wmo_counter_fd = -1; #ifdef WITH_ARCHIVE_COPY_INFO unsigned int archived_copied = 0; #endif off_t no_of_bytes; time_t connected, #ifdef _WITH_BURST_2 diff_time, #endif end_transfer_time_file, start_transfer_time_file = 0, last_update_time, now; #ifdef _OUTPUT_LOG clock_t end_time = 0, start_time = 0; struct tms tmsdummy; #endif char *p_file_name_buffer, *buffer, fullname[MAX_PATH_LENGTH + 1], file_path[MAX_PATH_LENGTH]; clock_t clktck; struct stat stat_buf; struct job *p_db; #ifdef SA_FULLDUMP struct sigaction sact; #endif CHECK_FOR_VERSION(argc, argv); #ifdef SA_FULLDUMP /* * When dumping core sure we do a FULL core dump! */ sact.sa_handler = SIG_DFL; sact.sa_flags = SA_FULLDUMP; sigemptyset(&sact.sa_mask); if (sigaction(SIGSEGV, &sact, NULL) == -1) { system_log(ERROR_SIGN, __FILE__, __LINE__, "sigaction() error : %s", strerror(errno)); exit(INCORRECT); } #endif /* Do some cleanups when we exit. */ if (atexit(sf_wmo_exit) != 0) { system_log(ERROR_SIGN, __FILE__, __LINE__, "Could not register exit function : %s", strerror(errno)); exit(INCORRECT); } /* Initialise variables. */ local_file_counter = 0; files_to_send = init_sf(argc, argv, file_path, WMO_FLAG); p_db = &db; if ((clktck = sysconf(_SC_CLK_TCK)) <= 0) { system_log(ERROR_SIGN, __FILE__, __LINE__, "Could not get clock ticks per second : %s", strerror(errno)); exit(INCORRECT); } if (fsa->trl_per_process > 0) { if (fsa->trl_per_process < fsa->block_size) { blocksize = fsa->trl_per_process; } else { blocksize = fsa->block_size; } } else { blocksize = fsa->block_size; } if ((signal(SIGINT, sig_kill) == SIG_ERR) || (signal(SIGQUIT, sig_exit) == SIG_ERR) || (signal(SIGTERM, SIG_IGN) == SIG_ERR) || (signal(SIGSEGV, sig_segv) == SIG_ERR) || (signal(SIGBUS, sig_bus) == SIG_ERR) || (signal(SIGHUP, SIG_IGN) == SIG_ERR) || (signal(SIGPIPE, SIG_IGN) == SIG_ERR)) { system_log(ERROR_SIGN, __FILE__, __LINE__, "signal() error : %s", strerror(errno)); exit(INCORRECT); } /* Now determine the real hostname. */ if (db.toggle_host == YES) { if (fsa->host_toggle == HOST_ONE) { (void)strcpy(db.hostname, fsa->real_hostname[HOST_TWO - 1]); current_toggle = HOST_TWO; } else { (void)strcpy(db.hostname, fsa->real_hostname[HOST_ONE - 1]); current_toggle = HOST_ONE; } } else { (void)strcpy(db.hostname, fsa->real_hostname[(int)(fsa->host_toggle - 1)]); current_toggle = (int)fsa->host_toggle; } /* Connect to remote WMO-server. */ #ifdef FTP_CTRL_KEEP_ALIVE_INTERVAL if (fsa->protocol_options & AFD_TCP_KEEPALIVE) { timeout_flag = transfer_timeout - 5; if (timeout_flag < MIN_KEEP_ALIVE_INTERVAL) { timeout_flag = MIN_KEEP_ALIVE_INTERVAL; } } #else timeout_flag = OFF; #endif if ((status = wmo_connect(db.hostname, db.port, db.sndbuf_size)) != SUCCESS) { trans_log(ERROR_SIGN, __FILE__, __LINE__, NULL, NULL, "WMO connection to <%s> at port %d failed (%d).", db.hostname, db.port, status); exit(eval_timeout(CONNECT_ERROR)); } else { if (fsa->debug > NORMAL_MODE) { trans_db_log(INFO_SIGN, __FILE__, __LINE__, NULL, "Connected to port %d.", db.port); } } connected = time(NULL); /* Inform FSA that we have finished connecting and */ /* will now start to transfer data. */ if (gsf_check_fsa(p_db) != NEITHER) { #ifdef LOCK_DEBUG lock_region_w(fsa_fd, db.lock_offset + LOCK_CON, __FILE__, __LINE__); #else lock_region_w(fsa_fd, db.lock_offset + LOCK_CON); #endif fsa->job_status[(int)db.job_no].connect_status = WMO_ACTIVE; fsa->job_status[(int)db.job_no].no_of_files = files_to_send; fsa->connections += 1; #ifdef LOCK_DEBUG unlock_region(fsa_fd, db.lock_offset + LOCK_CON, __FILE__, __LINE__); #else unlock_region(fsa_fd, db.lock_offset + LOCK_CON); #endif } /* Allocate buffer to read data from the source file. */ if ((buffer = malloc(blocksize + 1 + 4 /* For bulletin end. */)) == NULL) { system_log(ERROR_SIGN, __FILE__, __LINE__, "malloc() error : %s", strerror(errno)); exit(ALLOC_ERROR); } if (db.special_flag & WITH_SEQUENCE_NUMBER) { char counter_file_name[MAX_FILENAME_LENGTH]; (void)snprintf(counter_file_name, MAX_FILENAME_LENGTH, "/%s.%d", db.host_alias, db.port); if ((wmo_counter_fd = open_counter_file(counter_file_name, &wmo_counter)) < 0) { system_log(ERROR_SIGN, __FILE__, __LINE__, "Failed to open counter file `%s'.", counter_file_name); } } #ifdef _WITH_BURST_2 do { if (burst_2_counter > 0) { if (fsa->debug > NORMAL_MODE) { trans_db_log(INFO_SIGN, __FILE__, __LINE__, NULL, "WMO Bursting."); } } #endif /* Send all files. */ p_file_name_buffer = file_name_buffer; p_file_size_buffer = file_size_buffer; last_update_time = time(NULL); local_file_size = 0; for (files_send = 0; files_send < files_to_send; files_send++) { (void)snprintf(fullname, MAX_PATH_LENGTH + 1, "%s/%s", file_path, p_file_name_buffer); if (*p_file_size_buffer > 0) { int end_length = 0, header_length = 0, length_type_indicator = 10; if (gsf_check_fsa(p_db) != NEITHER) { fsa->job_status[(int)db.job_no].file_size_in_use = *p_file_size_buffer; (void)strcpy(fsa->job_status[(int)db.job_no].file_name_in_use, p_file_name_buffer); } /* Open local file. */ #ifdef O_LARGEFILE if ((fd = open(fullname, O_RDONLY | O_LARGEFILE)) == -1) #else if ((fd = open(fullname, O_RDONLY)) == -1) #endif { trans_log(ERROR_SIGN, __FILE__, __LINE__, NULL, NULL, "Failed to open local file `%s' : %s", fullname, strerror(errno)); wmo_quit(); exit(OPEN_LOCAL_ERROR); } if (fsa->debug > NORMAL_MODE) { trans_db_log(INFO_SIGN, __FILE__, __LINE__, NULL, "Open local file `%s'", fullname); } #ifdef _OUTPUT_LOG if (db.output_log == YES) { start_time = times(&tmsdummy); } #endif /* * When the contents does not contain a bulletin header * it must be stored in the file name. */ if (db.special_flag & FILE_NAME_IS_HEADER) { int space_count; char *ptr = p_file_name_buffer; buffer[length_type_indicator] = 1; /* SOH */ buffer[length_type_indicator + 1] = '\015'; /* CR */ buffer[length_type_indicator + 2] = '\015'; /* CR */ buffer[length_type_indicator + 3] = '\012'; /* LF */ header_length = 4; space_count = 0; if (wmo_counter_fd > 0) { if (next_counter(wmo_counter_fd, wmo_counter, MAX_WMO_COUNTER) < 0) { close_counter_file(wmo_counter_fd, &wmo_counter); wmo_counter_fd = -1; wmo_counter = NULL; system_log(ERROR_SIGN, __FILE__, __LINE__, "Failed to get next WMO counter."); } else { if (*wmo_counter < 10) { buffer[length_type_indicator + header_length] = '0'; buffer[length_type_indicator + header_length + 1] = '0'; buffer[length_type_indicator + header_length + 2] = *wmo_counter + '0'; } else if (*wmo_counter < 100) { buffer[length_type_indicator + header_length] = '0'; buffer[length_type_indicator + header_length + 1] = (*wmo_counter / 10) + '0'; buffer[length_type_indicator + header_length + 2] = (*wmo_counter % 10) + '0'; } else if (*wmo_counter < 1000) { buffer[length_type_indicator + header_length] = ((*wmo_counter / 100) % 10) + '0'; buffer[length_type_indicator + header_length + 1] = ((*wmo_counter / 10) % 10) + '0'; buffer[length_type_indicator + header_length + 2] = (*wmo_counter % 10) + '0'; } buffer[length_type_indicator + header_length + 3] = '\015'; /* CR */ buffer[length_type_indicator + header_length + 4] = '\015'; /* CR */ buffer[length_type_indicator + header_length + 5] = '\012'; /* LF */ header_length += 6; } } /* if (wmo_counter_fd > 0) */ for (;;) { while ((*ptr != '_') && (*ptr != '-') && (*ptr != ' ') && (*ptr != '\0') && (*ptr != '.') && (*ptr != ';')) { buffer[length_type_indicator + header_length] = *ptr; header_length++; ptr++; } if ((*ptr == '\0') || (*ptr == '.') || (*ptr == ';')) { break; } else { if (space_count == 2) { if ((isalpha((int)(*(ptr + 1)))) && (isalpha((int)(*(ptr + 2)))) && (isalpha((int)(*(ptr + 3))))) { buffer[length_type_indicator + header_length] = ' '; buffer[length_type_indicator + header_length + 1] = *(ptr + 1); buffer[length_type_indicator + header_length + 2] = *(ptr + 2); buffer[length_type_indicator + header_length + 3] = *(ptr + 3); header_length += 4; } break; } else { buffer[length_type_indicator + header_length] = ' '; header_length++; ptr++; space_count++; } } } /* for (;;) */ buffer[length_type_indicator + header_length] = '\015'; /* CR */ buffer[length_type_indicator + header_length + 1] = '\015'; /* CR */ buffer[length_type_indicator + header_length + 2] = '\012'; /* LF */ header_length += 3; end_length = 4; } /* Read (local) and write (remote) file. */ no_of_bytes = 0; loops = (length_type_indicator + header_length + *p_file_size_buffer) / blocksize; rest = (length_type_indicator + header_length + *p_file_size_buffer) % blocksize; if ((db.special_flag & FILE_NAME_IS_HEADER) && (rest == 0)) { loops--; rest = blocksize; } /* Write length and type indicator. */ (void)snprintf(buffer, 9, "%08lu", (unsigned long)(*p_file_size_buffer + header_length + end_length)); if (db.transfer_mode == 'I') { buffer[length_type_indicator - 2] = 'B'; buffer[length_type_indicator - 1] = 'I'; } else if (db.transfer_mode == 'A') { buffer[length_type_indicator - 2] = 'A'; buffer[length_type_indicator - 1] = 'N'; } else { buffer[length_type_indicator - 2] = 'F'; buffer[length_type_indicator - 1] = 'X'; } if (fsa->trl_per_process > 0) { init_limit_transfer_rate(); } if (fsa->protocol_options & TIMEOUT_TRANSFER) { start_transfer_time_file = time(NULL); } for (;;) { for (j = 0; j < loops; j++) { #ifdef _SIMULATE_SLOW_TRANSFER (void)sleep(_SIMULATE_SLOW_TRANSFER); #endif if ((status = read(fd, (buffer + length_type_indicator + header_length), (blocksize - length_type_indicator - header_length))) != (blocksize - length_type_indicator - header_length)) { trans_log(ERROR_SIGN, __FILE__, __LINE__, NULL, NULL, "Could not read() local file `%s' : %s", fullname, strerror(errno)); wmo_quit(); exit(READ_LOCAL_ERROR); } if ((status = wmo_write(buffer, blocksize)) != SUCCESS) { trans_log(ERROR_SIGN, __FILE__, __LINE__, NULL, NULL, "Failed to write block from file `%s' to remote port %d [%d].", p_file_name_buffer, db.port, status); wmo_quit(); exit(eval_timeout(WRITE_REMOTE_ERROR)); } if (fsa->trl_per_process > 0) { limit_transfer_rate(blocksize, fsa->trl_per_process, clktck); } no_of_bytes += blocksize; if (gsf_check_fsa(p_db) != NEITHER) { fsa->job_status[(int)db.job_no].file_size_in_use_done = no_of_bytes; fsa->job_status[(int)db.job_no].file_size_done += blocksize; fsa->job_status[(int)db.job_no].bytes_send += blocksize; if (fsa->protocol_options & TIMEOUT_TRANSFER) { end_transfer_time_file = time(NULL); if (end_transfer_time_file < start_transfer_time_file) { start_transfer_time_file = end_transfer_time_file; } else { if ((end_transfer_time_file - start_transfer_time_file) > transfer_timeout) { trans_log(INFO_SIGN, __FILE__, __LINE__, NULL, NULL, #if SIZEOF_TIME_T == 4 "Transfer timeout reached for `%s' after %ld seconds.", #else "Transfer timeout reached for `%s' after %lld seconds.", #endif fsa->job_status[(int)db.job_no].file_name_in_use, (pri_time_t)(end_transfer_time_file - start_transfer_time_file)); wmo_quit(); exitflag = 0; exit(STILL_FILES_TO_SEND); } } } } if (length_type_indicator > 0) { length_type_indicator = 0; header_length = 0; } } /* for (j = 0; j < loops; j++) */ if (rest > 0) { if ((status = read(fd, (buffer + length_type_indicator + header_length), (rest - length_type_indicator - header_length))) != (rest - length_type_indicator - header_length)) { trans_log(ERROR_SIGN, __FILE__, __LINE__, NULL, NULL, "Could not read() local file `%s' : %s", fullname, strerror(errno)); wmo_quit(); exit(READ_LOCAL_ERROR); } if (end_length == 4) { buffer[rest] = '\015'; buffer[rest + 1] = '\015'; buffer[rest + 2] = '\012'; buffer[rest + 3] = 3; /* ETX */ } if ((status = wmo_write(buffer, rest + end_length)) != SUCCESS) { trans_log(ERROR_SIGN, __FILE__, __LINE__, NULL, NULL, "Failed to write rest of file to remote port %d [%d].", p_file_name_buffer, db.port, status); wmo_quit(); exit(eval_timeout(WRITE_REMOTE_ERROR)); } if (fsa->trl_per_process > 0) { limit_transfer_rate(rest + end_length, fsa->trl_per_process, clktck); } no_of_bytes += rest + end_length; if (gsf_check_fsa(p_db) != NEITHER) { fsa->job_status[(int)db.job_no].file_size_in_use_done = no_of_bytes; fsa->job_status[(int)db.job_no].file_size_done += rest; fsa->job_status[(int)db.job_no].bytes_send += rest; } } /* * Since there are always some users sending files to the * AFD not in dot notation, lets check here if this is really * the EOF. * If not lets continue so long until we hopefully have reached * the EOF. * NOTE: This is NOT a fool proof way. There must be a better * way! */ if (fstat(fd, &stat_buf) == -1) { (void)rec(transfer_log_fd, DEBUG_SIGN, "Hmmm. Failed to stat() `%s' : %s (%s %d)\n", fullname, strerror(errno), __FILE__, __LINE__); break; } else { if (stat_buf.st_size > *p_file_size_buffer) { char sign[LOG_SIGN_LENGTH]; if (db.special_flag & SILENT_NOT_LOCKED_FILE) { (void)memcpy(sign, DEBUG_SIGN, LOG_SIGN_LENGTH); } else { (void)memcpy(sign, WARN_SIGN, LOG_SIGN_LENGTH); } loops = (stat_buf.st_size - *p_file_size_buffer) / blocksize; rest = (stat_buf.st_size - *p_file_size_buffer) % blocksize; *p_file_size_buffer = stat_buf.st_size; /* * Give a warning in the receive log, so some action * can be taken against the originator. */ receive_log(sign, __FILE__, __LINE__, 0L, db.id.job, "File `%s' for host %s was DEFINITELY send without any locking. #%x", p_file_name_buffer, fsa->host_dsp_name, db.id.job); } else { break; } } } /* for (;;) */ if (db.special_flag & WMO_CHECK_ACKNOWLEDGE) { int ret; if ((ret = wmo_check_reply()) == INCORRECT) { trans_log(ERROR_SIGN, __FILE__, __LINE__, NULL, NULL, "Failed to receive reply from port %d for file %s.", db.port, p_file_name_buffer); wmo_quit(); exit(eval_timeout(CHECK_REPLY_ERROR)); } else if (ret == NEGATIV_ACKNOWLEDGE) { trans_log(ERROR_SIGN, __FILE__, __LINE__, NULL, NULL, "Received negative acknowledge from remote port %d for file %s.", db.port, p_file_name_buffer); } } #ifdef _OUTPUT_LOG if (db.output_log == YES) { end_time = times(&tmsdummy); } #endif /* Close local file. */ if (close(fd) == -1) { (void)rec(transfer_log_fd, WARN_SIGN, "%-*s[%d]: Failed to close() local file %s : %s (%s %d)\n", MAX_HOSTNAME_LENGTH, tr_hostname, (int)db.job_no, p_file_name_buffer, strerror(errno), __FILE__, __LINE__); /* * Since we usually do not send more then 100 files and * sf_wmo() will exit(), there is no point in stopping * the transmission. */ } } else { trans_log(INFO_SIGN, __FILE__, __LINE__, NULL, NULL, "File `%s' is of zero length, ignoring.", p_file_name_buffer); } /* Update FSA, one file transmitted. */ if (gsf_check_fsa(p_db) != NEITHER) { fsa->job_status[(int)db.job_no].file_name_in_use[0] = '\0'; fsa->job_status[(int)db.job_no].no_of_files_done++; fsa->job_status[(int)db.job_no].file_size_in_use = 0; fsa->job_status[(int)db.job_no].file_size_in_use_done = 0; local_file_size += *p_file_size_buffer; local_file_counter += 1; now = time(NULL); if (now >= (last_update_time + LOCK_INTERVAL_TIME)) { last_update_time = now; update_tfc(local_file_counter, local_file_size, p_file_size_buffer, files_to_send, files_send, now); local_file_size = 0; local_file_counter = 0; } } #ifdef _WITH_TRANS_EXEC if (db.special_flag & TRANS_EXEC) { trans_exec(file_path, fullname, p_file_name_buffer, clktck); } #endif #ifdef _OUTPUT_LOG if (db.output_log == YES) { if (ol_fd == -2) { # ifdef WITHOUT_FIFO_RW_SUPPORT output_log_fd(&ol_fd, &ol_readfd, &db.output_log); # else output_log_fd(&ol_fd, &db.output_log); # endif } if ((ol_fd > -1) && (ol_data == NULL)) { output_log_ptrs(&ol_retries, &ol_job_number, &ol_data, /* Pointer to buffer. */ &ol_file_name, &ol_file_name_length, &ol_archive_name_length, &ol_file_size, &ol_unl, &ol_size, &ol_transfer_time, &ol_output_type, db.host_alias, (current_toggle - 1), WMO, &db.output_log); } } #endif /* Now archive file if necessary. */ if ((db.archive_time > 0) && (p_db->archive_dir[0] != FAILED_TO_CREATE_ARCHIVE_DIR)) { #ifdef WITH_ARCHIVE_COPY_INFO int ret; #endif /* * By telling the function archive_file() that this * is the first time to archive a file for this job * (in struct p_db) it does not always have to check * whether the directory has been created or not. And * we ensure that we do not create duplicate names * when adding db.archive_time to msg_name. */ #ifdef WITH_ARCHIVE_COPY_INFO if ((ret = archive_file(file_path, p_file_name_buffer, p_db)) < 0) #else if (archive_file(file_path, p_file_name_buffer, p_db) < 0) #endif { if (fsa->debug > NORMAL_MODE) { trans_db_log(ERROR_SIGN, __FILE__, __LINE__, NULL, "Failed to archive file `%s'", p_file_name_buffer); } /* * NOTE: We _MUST_ delete the file we just send, * else the file directory will run full! */ if (unlink(fullname) == -1) { system_log(ERROR_SIGN, __FILE__, __LINE__, "Could not unlink() local file `%s' after sending it successfully : %s", fullname, strerror(errno)); } #ifdef _OUTPUT_LOG if (db.output_log == YES) { (void)memcpy(ol_file_name, db.p_unique_name, db.unl); (void)strcpy(ol_file_name + db.unl, p_file_name_buffer); *ol_file_name_length = (unsigned short)strlen(ol_file_name); ol_file_name[*ol_file_name_length] = SEPARATOR_CHAR; ol_file_name[*ol_file_name_length + 1] = '\0'; (*ol_file_name_length)++; *ol_file_size = *p_file_size_buffer; *ol_job_number = fsa->job_status[(int)db.job_no].job_id; *ol_retries = db.retries; *ol_unl = db.unl; *ol_transfer_time = end_time - start_time; *ol_archive_name_length = 0; *ol_output_type = OT_NORMAL_DELIVERED + '0'; ol_real_size = *ol_file_name_length + ol_size; if (write(ol_fd, ol_data, ol_real_size) != ol_real_size) { system_log(ERROR_SIGN, __FILE__, __LINE__, "write() error : %s", strerror(errno)); } } #endif } else { if (fsa->debug > NORMAL_MODE) { trans_db_log(INFO_SIGN, __FILE__, __LINE__, NULL, "Archived file `%s'", p_file_name_buffer); } #ifdef WITH_ARCHIVE_COPY_INFO if (ret == DATA_COPIED) { archived_copied++; } #endif #ifdef _OUTPUT_LOG if (db.output_log == YES) { (void)memcpy(ol_file_name, db.p_unique_name, db.unl); (void)strcpy(ol_file_name + db.unl, p_file_name_buffer); *ol_file_name_length = (unsigned short)strlen(ol_file_name); ol_file_name[*ol_file_name_length] = SEPARATOR_CHAR; ol_file_name[*ol_file_name_length + 1] = '\0'; (*ol_file_name_length)++; (void)strcpy(&ol_file_name[*ol_file_name_length + 1], &db.archive_dir[db.archive_offset]); *ol_file_size = *p_file_size_buffer; *ol_job_number = fsa->job_status[(int)db.job_no].job_id; *ol_retries = db.retries; *ol_unl = db.unl; *ol_transfer_time = end_time - start_time; *ol_archive_name_length = (unsigned short)strlen(&ol_file_name[*ol_file_name_length + 1]); *ol_output_type = OT_NORMAL_DELIVERED + '0'; ol_real_size = *ol_file_name_length + *ol_archive_name_length + 1 + ol_size; if (write(ol_fd, ol_data, ol_real_size) != ol_real_size) { system_log(ERROR_SIGN, __FILE__, __LINE__, "write() error : %s", strerror(errno)); } } #endif } } else { #ifdef WITH_UNLINK_DELAY int unlink_loops = 0; try_again_unlink: #endif /* Delete the file we just have send. */ if (unlink(fullname) == -1) { #ifdef WITH_UNLINK_DELAY if ((errno == EBUSY) && (unlink_loops < 20)) { (void)my_usleep(100000L); unlink_loops++; goto try_again_unlink; } #endif system_log(ERROR_SIGN, __FILE__, __LINE__, "Could not unlink() local file %s after sending it successfully : %s", fullname, strerror(errno)); } #ifdef _OUTPUT_LOG if (db.output_log == YES) { (void)memcpy(ol_file_name, db.p_unique_name, db.unl); (void)strcpy(ol_file_name + db.unl, p_file_name_buffer); *ol_file_name_length = (unsigned short)strlen(ol_file_name); ol_file_name[*ol_file_name_length] = SEPARATOR_CHAR; ol_file_name[*ol_file_name_length + 1] = '\0'; (*ol_file_name_length)++; *ol_file_size = *p_file_size_buffer; *ol_job_number = fsa->job_status[(int)db.job_no].job_id; *ol_retries = db.retries; *ol_unl = db.unl; *ol_transfer_time = end_time - start_time; *ol_archive_name_length = 0; *ol_output_type = OT_NORMAL_DELIVERED + '0'; ol_real_size = *ol_file_name_length + ol_size; if (write(ol_fd, ol_data, ol_real_size) != ol_real_size) { system_log(ERROR_SIGN, __FILE__, __LINE__, "write() error : %s", strerror(errno)); } } #endif } /* * After each successful transfer set error * counter to zero, so that other jobs can be * started. */ if (gsf_check_fsa(p_db) != NEITHER) { if ((*p_file_size_buffer > 0) && (fsa->error_counter > 0)) { int fd, #ifdef WITHOUT_FIFO_RW_SUPPORT readfd, #endif j; char fd_wake_up_fifo[MAX_PATH_LENGTH]; #ifdef LOCK_DEBUG lock_region_w(fsa_fd, db.lock_offset + LOCK_EC, __FILE__, __LINE__); #else lock_region_w(fsa_fd, db.lock_offset + LOCK_EC); #endif fsa->error_counter = 0; /* * Wake up FD! */ (void)snprintf(fd_wake_up_fifo, MAX_PATH_LENGTH, "%s%s%s", p_work_dir, FIFO_DIR, FD_WAKE_UP_FIFO); #ifdef WITHOUT_FIFO_RW_SUPPORT if (open_fifo_rw(fd_wake_up_fifo, &readfd, &fd) == -1) #else if ((fd = open(fd_wake_up_fifo, O_RDWR)) == -1) #endif { system_log(WARN_SIGN, __FILE__, __LINE__, "Failed to open() FIFO %s : %s", fd_wake_up_fifo, strerror(errno)); } else { if (write(fd, "", 1) != 1) { system_log(WARN_SIGN, __FILE__, __LINE__, "Failed to write() to FIFO %s : %s", fd_wake_up_fifo, strerror(errno)); } #ifdef WITHOUT_FIFO_RW_SUPPORT if (close(readfd) == -1) { system_log(DEBUG_SIGN, __FILE__, __LINE__, "Failed to close() FIFO %s (read) : %s", fd_wake_up_fifo, strerror(errno)); } #endif if (close(fd) == -1) { system_log(DEBUG_SIGN, __FILE__, __LINE__, "Failed to close() FIFO %s : %s", fd_wake_up_fifo, strerror(errno)); } } /* * Remove the error condition (NOT_WORKING) from all jobs * of this host. */ for (j = 0; j < fsa->allowed_transfers; j++) { if ((j != db.job_no) && (fsa->job_status[j].connect_status == NOT_WORKING)) { fsa->job_status[j].connect_status = DISCONNECT; } } fsa->error_history[0] = 0; fsa->error_history[1] = 0; #ifdef LOCK_DEBUG unlock_region(fsa_fd, db.lock_offset + LOCK_EC, __FILE__, __LINE__); #else unlock_region(fsa_fd, db.lock_offset + LOCK_EC); #endif #ifdef LOCK_DEBUG lock_region_w(fsa_fd, db.lock_offset + LOCK_HS, __FILE__, __LINE__); #else lock_region_w(fsa_fd, db.lock_offset + LOCK_HS); #endif now = time(NULL); if (now > fsa->end_event_handle) { fsa->host_status &= ~(EVENT_STATUS_FLAGS | AUTO_PAUSE_QUEUE_STAT); if (fsa->end_event_handle > 0L) { fsa->end_event_handle = 0L; } if (fsa->start_event_handle > 0L) { fsa->start_event_handle = 0L; } } else { fsa->host_status &= ~(EVENT_STATUS_STATIC_FLAGS | AUTO_PAUSE_QUEUE_STAT); } #ifdef LOCK_DEBUG unlock_region(fsa_fd, db.lock_offset + LOCK_HS, __FILE__, __LINE__); #else unlock_region(fsa_fd, db.lock_offset + LOCK_HS); #endif /* * Since we have successfully transmitted a file, no need to * have the queue stopped anymore. */ if (fsa->host_status & AUTO_PAUSE_QUEUE_STAT) { char sign[LOG_SIGN_LENGTH]; error_action(fsa->host_alias, "stop", HOST_ERROR_ACTION); event_log(0L, EC_HOST, ET_EXT, EA_ERROR_END, "%s", fsa->host_alias); if ((fsa->host_status & HOST_ERROR_OFFLINE_STATIC) || (fsa->host_status & HOST_ERROR_OFFLINE) || (fsa->host_status & HOST_ERROR_OFFLINE_T)) { (void)memcpy(sign, OFFLINE_SIGN, LOG_SIGN_LENGTH); } else { (void)memcpy(sign, INFO_SIGN, LOG_SIGN_LENGTH); } trans_log(sign, __FILE__, __LINE__, NULL, NULL, "Starting input queue that was stopped by init_afd."); event_log(0L, EC_HOST, ET_AUTO, EA_START_QUEUE, "%s", fsa->host_alias); } } /* if (fsa->error_counter > 0) */ #ifdef WITH_ERROR_QUEUE if (fsa->host_status & ERROR_QUEUE_SET) { remove_from_error_queue(db.id.job, fsa, db.fsa_pos, fsa_fd); } #endif if (fsa->host_status & HOST_ACTION_SUCCESS) { error_action(fsa->host_alias, "start", HOST_SUCCESS_ACTION); } } p_file_name_buffer += MAX_FILENAME_LENGTH; p_file_size_buffer++; } /* for (files_send = 0; files_send < files_to_send; files_send++) */ #ifdef WITH_ARCHIVE_COPY_INFO if (archived_copied > 0) { trans_log(DEBUG_SIGN, __FILE__, __LINE__, NULL, NULL, "Copied %u files to archive.", archived_copied); archived_copied = 0; } #endif if (local_file_counter) { if (gsf_check_fsa(p_db) != NEITHER) { update_tfc(local_file_counter, local_file_size, p_file_size_buffer, files_to_send, files_send, time(NULL)); local_file_size = 0; local_file_counter = 0; } } /* * Remove file directory, but only when all files have * been transmitted. */ if ((files_to_send == files_send) || (files_to_send < 1)) { if (rmdir(file_path) < 0) { system_log(ERROR_SIGN, __FILE__, __LINE__, "Failed to remove directory %s : %s", file_path, strerror(errno)); } } else { system_log(WARN_SIGN, __FILE__, __LINE__, "There are still %d files for %s. Will NOT remove this job!", files_to_send - files_send, file_path); exit_status = STILL_FILES_TO_SEND; } #ifdef _WITH_BURST_2 burst_2_counter++; diff_time = time(NULL) - connected; if (((fsa->protocol_options & KEEP_CONNECTED_DISCONNECT) && (db.keep_connected > 0) && (diff_time > db.keep_connected)) || ((db.disconnect > 0) && (diff_time > db.disconnect))) { cb2_ret = NO; break; } } while ((cb2_ret = check_burst_sf(file_path, &files_to_send, 0, # ifdef _WITH_INTERRUPT_JOB 0, # endif # ifdef _OUTPUT_LOG &ol_fd, # endif # ifndef AFDBENCH_CONFIG NULL, # endif NULL)) == YES); burst_2_counter--; if (cb2_ret == NEITHER) { exit_status = STILL_FILES_TO_SEND; } #endif /* _WITH_BURST_2 */ free(buffer); /* Disconnect from remote port. */ wmo_quit(); if ((fsa != NULL) && (fsa->debug > NORMAL_MODE)) { trans_db_log(INFO_SIGN, __FILE__, __LINE__, NULL, "Disconnected from port %d.", db.port); } if (wmo_counter_fd > 0) { close_counter_file(wmo_counter_fd, &wmo_counter); } exitflag = 0; exit(exit_status); }
/*+++++++++++++++++++++++++++++ sf_wmo_exit() +++++++++++++++++++++++++++*/ static void sf_wmo_exit(void) { if ((fsa != NULL) && (db.fsa_pos >= 0)) { int diff_no_of_files_done; u_off_t diff_file_size_done; if (local_file_counter) { if (gsf_check_fsa((struct job *)&db) != NEITHER) { update_tfc(local_file_counter, local_file_size, p_file_size_buffer, files_to_send, files_send, time(NULL)); } } diff_no_of_files_done = fsa->job_status[(int)db.job_no].no_of_files_done - prev_no_of_files_done; diff_file_size_done = fsa->job_status[(int)db.job_no].file_size_done - prev_file_size_done; if ((diff_file_size_done > 0) || (diff_no_of_files_done > 0)) { int length; #ifdef _WITH_BURST_2 char buffer[MAX_INT_LENGTH + 5 + MAX_OFF_T_LENGTH + 16 + MAX_INT_LENGTH + 11 + MAX_INT_LENGTH + 1]; length = MAX_INT_LENGTH + 5 + MAX_OFF_T_LENGTH + 16 + MAX_INT_LENGTH + 11 + MAX_INT_LENGTH + 1; #else char buffer[MAX_INT_LENGTH + 5 + MAX_OFF_T_LENGTH + 16 + MAX_INT_LENGTH + 1]; length = MAX_INT_LENGTH + 5 + MAX_OFF_T_LENGTH + 16 + MAX_INT_LENGTH + 1; #endif WHAT_DONE_BUFFER(length, buffer, "send", diff_file_size_done, diff_no_of_files_done); #ifdef _WITH_BURST_2 if (burst_2_counter == 1) { if ((length + 9) <= (MAX_INT_LENGTH + 5 + MAX_OFF_T_LENGTH + 16 + MAX_INT_LENGTH + 11 + MAX_INT_LENGTH + 1)) { /* Write " [BURST]" */ buffer[length] = ' '; buffer[length + 1] = '['; buffer[length + 2] = 'B'; buffer[length + 3] = 'U'; buffer[length + 4] = 'R'; buffer[length + 5] = 'S'; buffer[length + 6] = 'T'; buffer[length + 7] = ']'; buffer[length + 8] = '\0'; } } else if (burst_2_counter > 1) { (void)snprintf(buffer + length, MAX_INT_LENGTH + 5 + MAX_OFF_T_LENGTH + 16 + MAX_INT_LENGTH + 11 + MAX_INT_LENGTH + 1 - length, " [BURST * %u]", burst_2_counter); } #endif /* _WITH_BURST_2 */ trans_log(INFO_SIGN, NULL, 0, NULL, NULL, "%s #%x", buffer, db.id.job); } reset_fsa((struct job *)&db, exitflag, 0, 0); } free(file_name_buffer); free(file_size_buffer); send_proc_fin(NO); if (sys_log_fd != STDERR_FILENO) { (void)close(sys_log_fd); } return; }
/*----------------------------- read_msg() ------------------------------*/ static int read_msg(void) { static int bytes_buffered, bytes_read = 0; static char *read_ptr = NULL; int status; fd_set rset; if (bytes_read == 0) { bytes_buffered = 0; } else { (void)memmove(msg_str, read_ptr + 1, bytes_read); bytes_buffered = bytes_read; read_ptr = msg_str; } FD_ZERO(&rset); for (;;) { if (bytes_read <= 0) { /* Initialise descriptor set. */ FD_SET(pop3_fd, &rset); timeout.tv_usec = 0L; timeout.tv_sec = transfer_timeout; /* Wait for message x seconds and then continue. */ status = select(pop3_fd + 1, &rset, NULL, NULL, &timeout); if (status == 0) { /* Timeout has arrived. */ timeout_flag = ON; bytes_read = 0; return(INCORRECT); } else if (FD_ISSET(pop3_fd, &rset)) { #ifdef WITH_SSL if (ssl_con == NULL) { #endif if ((bytes_read = read(pop3_fd, &msg_str[bytes_buffered], (MAX_RET_MSG_LENGTH - bytes_buffered))) < 1) { if (bytes_read == 0) { trans_log(ERROR_SIGN, __FILE__, __LINE__, "read_msg", NULL, _("Remote hang up.")); timeout_flag = NEITHER; } else { if (errno == ECONNRESET) { timeout_flag = CON_RESET; } trans_log(ERROR_SIGN, __FILE__, __LINE__, "read_msg", NULL, _("read() error (after reading %d bytes) : %s"), bytes_buffered, strerror(errno)); bytes_read = 0; } return(INCORRECT); } #ifdef WITH_SSL } else { if ((bytes_read = SSL_read(ssl_con, &msg_str[bytes_buffered], (MAX_RET_MSG_LENGTH - bytes_buffered))) < 1) { if (bytes_read == 0) { trans_log(ERROR_SIGN, __FILE__, __LINE__, "read_msg", NULL, _("Remote hang up.")); timeout_flag = NEITHER; } else { if ((status = SSL_get_error(ssl_con, bytes_read)) == SSL_ERROR_SYSCALL) { if (errno == ECONNRESET) { timeout_flag = CON_RESET; } trans_log(ERROR_SIGN, __FILE__, __LINE__, "read_msg", NULL, _("SSL_read() error (after reading %d bytes) : %s"), bytes_buffered, strerror(errno)); } else { trans_log(ERROR_SIGN, __FILE__, __LINE__, "read_msg", NULL, _("SSL_read() error (after reading %d bytes) (%d)"), bytes_buffered, status); } bytes_read = 0; } return(INCORRECT); } } #endif #ifdef WITH_TRACE trace_log(NULL, 0, R_TRACE, &msg_str[bytes_buffered], bytes_read, NULL); #endif read_ptr = &msg_str[bytes_buffered]; bytes_buffered += bytes_read; } else if (status < 0) { trans_log(ERROR_SIGN, __FILE__, __LINE__, "read_msg", NULL, _("select() error : %s"), strerror(errno)); return(INCORRECT); } else { trans_log(ERROR_SIGN, __FILE__, __LINE__, "read_msg", NULL, _("Unknown condition.")); return(INCORRECT); } } /* Evaluate what we have read. */ do { if ((*read_ptr == '\n') && (*(read_ptr - 1) == '\r')) { *(read_ptr - 1) = '\0'; bytes_read--; return(bytes_buffered); } read_ptr++; bytes_read--; } while(bytes_read > 0); } /* for (;;) */ }
/*############################# pop3_quit() #############################*/ int pop3_quit(void) { int reply; if (pop3_fd != -1) { (void)command(pop3_fd, "QUIT"); if (timeout_flag == OFF) { if ((reply = get_reply()) == INCORRECT) { (void)close(pop3_fd); return(INCORRECT); } #ifdef _WITH_SHUTDOWN if (shutdown(pop3_fd, 1) < 0) { trans_log(DEBUG_SIGN, __FILE__, __LINE__, "pop3_quit", NULL, _("shutdown() error : %s"), strerror(errno)); } else { int status; char buffer[32]; fd_set rset; /* Initialise descriptor set */ FD_ZERO(&rset); FD_SET(pop3_fd, &rset); timeout.tv_usec = 0L; timeout.tv_sec = transfer_timeout; /* Wait for message x seconds and then continue. */ status = select(pop3_fd + 1, &rset, NULL, NULL, &timeout); if (status > 0) { if (FD_ISSET(pop3_fd, &rset)) { if ((status = read(pop3_fd, buffer, 32)) < 0) { trans_log(ERROR_SIGN, __FILE__, __LINE__, "pop3_quit", NULL, _("read() error (%d) : %s"), status, strerror(errno)); reply = INCORRECT; } } } else if (status == 0) { /* Timeout has arrived. */ timeout_flag = ON; reply = INCORRECT; } else { trans_log(ERROR_SIGN, __FILE__, __LINE__, "pop3_quit", NULL, _("select() error : %s"), strerror(errno)); reply = INCORRECT; } } #endif } else { reply = SUCCESS; } #ifdef WITH_SSL if (ssl_con != NULL) { SSL_free(ssl_con); ssl_con = NULL; } #endif if (close(pop3_fd) == -1) { trans_log(DEBUG_SIGN, __FILE__, __LINE__, "pop3_quit", NULL, _("close() error : %s"), strerror(errno)); } pop3_fd = -1; } else { reply = SUCCESS; } return(reply); }
/*############################# pop3_read() #############################*/ int pop3_read(char *block, int blocksize) { int bytes_read, status; fd_set rset; /* Initialise descriptor set. */ FD_ZERO(&rset); FD_SET(pop3_fd, &rset); timeout.tv_usec = 0L; timeout.tv_sec = transfer_timeout; /* Wait for message x seconds and then continue. */ status = select(pop3_fd + 1, &rset, NULL, NULL, &timeout); if ((status > 0) && (FD_ISSET(pop3_fd, &rset))) { #ifdef WITH_SSL if (ssl_con == NULL) { #endif if ((bytes_read = read(pop3_fd, &block[rb_offset], blocksize - rb_offset)) == -1) { if (errno == ECONNRESET) { timeout_flag = CON_RESET; } trans_log(ERROR_SIGN, __FILE__, __LINE__, "pop3_read", NULL, _("read() error : %s"), strerror(errno)); return(INCORRECT); } #ifdef WITH_SSL } else { if ((bytes_read = SSL_read(ssl_con, &block[rb_offset], blocksize - rb_offset)) == INCORRECT) { if ((status = SSL_get_error(ssl_con, bytes_read)) == SSL_ERROR_SYSCALL) { if (errno == ECONNRESET) { timeout_flag = CON_RESET; } trans_log(ERROR_SIGN, __FILE__, __LINE__, "pop3_read", NULL, _("SSL_read() error : %s"), strerror(errno)); } else { trans_log(ERROR_SIGN, __FILE__, __LINE__, "pop3_read", NULL, _("SSL_read() error %d"), status); } return(INCORRECT); } } #endif #ifdef WITH_TRACE trace_log(NULL, 0, BIN_R_TRACE, block, bytes_read, NULL); #endif /* * Whatch out for byte stuffing and maybe end of file * marker. */ if (bytes_read > 0) { char *p_end, *ptr; if (rb_offset > 0) { (void)memcpy(block, read_buffer, rb_offset); } ptr = block; p_end = block + rb_offset + bytes_read; while (ptr < p_end) { if ((*ptr == '\r') && ((p_end - ptr) > 3) && (*(ptr + 1) == '\n') && (*(ptr + 2) == '.')) { if (*(ptr + 3) == '.') { (void)memcpy((ptr + 3), (ptr + 4), (p_end - (ptr + 4))); p_end--; bytes_read--; ptr += 3; } else if ((*(ptr + 3) == '\r') && (*(ptr + 4) == '\n')) { } else { ptr += 2; } } else { ptr++; } } if (read_buffer[0] == '\0') { int i = 0; status = bytes_read - 1; while ((status >= 0) && (i < 3)) { read_buffer[i] = block[status]; i++; status--; } read_buffer[i] = '\0'; } else { } } } else if (status == 0) { /* Timeout has arrived. */ timeout_flag = ON; return(INCORRECT); } else { trans_log(ERROR_SIGN, __FILE__, __LINE__, "pop3_read", NULL, _("select() error : %s"), strerror(errno)); return(INCORRECT); } return(bytes_read); }
/*################## get_remote_file_names_ftp_list() ###################*/ int get_remote_file_names_ftp_list(off_t *file_size_to_retrieve, int *more_files_in_list) { int files_to_retrieve = 0, i = 0; *file_size_to_retrieve = 0; #ifdef DO_NOT_PARALLELIZE_ALL_FETCH if ((*more_files_in_list == YES) || (db.special_flag & DISTRIBUTED_HELPER_JOB) || ((db.special_flag & OLD_ERROR_JOB) && (db.retries < 30) && (fra[db.fra_pos].stupid_mode != YES) && (fra[db.fra_pos].remove != YES))) #else if (rl_fd == -1) { try_attach_again: if (attach_ls_data(db.fra_pos, db.fsa_pos, db.special_flag, YES) == INCORRECT) { (void)ftp_quit(); exit(INCORRECT); } if ((db.special_flag & DISTRIBUTED_HELPER_JOB) && ((fra[db.fra_pos].stupid_mode == YES) || (fra[db.fra_pos].remove == YES))) { # ifdef LOCK_DEBUG if (rlock_region(rl_fd, LOCK_RETR_PROC, __FILE__, __LINE__) == LOCK_IS_SET) # else if (rlock_region(rl_fd, LOCK_RETR_PROC) == LOCK_IS_SET) # endif { if (i == 0) { system_log(DEBUG_SIGN, __FILE__, __LINE__, "Hmm, lock is set. Assume ls_data file was just modified. Lets try it again. (job_no=%d fsa_pos=%d)", (int)db.job_no, db.fsa_pos); } else { if (i == 30) { trans_log(DEBUG_SIGN, __FILE__, __LINE__, NULL, NULL, "Have waited %d seconds, but unable to get a lock. Terminating.", (i * 100000) / 1000000); (void)ftp_quit(); exit(SUCCESS); } my_usleep(100000L); } detach_ls_data(NO); i++; goto try_attach_again; } } } if ((*more_files_in_list == YES) || (db.special_flag & DISTRIBUTED_HELPER_JOB) || ((db.special_flag & OLD_ERROR_JOB) && (db.retries < 30))) #endif { #ifdef DO_NOT_PARALLELIZE_ALL_FETCH if (rl_fd == -1) { if (attach_ls_data(db.fra_pos, db.fsa_pos, db.special_flag, YES) == INCORRECT) { (void)ftp_quit(); exit(INCORRECT); } } #endif *more_files_in_list = NO; for (i = 0; i < no_of_listed_files; i++) { if ((rl[i].retrieved == NO) && (rl[i].assigned == 0)) { #ifdef DO_NOT_PARALLELIZE_ALL_FETCH if ((fra[db.fra_pos].stupid_mode == YES) || (fra[db.fra_pos].remove == YES) || ((files_to_retrieve < fra[db.fra_pos].max_copied_files) && (*file_size_to_retrieve < fra[db.fra_pos].max_copied_file_size))) #else if ((files_to_retrieve < fra[db.fra_pos].max_copied_files) && (*file_size_to_retrieve < fra[db.fra_pos].max_copied_file_size)) #endif { /* Lock this file in list. */ #ifdef LOCK_DEBUG if (lock_region(rl_fd, (off_t)(LOCK_RETR_FILE + i), __FILE__, __LINE__) == LOCK_IS_NOT_SET) #else if (lock_region(rl_fd, (off_t)(LOCK_RETR_FILE + i)) == LOCK_IS_NOT_SET) #endif { if ((fra[db.fra_pos].ignore_size == -1) || ((fra[db.fra_pos].gt_lt_sign & ISIZE_EQUAL) && (fra[db.fra_pos].ignore_size == rl[i].size)) || ((fra[db.fra_pos].gt_lt_sign & ISIZE_LESS_THEN) && (fra[db.fra_pos].ignore_size < rl[i].size)) || ((fra[db.fra_pos].gt_lt_sign & ISIZE_GREATER_THEN) && (fra[db.fra_pos].ignore_size > rl[i].size))) { if ((rl[i].got_date == NO) || (fra[db.fra_pos].ignore_file_time == 0)) { files_to_retrieve++; if ((fra[db.fra_pos].stupid_mode == APPEND_ONLY) && (rl[i].size > rl[i].prev_size)) { *file_size_to_retrieve += (rl[i].size - rl[i].prev_size); } else { *file_size_to_retrieve += rl[i].size; } rl[i].assigned = (unsigned char)db.job_no + 1; } else { time_t diff_time; diff_time = current_time - rl[i].file_mtime; if (((fra[db.fra_pos].gt_lt_sign & IFTIME_EQUAL) && (fra[db.fra_pos].ignore_file_time == diff_time)) || ((fra[db.fra_pos].gt_lt_sign & IFTIME_LESS_THEN) && (fra[db.fra_pos].ignore_file_time < diff_time)) || ((fra[db.fra_pos].gt_lt_sign & IFTIME_GREATER_THEN) && (fra[db.fra_pos].ignore_file_time > diff_time))) { files_to_retrieve++; if ((fra[db.fra_pos].stupid_mode == APPEND_ONLY) && (rl[i].size > rl[i].prev_size)) { *file_size_to_retrieve += (rl[i].size - rl[i].prev_size); } else { *file_size_to_retrieve += rl[i].size; } rl[i].assigned = (unsigned char)db.job_no + 1; } } #ifdef DEBUG_ASSIGNMENT trans_log(DEBUG_SIGN, __FILE__, __LINE__, NULL, NULL, # if SIZEOF_OFF_T == 4 "%s assigned %d: file_name=%s assigned=%d size=%ld", # else "%s assigned %d: file_name=%s assigned=%d size=%lld", # endif (fra[db.fra_pos].ls_data_alias[0] == '\0') ? fra[db.fra_pos].dir_alias : fra[db.fra_pos].ls_data_alias, i, rl[i].file_name, (int)rl[i].assigned, (pri_off_t)rl[i].size); #endif /* DEBUG_ASSIGNMENT */ } #ifdef LOCK_DEBUG unlock_region(rl_fd, (off_t)(LOCK_RETR_FILE + i), __FILE__, __LINE__); #else unlock_region(rl_fd, (off_t)(LOCK_RETR_FILE + i)); #endif } } else { *more_files_in_list = YES; break; } } } } else { unsigned int files_deleted = 0, list_length = 0; int gotcha, j, k, nfg, /* Number of file mask. */ status, type; char file_name[MAX_FILENAME_LENGTH + 1], *list = NULL, *p_end, *p_mask, *p_start; time_t file_mtime; off_t file_size, file_size_deleted = 0, list_size = 0; struct ftpparse fp; struct file_mask *fml = NULL; struct tm *p_tm; /* * Get a directory listing from the remote site so we can see * what files are there. */ #ifdef WITH_SSL if (db.auth == BOTH) { type = LIST_CMD | BUFFERED_LIST | ENCRYPT_DATA; } else { #endif type = LIST_CMD | BUFFERED_LIST; #ifdef WITH_SSL } #endif if ((status = ftp_list(db.mode_flag, type, &list)) != SUCCESS) { trans_log(ERROR_SIGN, __FILE__, __LINE__, NULL, msg_str, "Failed to send LIST command (%d).", status); (void)ftp_quit(); exit(LIST_ERROR); } if (list != NULL) { /* Get all file masks for this directory. */ if ((j = read_file_mask(fra[db.fra_pos].dir_alias, &nfg, &fml)) == INCORRECT) { if (j == LOCKFILE_NOT_THERE) { system_log(ERROR_SIGN, __FILE__, __LINE__, "Failed to set lock in file masks for %s, because the file is not there.", fra[db.fra_pos].dir_alias); } else if (j == LOCK_IS_SET) { system_log(ERROR_SIGN, __FILE__, __LINE__, "Failed to get the file masks for %s, because lock is already set", fra[db.fra_pos].dir_alias); } else { system_log(ERROR_SIGN, __FILE__, __LINE__, "Failed to get the file masks for %s. (%d)", fra[db.fra_pos].dir_alias, j); } if (fml != NULL) { free(fml); } (void)ftp_quit(); exit(INCORRECT); } #ifdef DO_NOT_PARALLELIZE_ALL_FETCH if ((fra[db.fra_pos].stupid_mode == YES) || (fra[db.fra_pos].remove == YES)) { if (reset_ls_data(db.fra_pos) == INCORRECT) { (void)ftp_quit(); exit(INCORRECT); } } else { if (rl_fd == -1) { if (attach_ls_data(db.fra_pos, db.fsa_pos, db.special_flag, YES) == INCORRECT) { (void)ftp_quit(); exit(INCORRECT); } } } #else if (rl_fd == -1) { if (attach_ls_data(db.fra_pos, db.fsa_pos, db.special_flag, YES) == INCORRECT) { (void)ftp_quit(); exit(INCORRECT); } } if ((fra[db.fra_pos].stupid_mode == YES) || (fra[db.fra_pos].remove == YES)) { /* * If all files from the previous listing have been * collected, lets reset the ls_data structure or otherwise * it keeps on growing forever. */ # ifdef LOCK_DEBUG if (lock_region(rl_fd, LOCK_RETR_PROC, __FILE__, __LINE__) == LOCK_IS_NOT_SET) # else if (lock_region(rl_fd, LOCK_RETR_PROC) == LOCK_IS_NOT_SET) # endif { if (reset_ls_data(db.fra_pos) == INCORRECT) { (void)ftp_quit(); exit(INCORRECT); } } # ifdef LOCK_DEBUG unlock_region(rl_fd, LOCK_RETR_PROC, __FILE__, __LINE__); # else unlock_region(rl_fd, LOCK_RETR_PROC); # endif } #endif if ((fra[db.fra_pos].ignore_file_time != 0) || (fra[db.fra_pos].delete_files_flag & UNKNOWN_FILES)) { /* Note: FTP returns GMT so we need to convert this to GMT! */ current_time = time(NULL); p_tm = gmtime(¤t_time); current_time = mktime(p_tm); } /* * Evaluate the list from the LIST command. */ p_end = list; do { p_start = p_end; while ((*p_end != '\r') && (*p_end != '\n') && (*p_end != '\0')) { p_end++; } if ((ftpparse(&fp, &file_size, &file_mtime, p_start, p_end - p_start) == 1) && ((fp.flagtryretr == 1) && ((fp.name[0] != '.') || (fra[db.fra_pos].dir_flag & ACCEPT_DOT_FILES)))) { list_length++; list_size += file_size; if (fp.namelen < MAX_FILENAME_LENGTH) { /* Store file name */ (void)memcpy(file_name, fp.name, fp.namelen); file_name[fp.namelen] = '\0'; if (fra[db.fra_pos].dir_flag == ALL_DISABLED) { delete_remote_file(FTP, file_name, fp.namelen, #ifdef _DELETE_LOG DELETE_HOST_DISABLED, #endif &files_deleted, &file_size_deleted, file_size); } else { gotcha = NO; for (k = 0; k < nfg; k++) { p_mask = fml[k].file_list; for (j = 0; j < fml[k].fc; j++) { if ((status = pmatch(p_mask, file_name, NULL)) == 0) { if (check_list(file_name, file_size, file_mtime, &files_to_retrieve, file_size_to_retrieve, more_files_in_list) == 0) { gotcha = YES; } else { gotcha = NEITHER; } break; } else if (status == 1) { /* This file is definitly NOT wanted! */ /* Lets skip the rest of this group. */ break; } #ifdef SHOW_FILTER_MISSES if ((status == -1) || (fsa->debug > NORMAL_MODE)) { char tmp_mask[MAX_FILENAME_LENGTH]; if (expand_filter(p_mask, tmp_mask, time(NULL)) == YES) { trans_db_log(INFO_SIGN, __FILE__, __LINE__, NULL, "%s (%s) not fitting %s", p_mask, tmp_mask, file_name); } else { trans_db_log(INFO_SIGN, __FILE__, __LINE__, NULL, "%s not fitting %s", p_mask, file_name); } } #endif NEXT(p_mask); } if ((gotcha == YES) || (gotcha == NEITHER)) { break; } } if ((gotcha == NO) && (status != 0) && (fra[db.fra_pos].delete_files_flag & UNKNOWN_FILES)) { time_t diff_time = current_time - file_mtime; if ((fra[db.fra_pos].unknown_file_time == -2) || ((diff_time > fra[db.fra_pos].unknown_file_time) && (diff_time > DEFAULT_TRANSFER_TIMEOUT))) { delete_remote_file(FTP, file_name, fp.namelen, #ifdef _DELETE_LOG DEL_UNKNOWN_FILE, #endif &files_deleted, &file_size_deleted, file_size); } } } } else { (void)memcpy(file_name, fp.name, MAX_FILENAME_LENGTH); file_name[MAX_FILENAME_LENGTH] = '\0'; trans_log(ERROR_SIGN, __FILE__, __LINE__, NULL, NULL, "Remote file name `%s' is to long, it may only be %d bytes long.", file_name, MAX_FILENAME_LENGTH); } } while ((*p_end == '\r') || (*p_end == '\n')) { p_end++; } } while (*p_end != '\0'); free(list); /* Free file mask list. */ for (i = 0; i < nfg; i++) { free(fml[i].file_list); } free(fml); } if (files_deleted > 0) { trans_log(DEBUG_SIGN, NULL, 0, NULL, NULL, #if SIZEOF_OFF_T == 4 "%d files %ld bytes found for retrieving %s[%u files with %ld bytes in %s (deleted %u files with %ld bytes)]. @%x", #else "%d files %lld bytes found for retrieving %s[%u files with %lld bytes in %s (deleted %u files with %lld bytes)]. @%x", #endif files_to_retrieve, (pri_off_t)(*file_size_to_retrieve), (*more_files_in_list == YES) ? "(+) " : "", list_length, (pri_off_t)list_size, (db.target_dir[0] == '\0') ? "home dir" : db.target_dir, files_deleted, (pri_off_t)file_size_deleted, db.id.dir); } else { trans_log(DEBUG_SIGN, NULL, 0, NULL, NULL, #if SIZEOF_OFF_T == 4 "%d files %ld bytes found for retrieving %s[%u files with %ld bytes in %s]. @%x", #else "%d files %lld bytes found for retrieving %s[%u files with %lld bytes in %s]. @%x", #endif files_to_retrieve, (pri_off_t)(*file_size_to_retrieve), (*more_files_in_list == YES) ? "(+) " : "", list_length, (pri_off_t)list_size, (db.target_dir[0] == '\0') ? "home dir" : db.target_dir, db.id.dir); } /* * Remove all files from the remote_list structure that are not * in the current buffer. */ if ((fra[db.fra_pos].stupid_mode != YES) && (fra[db.fra_pos].remove == NO)) { int files_removed = 0, i; size_t move_size; for (i = 0; i < (no_of_listed_files - files_removed); i++) { if (rl[i].in_list == NO) { int j = i; while ((rl[j].in_list == NO) && (j < (no_of_listed_files - files_removed))) { j++; } if (j != (no_of_listed_files - files_removed)) { move_size = (no_of_listed_files - files_removed - j) * sizeof(struct retrieve_list); (void)memmove(&rl[i], &rl[j], move_size); } files_removed += (j - i); } } if (files_removed > 0) { int current_no_of_listed_files = no_of_listed_files; size_t new_size, old_size; no_of_listed_files -= files_removed; if (no_of_listed_files < 0) { system_log(DEBUG_SIGN, __FILE__, __LINE__, "Hmmm, no_of_listed_files = %d", no_of_listed_files); no_of_listed_files = 0; } if (no_of_listed_files == 0) { new_size = (RETRIEVE_LIST_STEP_SIZE * sizeof(struct retrieve_list)) + AFD_WORD_OFFSET; } else { new_size = (((no_of_listed_files / RETRIEVE_LIST_STEP_SIZE) + 1) * RETRIEVE_LIST_STEP_SIZE * sizeof(struct retrieve_list)) + AFD_WORD_OFFSET; } old_size = (((current_no_of_listed_files / RETRIEVE_LIST_STEP_SIZE) + 1) * RETRIEVE_LIST_STEP_SIZE * sizeof(struct retrieve_list)) + AFD_WORD_OFFSET; if (old_size != new_size) { char *ptr; ptr = (char *)rl - AFD_WORD_OFFSET; #ifdef DO_NOT_PARALLELIZE_ALL_FETCH if ((fra[db.fra_pos].stupid_mode == YES) || (fra[db.fra_pos].remove == YES)) { if ((ptr = realloc(ptr, new_size)) == NULL) { system_log(ERROR_SIGN, __FILE__, __LINE__, "realloc() error : %s", strerror(errno)); (void)ftp_quit(); exit(INCORRECT); } } else { #endif if ((ptr = mmap_resize(rl_fd, ptr, new_size)) == (caddr_t) -1) { system_log(ERROR_SIGN, __FILE__, __LINE__, "mmap_resize() error : %s", strerror(errno)); (void)ftp_quit(); exit(INCORRECT); } rl_size = new_size; #ifdef DO_NOT_PARALLELIZE_ALL_FETCH } #endif ptr += AFD_WORD_OFFSET; rl = (struct retrieve_list *)ptr; } *(int *)((char *)rl - AFD_WORD_OFFSET) = no_of_listed_files; } } } return(files_to_retrieve); }
/*############################# reset_fsa() #############################*/ void reset_fsa(struct job *p_db, int mode, int file_total_shown, off_t file_size_total_shown) { if ((fsa != NULL) && (p_db->fsa_pos != INCORRECT)) { if (gsf_check_fsa(p_db) != NEITHER) { if (mode & IS_FAULTY_VAR) { fsa->job_status[(int)p_db->job_no].connect_status = NOT_WORKING; } else { fsa->job_status[(int)p_db->job_no].connect_status = DISCONNECT; } fsa->job_status[(int)p_db->job_no].no_of_files_done = 0; fsa->job_status[(int)p_db->job_no].file_size_done = 0; fsa->job_status[(int)p_db->job_no].file_size_in_use = 0; fsa->job_status[(int)p_db->job_no].file_size_in_use_done = 0; fsa->job_status[(int)p_db->job_no].file_name_in_use[0] = '\0'; fsa->job_status[(int)p_db->job_no].file_name_in_use[1] = 0; fsa->job_status[(int)p_db->job_no].no_of_files = 0; fsa->job_status[(int)p_db->job_no].file_size = 0; if ((file_total_shown != 0) || (file_size_total_shown != 0)) { #ifdef LOCK_DEBUG lock_region_w(fsa_fd, p_db->lock_offset + LOCK_TFC, __FILE__, __LINE__); #else lock_region_w(fsa_fd, p_db->lock_offset + LOCK_TFC); #endif if (file_total_shown != 0) { fsa->total_file_counter -= file_total_shown; #ifdef _VERIFY_FSA if (fsa->total_file_counter < 0) { trans_log(DEBUG_SIGN, __FILE__, __LINE__, NULL, NULL, "Total file counter less then zero. Correcting to 0."); fsa->total_file_counter = 0; } #endif } if (file_size_total_shown != 0) { fsa->total_file_size -= file_size_total_shown; #ifdef _VERIFY_FSA if (fsa->total_file_size < 0) { trans_log(DEBUG_SIGN, __FILE__, __LINE__, NULL, NULL, "Total file size less then zero. Correcting to 0."); fsa->total_file_size = 0; } #endif } #ifdef LOCK_DEBUG unlock_region(fsa_fd, p_db->lock_offset + LOCK_TFC, __FILE__, __LINE__); #else unlock_region(fsa_fd, p_db->lock_offset + LOCK_TFC); #endif } } } return; }
/*########################## pop3_connect() #############################*/ int pop3_connect(char *hostname, int port) { int loop_counter = 0; struct sockaddr_in sin; register struct hostent *p_host = NULL; #ifdef FTX struct linger l; #endif (void)memset((struct sockaddr *) &sin, 0, sizeof(sin)); if ((sin.sin_addr.s_addr = inet_addr(hostname)) == -1) { if ((p_host = gethostbyname(hostname)) == NULL) { #if !defined (_HPUX) && !defined (_SCO) if (h_errno != 0) { #ifdef LINUX if ((h_errno > 0) && (h_errno < h_nerr)) { trans_log(ERROR_SIGN, __FILE__, __LINE__, "pop3_connect", NULL, _("Failed to gethostbyname() %s : %s"), hostname, h_errlist[h_errno]); } else { trans_log(ERROR_SIGN, __FILE__, __LINE__, "pop3_connect", NULL, _("Failed to gethostbyname() %s (h_errno = %d) : %s"), hostname, h_errno, strerror(errno)); } #else trans_log(ERROR_SIGN, __FILE__, __LINE__, "pop3_connect", NULL, _("Failed to gethostbyname() %s (h_errno = %d) : %s"), hostname, h_errno, strerror(errno)); #endif } else { #endif /* !_HPUX && !_SCO */ trans_log(ERROR_SIGN, __FILE__, __LINE__, "pop3_connect", NULL, _("Failed to gethostbyname() %s : %s"), hostname, strerror(errno)); #if !defined (_HPUX) && !defined (_SCO) } #endif return(INCORRECT); } /* Copy IP number to socket structure. */ memcpy((char *)&sin.sin_addr, p_host->h_addr, p_host->h_length); } if ((pop3_fd = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP)) < 0) { trans_log(ERROR_SIGN, __FILE__, __LINE__, "pop3_connect", NULL, _("socket() error : %s"), strerror(errno)); return(INCORRECT); } sin.sin_family = AF_INET; sin.sin_port = htons((u_short)port); #ifdef FTP_CTRL_KEEP_ALIVE_INTERVAL if (timeout_flag != OFF) { int reply = 1; if (setsockopt(pop3_fd, SOL_SOCKET, SO_KEEPALIVE, (char *)&reply, sizeof(reply)) < 0) { trans_log(WARN_SIGN, __FILE__, __LINE__, "pop3_connect", NULL, _("setsockopt() SO_KEEPALIVE error : %s"), strerror(errno)); } # ifdef TCP_KEEPALIVE reply = timeout_flag; if (setsockopt(pop3_fd, IPPROTO_IP, TCP_KEEPALIVE, (char *)&reply, sizeof(reply)) < 0) { trans_log(WARN_SIGN, __FILE__, __LINE__, "pop3_connect", NULL, _("setsockopt() TCP_KEEPALIVE error : %s"), strerror(errno)); } # endif timeout_flag = OFF; } #endif while (connect(pop3_fd, (struct sockaddr *) &sin, sizeof(sin)) < 0) { loop_counter++; if (loop_counter <= 8) { /* * Lets not give up to early. When we just have closed * the connection and immediatly retry, the other side * might not be as quick and still have the socket open. */ (void)sleep(1); } else { trans_log(ERROR_SIGN, __FILE__, __LINE__, "pop3_connect", NULL, _("Failed to connect() to %s, have tried %d times : %s"), hostname, loop_counter, strerror(errno)); (void)close(pop3_fd); pop3_fd = -1; return(INCORRECT); } if (close(pop3_fd) == -1) { trans_log(DEBUG_SIGN, __FILE__, __LINE__, "pop3_connect", NULL, _("close() error : %s"), strerror(errno)); } if ((pop3_fd = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP)) < 0) { trans_log(ERROR_SIGN, __FILE__, __LINE__, "pop3_connect", NULL, _("socket() error : %s"), strerror(errno)); (void)close(pop3_fd); return(INCORRECT); } } #ifdef FTX l.l_onoff = 1; l.l_linger = 240; if (setsockopt(pop3_fd, SOL_SOCKET, SO_LINGER, (char *)&l, sizeof(struct linger)) < 0) { trans_log(ERROR_SIGN, __FILE__, __LINE__, "pop3_connect", NULL, _("setsockopt() error : %s"), strerror(errno)); return(INCORRECT); } #endif if (get_reply() != POP3_OK) { (void)close(pop3_fd); return(INCORRECT); } return(SUCCESS); }